From c74b5aa162e220c4c9e175aeec449caf6f94be22 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 11:44:06 +0800 Subject: [PATCH 001/473] docs: plan full PostgreSQL migration --- docs/plans/2026-10-21-postgresql-migration.md | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 docs/plans/2026-10-21-postgresql-migration.md diff --git a/docs/plans/2026-10-21-postgresql-migration.md b/docs/plans/2026-10-21-postgresql-migration.md new file mode 100644 index 000000000..555cf5fda --- /dev/null +++ b/docs/plans/2026-10-21-postgresql-migration.md @@ -0,0 +1,213 @@ +# SQLite 全面迁移到 PostgreSQL + +日期：2026-10-21（北京时间） +调研基线：`eee8f6ecb6ca04455e6edef0c8e498fb3be21f21` +实施分支：`codex/postgresql-migration` +状态：计划已制定，业务代码尚未迁移；各任务完成后填写实际提交和验证结果。 + +## 1. 目标与边界 + +将控制面的运行数据库统一为 PostgreSQL，包括用户、API 密钥、分组、账号、调度状态、代理、集群节点、虚拟机凭据镜像、请求日志、计费、备份记录和思考签名归属。 + +应用运行、单元测试、端到端测试、运维查询和新备份都使用 PostgreSQL。SQLite 仅用于离线读取旧数据库、构造历史迁移测试数据及转换旧备份。运行时不保留双写或 SQLite 降级路径。 + +本分支完成代码、测试、迁移工具和部署说明。执行现有环境的数据迁移、停服、发布、修改现有服务配置和推送分支均不包含在本次本地实施范围内，需要相应操作的明确授权。 + +保留现有业务对象、权限边界、文件目录用途及对外 API 契约。`vms/*.json` 目前仍是 VM 运行操作的文件入口；本次不额外将整个 VM 文件模型改为数据库唯一存储。`VM2API_DB_SECRET` / `KIN_DB_SECRET` 仍是业务字段加密密钥，不能当作 PostgreSQL 登录密码，也不能在迁移时重新生成。 + +## 2. 已核实的现状 + +| 范围 | 现有实现 | 迁移影响 | +| --- | --- | --- | +| 数据库入口 | `src/lib/db/database.mjs` 使用 `node:sqlite.DatabaseSync`，默认 `/kin.db`，支持 `KIN_DB_PATH` | 打开、迁移、查询、事务和关闭均需异步化 | +| 数据访问 | `src/lib/db/repos/` 中 21 个模块，包含 20 个 repository/view 和 1 个共享 SQL 模块 | 调用者不能继续把查询结果当作同步值 | +| 版本迁移 | `src/lib/db/migrations/001_init.sql` 至 `041_usage_logs_inbound_device.sql`，逐文件事务和 SHA-256 校验 | PostgreSQL 建立独立迁移序列；原 SQLite 文件保持不变 | +| 最终表结构 | 在临时 SQLite 执行 41 个迁移后，有 29 张非 SQLite 内部表，包含 `schema_migrations` 和 3 张 `_legacy` 表 | 建立逐表清单，核对列、索引、默认值、外键及保留数据 | +| 额外表 | `src/lib/pool/thinking-owner.mjs` 在运行时创建 `thinking_signature_owners` | 纳入 PostgreSQL 正式迁移与存量导入 | +| VM 凭据镜像 | `src/lib/vm/vm-db-sync.mjs` 通过写钩子、启动校对及文件监听镜像到数据库 | 异步写入必须可等待、可排空，删除顺序不能改变 | +| 备份 | `src/lib/admin/backup-core.mjs` 使用 `VACUUM INTO`，归档含 `db/kin.db`、VM 和配置文件 | 更换数据库快照格式，保留原归档的其他内容 | +| 恢复 | `backup-service.mjs` 关闭 SQLite、替换文件、重新绑定 store | PostgreSQL 恢复需维护模式、排空写入及恢复后重新加载 | +| 恢复入口隔离 | `src/server.mjs` 的恢复标志目前主要拦截推理入口 | 需要同时控制面板和后台数据库写入 | +| 监控与界面 | `database-metrics.mjs` 和数据库页面展示 SQLite/WAL/PRAGMA/页信息 | 改为 PostgreSQL 数据库、连接池和迁移指标 | +| 部署 | 控制面 Compose 使用 `network_mode: host`；存在 amd64、arm64 两种镜像构建 | PostgreSQL 地址、网络暴露、客户端版本及两种构建都需覆盖 | +| 测试 | 临时目录隔离 SQLite，`test/harness.mjs` 派生网关进程 | 改用独立 PostgreSQL 测试数据库，并隔离每个测试实例 | + +当前 SQLite 表清单： + +```text +account_allocations, account_groups, account_runtime, +account_runtime_states_legacy, accounts, accounts_legacy, +api_endpoint_keys, api_endpoint_models, api_endpoints, api_keys, +backup_records, cluster_node_direct_links, cluster_nodes, groups, +panel_users_legacy, proxies, redeem_codes, refusal_device_blocks, +refusal_guards, request_attempts, request_log_debug, schema_migrations, +settings, sticky_sessions, usage_logs, users, vm_pool_members, vm_pools, vms +``` + +`thinking_signature_owners` 为以上清单之外的运行时表。数据导入不能仅按 migrations 中的建表语句搜索结果决定范围。 + +## 3. 实现方案 + +### 3.1 驱动、生命周期与事务 + +- 使用 `pg`（node-postgres）连接池和原生异步查询。SQL 参数使用 `$1`、`$2` 等占位符；仓库不增加运行时 SQLite SQL 翻译器或同步线程代理。 +- 连接配置沿用现有 `VM2API_*` 优先、`KIN_*` 作为 fallback 的规则：`VM2API_DATABASE_URL` 优先于 `KIN_DATABASE_URL`，仅接受 PostgreSQL 连接字符串。未配置或连接失败时，在监听 HTTP 端口之前失败。`KIN_DB_PATH` 仅保留在离线源文件工具中，应用不得悄悄回退到 SQLite。 +- `await openDatabase()` 完成连接探针、迁移和必要初始化，再创建或初始化 store。构造函数不执行未等待的异步数据库写入；需要初始化时使用显式 `init()` 或工厂，并在启动阶段等待。 +- `await closeDatabase()` 排空连接池。关闭、恢复和测试清理必须等待正在执行的操作结束。 +- `withTransaction` 等待回调完成后提交。一个事务中的查询固定使用同一个连接，嵌套调用沿用同一事务。实现时选择显式传递事务对象；调用层级确有必要时再使用 `AsyncLocalStorage`，不同时维护两套事务机制。 +- 迁移使用数据库级 advisory lock 串行化并发启动。迁移锁、SQL 与迁移记录必须在同一个专用连接执行；连接在成功和异常路径都释放。 +- SQL 错误不得打印连接字符串、密码、参数值或凭据列。事务中不得沿用 SQLite 的“捕获单行错误后继续执行”方式；可预先校验的数据先校验，必须局部容错的操作使用 savepoint。 + +### 3.2 表结构和数据契约 + +- PostgreSQL 基线按当前 SQLite 的最终结构建立，不在 PostgreSQL 上重放 41 个旧迁移。新旧 `schema_migrations` 分别记录各自引擎的版本，不把 SQLite 校验和当作 PostgreSQL 校验和。 +- 基线包含 `thinking_signature_owners`。3 张 `_legacy` 表中的历史记录也纳入显式保留清单，不静默丢弃。旧 SQLite 的迁移记录写入导入清单作为源版本证据。 +- JSON 和 ISO 时间先保留为 `text`；整数布尔值保持 `0/1`。本次不混入 `jsonb`、时间输出格式或业务字段加密格式的改变。 +- 原 SQLite `REAL` 按其双精度语义映射到 `double precision`。不在同一次迁移中改变计费舍入规则。余额和费用使用固定测试数据比较迁移前后结果。 +- SQLite `INTEGER` 是 64 位。毫秒时间戳、token 计数、累计次数、字节数等不能机械改为 PostgreSQL 32 位 `integer`；逐列选择 `bigint` 或有界 `integer`。统一处理 `pg` 返回的 bigint/count/sum，超出 JavaScript 安全整数时不得静默截断。 +- 自增列使用 identity/sequence，并用 `INSERT ... RETURNING id` 取值。固定分组种子和导入的显式 ID 完成后更新 sequence。 +- 保留现有外键和级联语义。本次不额外为旧数据增加未评估的业务约束。 +- 用户名和 VM 池名称保持现有 SQLite `NOCASE` 行为。其默认大小写折叠主要针对 ASCII，不能直接用 Unicode `lower()` 宣称等价。实现可用固定 ASCII 大小写映射表达式及一致的唯一索引；登录查询与索引必须采用同一规则，并测试非 ASCII 名称。 +- PostgreSQL 新空库包含默认分组；存量导入时应在同一事务中移除基线种子并导入源分组，不能因为种子存在而绕过“目标为空”的检查。 + +### 3.3 SQL 与异步调用链 + +| SQLite 用法 | PostgreSQL 实现要求 | +| --- | --- | +| `?` | 调用处显式使用 `$n`，保持参数化 | +| `IFNULL` | 改为 `COALESCE`，核对聚合返回类型 | +| `LIKE` | 日志搜索保留默认 ASCII 大小写不敏感和反斜杠字面量语义；两侧使用一致 ASCII fold 并显式 `ESCAPE ''`，对照验证 `%` / `_` 通配行为，不直接将 `ILIKE` 当作等价替代 | +| `INSERT OR IGNORE` | 改为带实际冲突目标的 `ON CONFLICT DO NOTHING`，不吞掉其他约束错误 | +| `lastInsertRowid` | `RETURNING id` | +| `strftime` / `printf` | PostgreSQL 时间或字符串表达式；保持 UTC、上海自然日和 IANA 时区/夏令时行为 | +| `json_each` | 原生数组参数及 `ANY`，或显式 JSON 数组展开 | +| `rowid` 分页 | 稳定业务主键的 keyset pagination，不使用 `ctid` | +| 宽松 `GROUP BY` | 选择列与分组表达式一致；重点修正 statistics leaderboard | +| 浮点 `CAST(... AS INTEGER)` | 明确截断或舍入，不能依赖两种引擎的隐含差异 | +| PRAGMA / `sqlite_master` | 正式迁移、PostgreSQL catalog 或 information_schema | + +新增 `await` 后还要检查下列语义： + +1. `ApiKeyStore` 的鉴权、额度和并发准入：数据库读取完成后，最后的内存限流检查与 inflight 占位连续执行；失败路径释放占位。 +2. 账号状态、额度、余额等读改写：改为原子 UPDATE 或行锁事务，避免两个 await 路径相互覆盖。兑换码保持一次使用与余额更新的原子性。 +3. 同步配置回调：在异步边界预取，或复用已有配置的内存状态并在启动/修改/恢复时等待更新，不能把 Promise 当配置对象。 +4. 定时任务、事件监听、数组遍历和流式响应结束回调：明确等待或排队，禁止 `forEach(async ...)` 等未跟踪写入。 +5. VM 写钩子：保留现有文件入口，提供有顺序的数据库镜像写入与 flush；读取镜像、备份、删除、恢复和关闭前等待相关写入。失败不能报告镜像成功；删除后不能被迟到 upsert 或启动恢复重新创建。 + +### 3.4 存量迁移 + +提供显式离线命令，暂定 `npm run db:import:sqlite -- --source `。目标连接只从环境读取，命令行不携带密码。应用启动时不自动发现或导入旧文件。 + +1. 切换操作前冻结原应用所有写入，排空在途请求和后台任务。以 SQLite 一致性快照和文件归档作为输入，不直接复制正在写入的 `.db` 主文件而遗漏 WAL。 +2. 原 SQLite 文件只读保留。历史版本升级仅在工具创建的临时副本执行，复用原校验和及历史 001 schema 修复逻辑；不就地修改原库。 +3. 识别旧版本和未知表/列。未知版本、校验和错误或存在未映射业务数据时拒绝导入，给出不含业务行的诊断。额外历史表和运行时表按清单迁移。 +4. 校验目标为新基线且没有业务数据。导入使用一个事务并互斥执行，按依赖顺序导入；对集群节点自引用关系采用父节点优先顺序或可延迟约束，不能靠关闭约束忽略问题。 +5. 保留主键、NULL、租户归属、密文、JSON 字符串、时间文本和业务值。SQLite 源查询启用 `statement.setReadBigInts(true)`，超出 JavaScript 安全整数的值用无损十进制字符串绑定 PG、计算摘要和序列化，不能先转成 `Number`。保持原业务加密密钥，禁止将凭据解密后输出或重写。 +6. 更新 sequence 时取源 `sqlite_sequence.seq` 和最大现存 ID 的较大值，保留已经删除记录的自增高水位。采用可回滚的 sequence restart；若使用不可回滚的 `setval`，必须保存并恢复失败前序列状态并测试。逐表比较行数、主键集合和按稳定主键顺序计算的规范化内容摘要；另核对余额、费用、token 合计及外键关系。浮点聚合约定容差，逐行 double precision 应保留相同数值。 +7. 校验通过后，将源版本、源文件摘要、逐表验证结果和完成标记与数据一并提交。失败整体回滚；不能留下“已导入”标记或半成品。再次执行对已导入目标明确拒绝。 +8. 保留输入快照、文件归档和不含敏感值的结果清单。目标开始接受新写入前可以回到原 SQLite；目标接受写入后，原库已过时，不能直接切回并声称无数据丢失。 + +### 3.5 备份、恢复与运维 + +- 新备份格式使用 `pg_dump` custom archive，并继续归档 VM JSON、现有配置和清单。清单写明备份格式、数据库引擎、schema 版本和文件摘要，不包含连接凭据。 +- 数据库快照与 VM/配置文件必须有明确的一致性边界。暂停应用的数据库与相关文件写入、排空 VM 镜像队列，在专用事务中用 `pg_export_snapshot()` 建立快照，并在该冻结窗口复制文件。随后恢复写入，用 `pg_dump --snapshot` 导出；导出事务保持到 dump 完成为止。子进程已启动不代表快照已建立。不能把独立时刻的数据库 dump 与凭据文件称为一致快照。 +- 备份在异步子进程/现有 worker 执行，保证 HTTP 事件循环可响应。PostgreSQL 客户端主版本与支持的服务端版本对齐；两种 Debian Bookworm 镜像通过官方 PGDG 来源安装 `postgresql-client-17`，验证 amd64、arm64 的 `pg_dump` / `pg_restore` 主版本。不能只安装发行版默认客户端而得到 PG 15。 +- 新版恢复显式进入维护模式：拒绝新增推理和面板写入，暂停后台写任务，排空请求、事务及 VM 镜像队列，再制作恢复前备份。维护操作使用受控内部入口，避免被自身写入门禁止。 +- 恢复实现先在隔离测试数据库验证。实际恢复前检查归档校验和与 `pg_restore --list` 的对象清单，只允许应用所属 schema 和约定对象；目标恢复使用单事务的受限操作，不能通过删除整个数据库扩大权限。数据库恢复与文件替换不是跨资源原子事务，文件提前校验并准备临时目录；切换失败时使用恢复前快照回退，未恢复一致前保持维护状态。 +- 生产应用使用专属数据库和固定应用 schema（默认 `public`）。数据库查询、备份 worker 和恢复子进程均传播同一目标及 schema；`pg_dump` 显式限定应用 schema，`search_path` 不代替备份范围限制。旧归档 schema 与目标不匹配时拒绝直接恢复，通过明确转换入口处理，不用字符串替换改写 dump。 +- 恢复成功必须等待 store、分组、API 调度、路由、监控、账号运行状态和 VM 镜像重新绑定/加载，随后才退出维护模式。恢复失败不能沿用无条件 `finally` 清除维护标志的行为。 +- 保留 `_registerOrphanArchives` 的归档重新登记行为。恢复后仍可列出恢复前备份和被恢复的归档，并能从面板发起撤销恢复；数据库记录恢复不等于归档文件自动恢复。 +- 旧 `.tar.gz` 的 `db/kin.db` 及旧 `.db.gz` 提供显式转换入口，复用离线导入和校验逻辑。旧记录中的文件路径不会因为迁移到 PG 自动恢复文件；保留原备份目录并测试记录与文件关系。 +- 数据库 API 和页面改用健康探针、数据库大小、连接池 total/idle/waiting、迁移状态。字段不暴露数据库地址、用户名、密码或连接字符串。 +- Compose 提供持久化 PostgreSQL 服务及健康检查。控制面保留 host 网络时，PG 仅绑定宿主 `127.0.0.1` 的可配置端口，连接 URL 与此一致；支持外部 PostgreSQL，不能硬编码容器服务名为 host 网络下的地址。 +- 更新 `.env.example`、安装脚本、容器入口、systemd 示例、部署说明和 ARM64 测试，处理旧安装升级后缺少数据库连接的情况。`scripts/panel-upgrade.sh` 必须在替换旧容器之前预检 PG 连接、迁移完成标记及所需 Compose 服务；其现有 `--no-deps` 升级不会自动创建 PG 服务，镜像分支也不会刷新 Compose，必须显式覆盖。未迁移的 SQLite 安装在替换容器之前拒绝升级。启动也应拒绝将存在旧 SQLite 数据却没有导入标记的安装作为新空 PG 安装提供服务。 +- `deploy/init-arm64-env.py`、`test/deploy/smoke_arm64_image.py` 和调用它的 test/release workflow 一并更新。现有 ARM64 smoke 使用 `--network none`，迁移后改为专用测试网络和临时 PG 17，并清理数据库、网络和容器；不连接现有服务或扩大 Docker socket 权限。 + +## 4. 实施任务与提交顺序 + +| 任务 | 依赖 | 文件范围 | 交付与验收 | +| --- | --- | --- | --- | +| T0 结构和行为基线 | 无 | 本计划、`test/fixtures/`、必要的只读测试辅助工具 | 29 表及动态表清单、索引/约束/type 映射、当前/历史 SQLite fixture、计费与时区对照结果；fixtures 只含合成数据 | +| T1 PostgreSQL 基础 | T0 | `package.json`、lock、`src/lib/db/database.mjs`、PG 新迁移目录、数据库测试辅助工具 | 异步连接池、初始化/关闭、校验和、迁移锁、单连接事务；先在隔离 PG 测试中覆盖错误路径 | +| T2 基础 repository | T1 | `src/lib/db/repos/` 中非日志统计模块及对应测试 | 用户/分组/密钥/账号/代理/端点/集群/VM/账号池/拒绝记录/兑换码异步 SQL；唯一性、外键和并发测试通过 | +| T3 日志、统计和计费 | T1 | `usage-logs-repo.mjs`、`usage-logs-view.mjs`、`statistics-repo.mjs`、`usage-log-preds.mjs`、`admin/request-log.mjs`、`admin/error-class.mjs` 及测试 | 原生 PG SQL、稳定重定价分页、时区分桶、权限筛选、费用/余额及 retention 结果与基线一致 | +| T4 store 和业务调用链 | T2、T3 | `src/server.mjs`、`src/lib/admin/` 的相关 store/路由、`src/lib/pool/`、`src/lib/protocol/`、`src/lib/vm/` 相关调用者 | 启动到关闭全链路 await；内存并发准入、配置、思考签名、流式结束日志和 VM 镜像语义验证 | +| T5 旧数据与备份转换工具 | T0、T1、T2、T3 | `src/lib/db/legacy-import.mjs`、SQLite 专用离线模块、`scripts/` 迁移命令、迁移测试 | 源只读、历史临时升级、空目标检查、事务导入、校验、sequence、重复执行和失败回滚；包含 JSON 旧数据导入 | +| T6 备份与恢复 | T4、T5 | `backup-core.mjs`、`backup-worker.mjs`、`backup-service.mjs`、server 恢复回调和测试 | PG 备份、全写入口维护隔离、数据库和文件一致性、异步重新绑定、旧归档转换及失败回退 | +| T7 运维与界面 | T1、T4、T6 | `database-metrics.mjs`、数据库 API/类型/页面、4 个 audit 脚本、Compose、两种 Dockerfile、安装/入口/升级脚本、`deploy/init-arm64-env.py`、`test/deploy/smoke_arm64_image.py`、`test/unit/panel-upgrade-script.test.mjs`、部署文档 | PG 指标、无敏感信息的配置、amd64/arm64 构建、旧安装升级前检查；数据库页面同窗口宽度截图对照 | +| T8 全量回归与迁移演练 | T2–T7 | `test/harness.mjs`、相关单元/E2E、`.github/workflows/test.yml`、`.github/workflows/release.yml`、本计划验证记录 | CI 使用真实隔离 PG，当前/历史源导入、新请求持久化、重启、备份恢复及失败演练全部记录实际结论 | + +T2/T3 可按文件范围并行。T5 在数据库契约稳定后并行。T4 与 T6 都会修改 server，必须由同一集成负责人顺序合入。数据库基础与所有调用者异步化完成前，分支不作为可部署版本。 + +每个任务独立提交。若实施中复现现有缺陷，低成本回归测试先单独提交，再提交修复。每次验证记录实际 commit SHA；后续代码改动后重新执行受影响的检查。 + +## 5. 验证方案与完成条件 + +### 5.1 隔离环境 + +选 PostgreSQL 17 作为首次实现和 CI 的固定主版本，匹配相同主版本客户端。使用专用本地测试数据库和合成数据，不读取现有部署的凭据或业务数据。测试配置独立于生产配置，名字与连接目标都必须通过 `_test` / 专用测试库保护检查。 + +普通测试文件和派生网关实例有独立 schema；测试辅助工具向所有连接和子进程传播固定 search_path，并在结束后关闭连接再删除对应 schema。备份/恢复、数据库关闭和恢复维护模式测试使用独立的专用 `_test` 数据库，应用 schema 与归档保持一致。`pg_restore` 不承担跨 schema 重命名。测试用账号的建库权限仅限隔离测试环境，不作为生产账号要求。只清理由本测试实例创建的 schema/数据库。集成测试缺少 PG 时明确失败或由单独命令报告缺失，不静默改用 SQLite。 + +### 5.2 必须通过的场景 + +| 范围 | 场景 | +| --- | --- | +| 初始化 | 空库、重复启动、两个进程并发迁移、校验和不匹配、失败 SQL 回滚、连接不可用、关闭释放连接 | +| 数据导入 | 当前 41 版与有覆盖的历史源、历史 001 proxy schema、旧 JSON 导入、3 张 legacy 表和动态表、NULL/密文/大整数、未知结构拒绝、失败回滚、非空目标拒绝、sequence 后续插入 | +| 用户和权限 | 登录、大小写唯一性/非 ASCII、停用与软删除、租户隔离、API 密钥权限和额度、VM 池成员隔离 | +| 并发 | 同时兑换同一码仅一次成功；余额和用量更新不丢失；同一 key 的并发准入不超限；事务失败无部分更新；VM 写入与删除不乱序 | +| 正常请求 | 通过现有 mock 上游的正常 HTTP 推理入口产生请求、日志和账单；包含流式/非流式、失败、重试、思考签名归属；重启后再查询持久化结果 | +| 统计 | UTC、Asia/Shanghai、含夏令时的 IANA 时区，零值/NULL，租户筛选，leaderboard，分页和全量重定价无遗漏重复，费用与 token 合计 | +| VM | 凭据文件变更镜像、外部文件变更、启动恢复、删除与在途写入、镜像失败、备份/恢复前 flush | +| 备份恢复 | 手动/定时备份、校验与下载、保留策略、恢复前备份、旧归档转换、损坏文件拒绝、恢复中所有写入口受控、恢复失败维护状态、恢复后业务请求 | +| 监控和部署 | PG 指标类型、连接信息脱敏、测试隔离清理、amd64/arm64 镜像和启动、Compose 内置 PG 与外部 PG 配置 | + +重点保留并迁移的现有测试包括： + +```text +test/unit/database.test.mjs +test/unit/db-migration-sub2api.test.mjs +test/unit/legacy-import.test.mjs +test/unit/database-metrics.test.mjs +test/unit/backup-service.test.mjs +test/unit/statistics-repo.test.mjs +test/unit/usage-logs-repo.test.mjs +test/unit/tenant-billing.test.mjs +test/e2e/persistence.e2e.test.mjs +test/e2e/backup.e2e.test.mjs +test/e2e/panel-users.e2e.test.mjs +``` + +### 5.3 验收门槛 + +1. `npm run test:unit` 与 `npm run test:e2e` 在隔离 PostgreSQL 上通过；记录通过/失败/跳过的实际数目和原因。只在离线源 fixture 中保留 SQLite 测试。 +2. `npm run test:go`、api-kernel 现有 Go 测试、前端测试/构建和相关格式、语法检查通过。运行所需的现有预编译 payload 需按仓库脚本准备；缺失时记录未验证，不把依赖缺失误报为数据库通过。 +3. 数据库页面与现有页面各提供一张相同窗口宽度的截图；保留现有组件和布局，只调整引擎相关信息。 +4. 当前/历史 SQLite 迁移演练、迁移回滚、PG 备份恢复演练以及正常 HTTP 请求持久化均通过，不能以连接成功代替业务验证。 +5. 检索 `node:sqlite`、`KIN_DB_PATH`、`VACUUM INTO`、`sqlite_master`、`PRAGMA`、`rowid`、SQLite 方言及旧同步调用。每个剩余命中都解释为离线兼容、历史迁移或历史文档；运行路径不得依赖它们。 +6. 报告标明被验证 SHA。若获准部署，用新行为和运行镜像 digest 验证实际版本；版本号或进程上报 SHA 不作为唯一依据。 + +## 6. 调研验证记录 + +2026-10-21，对源码基线 `eee8f6ecb6ca04455e6edef0c8e498fb3be21f21` 执行： + +```sh +node --test test/unit/database.test.mjs test/unit/db-migration-sub2api.test.mjs test/unit/database-metrics.test.mjs +``` + +结果：3 个测试文件通过，0 个失败，0 个跳过。此结果只证明现有 SQLite 基线的上述测试文件通过，不构成 PostgreSQL 验证。 + +使用临时目录执行全部现有迁移并只读查询 schema：41 个迁移，29 张非 SQLite 内部表。临时目录已删除；未读取现有业务数据库。 + +本机 Node.js 为 `v24.19.0`。调研时 `psql`、`postgres`、`initdb` 不在 PATH；Docker 查询返回 `permission denied while trying to connect to the docker API at unix:///var/run/docker.sock`。PostgreSQL 运行、迁移、备份恢复及业务验证均未执行。实施前需提供有权限的隔离测试环境，不能据此推断 PostgreSQL 代码或业务失败。 + +参考官方资料： + +- [node-postgres：事务必须使用同一 client](https://node-postgres.com/features/transactions) +- [node-postgres：连接池](https://node-postgres.com/apis/pool) +- [PostgreSQL 17：pg_dump](https://www.postgresql.org/docs/17/app-pgdump.html) +- [PostgreSQL 17：pg_restore](https://www.postgresql.org/docs/17/app-pgrestore.html) +- [PostgreSQL：Debian PGDG 客户端安装](https://www.postgresql.org/download/linux/debian/) + +计划已由两个只读审查任务分别核对数据层和运维/测试范围。已补充 schema 备份隔离、升级前检查、导出快照同步、客户端主版本、LIKE 语义、大整数源读取和自增高水位要求。该审查仅验证计划覆盖与源码事实，不代表迁移实现已经通过验证。 From fa0decc2253106d1fe6f64ecd1d9d38779c61694 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:15:10 +0800 Subject: [PATCH 002/473] feat(db): add asynchronous PostgreSQL persistence and baseline --- package-lock.json | 147 ++++ package.json | 3 +- src/lib/db/database.mjs | 473 ++++++------- .../db/postgres-migrations/001_baseline.sql | 646 ++++++++++++++++++ src/lib/db/sqlite-legacy.mjs | 273 ++++++++ test/unit/database.test.mjs | 531 ++++---------- test/unit/db-migration-sub2api.test.mjs | 2 +- test/unit/sqlite-legacy.test.mjs | 417 +++++++++++ 8 files changed, 1871 insertions(+), 621 deletions(-) create mode 100644 src/lib/db/postgres-migrations/001_baseline.sql create mode 100644 src/lib/db/sqlite-legacy.mjs create mode 100644 test/unit/sqlite-legacy.test.mjs diff --git a/package-lock.json b/package-lock.json index feb8b62ab..5192f0966 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "croner": "^10.0.1", "https-proxy-agent": "^9.1.0", "node-fetch": "^2.7.0", + "pg": "^8.23.1", "socks-proxy-agent": "^10.1.0", "ssh2": "^1.17.0", "ws": "^8.22.0" @@ -322,6 +323,134 @@ } } }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/proxy-agent-negotiate": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-agent-negotiate/-/proxy-agent-negotiate-1.1.0.tgz", @@ -383,6 +512,15 @@ "node": ">= 20" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/ssh2": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", @@ -448,6 +586,15 @@ "optional": true } } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/package.json b/package.json index e226872f2..a72b1d4bc 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "node-fetch": "^2.7.0", "socks-proxy-agent": "^10.1.0", "ssh2": "^1.17.0", - "ws": "^8.22.0" + "ws": "^8.22.0", + "pg": "^8.23.1" }, "devDependencies": { "@biomejs/biome": "2.5.11" diff --git a/src/lib/db/database.mjs b/src/lib/db/database.mjs index 4d2f5ca66..3566279ef 100644 --- a/src/lib/db/database.mjs +++ b/src/lib/db/database.mjs @@ -1,272 +1,279 @@ -/** - * SQLite database layer — sub2api data-layer inspired, on node:sqlite. - * - * - Single-process gateway → embedded SQLite (WAL) instead of sub2api's - * PostgreSQL+Redis. Same patterns: versioned SQL migrations with SHA-256 - * checksum verification (sub2api migrations_runner), repository modules, - * settings table, backup records. - * - Zero new dependencies: uses the built-in `node:sqlite` DatabaseSync. - * - * Env: - * KIN_DB_PATH — override db file path (default /kin.db) - */ - +/** PostgreSQL persistence. SQLite is supported only by the offline importer. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' import { fileURLToPath } from 'node:url' -import { DatabaseSync } from 'node:sqlite' +import pg from 'pg' -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const MIGRATIONS_DIR = path.join(__dirname, 'migrations') +const MIGRATIONS_DIR = fileURLToPath(new URL('./postgres-migrations/', import.meta.url)) +const transactions = new AsyncLocalStorage() +const maintenance = new AsyncLocalStorage() +let singleton = null +let opening = null -function sha256Hex(s) { - return crypto.createHash('sha256').update(s).digest('hex') +export function quoteIdentifier(value) { + return '"' + String(value).replaceAll('"', '""') + '"' } -/** LF-canonical hash plus CRLF/raw variants so Windows-applied checksums still match. */ -function migrationChecksums(sql) { - const lf = sql.replace(/\r\n/g, '\n') - const crlf = lf.replace(/\n/g, '\r\n') - return { - canonical: sha256Hex(lf), - variants: new Set([sha256Hex(sql), sha256Hex(lf), sha256Hex(crlf)]), +export function resolveDatabaseUrl({ connectionString } = {}) { + const value = connectionString || process.env.VM2API_DATABASE_URL || process.env.KIN_DATABASE_URL + if (!value || !/^postgres(?:ql)?:\/\//.test(value)) { + throw new Error('PostgreSQL requires VM2API_DATABASE_URL or KIN_DATABASE_URL') } + return value } -// 001 was published briefly before the proxy pool switched to its persisted -// record shape. Its checksum cannot be treated as the current 001: migration -// 018 assumes the newer columns and otherwise fails after accepting the hash. -const HISTORICAL_001_PROXY_CHECKSUMS = new Set([ - '376c686118bbd8aaa6db6ea13fb2ff0afecc0c7464bc3d668743a55c9ccafdde', - 'd6769ea311718b5af91e47295f97c3bf2722ece6d26acee7e89df84b88599ae2', -]) - -function reconcileHistorical001ProxySchema(db, checksum) { - if (!HISTORICAL_001_PROXY_CHECKSUMS.has(checksum)) return false - const columns = new Set( - db - .prepare('PRAGMA table_info(proxies)') - .all() - .map((row) => row.name), - ) - const current = ['raw', 'enabled', 'consecutive_failures', 'last_error'].every((name) => columns.has(name)) - if (current && !columns.has('url')) return true - const historical = ['url', 'failures', 'last_probe_ok', 'disabled_reason', 'updated_at'].every((name) => - columns.has(name), - ) - if (!historical || columns.has('raw')) { - throw new Error('historical migration 001 proxy schema is not recognized') - } - - withTransaction(db, () => { - db.exec(` - ALTER TABLE proxies RENAME TO proxies_historical_001; - CREATE TABLE proxies ( - id TEXT PRIMARY KEY, - scheme TEXT DEFAULT 'socks5', - host TEXT, - port INTEGER, - username TEXT, - password TEXT, - raw TEXT, - status TEXT DEFAULT 'unknown', - enabled INTEGER DEFAULT 1, - bound_vm_id TEXT, - consecutive_failures INTEGER DEFAULT 0, - latency_ms INTEGER, - last_probe_at TEXT, - last_error TEXT, - created_at TEXT - ); - INSERT INTO proxies ( - id, scheme, host, port, username, password, raw, status, enabled, - bound_vm_id, consecutive_failures, latency_ms, last_probe_at, - last_error, created_at - ) - SELECT - id, scheme, host, port, username, password, url, - CASE - WHEN status IN ('unknown', 'ok', 'dead') THEN status - WHEN last_probe_ok = 1 THEN 'ok' - WHEN last_probe_ok = 0 THEN 'dead' - ELSE 'unknown' - END, - CASE WHEN status = 'disabled' THEN 0 ELSE 1 END, - bound_vm_id, COALESCE(failures, 0), latency_ms, last_probe_at, - disabled_reason, created_at - FROM proxies_historical_001; - DROP TABLE proxies_historical_001; - `) - }) - return true +function safeInteger(value) { + const n = Number(value) + return Number.isSafeInteger(n) ? n : value } -let _db = null -let _dbPath = null -export function resolveDbPath({ dataDir, dbPath } = {}) { - if (dbPath) return dbPath - if (process.env.KIN_DB_PATH) return process.env.KIN_DB_PATH - const dir = dataDir || path.join(process.cwd(), 'data') - return path.join(dir, 'kin.db') +function databaseError(error, operation = 'query') { + const out = new Error(`PostgreSQL ${operation} failed (${error.code || 'database_error'})`) + out.code = error.code || 'database_error' + if (error.constraint) out.constraint = error.constraint + return out } -/** - * Standalone (non-singleton) open: PRAGMAs + migrations applied. - * Used by unit tests and by stores constructed without a global db. - * @returns {DatabaseSync} - */ -export function createDatabase({ dataDir, dbPath, migrationsDir } = {}) { - const file = resolveDbPath({ dataDir, dbPath }) - fs.mkdirSync(path.dirname(file), { recursive: true }) - const db = new DatabaseSync(file) - try { - fs.chmodSync(file, 0o600) - } catch {} - db.exec('PRAGMA journal_mode = WAL') - db.exec('PRAGMA synchronous = NORMAL') - db.exec('PRAGMA busy_timeout = 5000') - db.exec('PRAGMA foreign_keys = ON') - applyMigrations(db, { migrationsDir }) - return db -} +export class PostgresDatabase { + constructor({ connectionString, schema = 'public', max = 10 } = {}) { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(schema)) throw new Error('invalid database schema') + this.connectionString = resolveDatabaseUrl({ connectionString }) + this.schema = schema + this.pool = new pg.Pool({ + connectionString: this.connectionString, + max, + connectionTimeoutMillis: 5000, + idleTimeoutMillis: 10000, + statement_timeout: 60000, + options: `-c search_path=${quoteIdentifier(schema)},pg_catalog -c timezone=UTC`, + types: { + getTypeParser(oid, format) { + if (format !== 'binary' && (oid === 20 || oid === 1700)) { + return (value) => (/^-?\d+$/.test(value) ? safeInteger(value) : Number(value)) + } + return pg.types.getTypeParser(oid, format) + }, + }, + }) + this.closed = false + this.active = 0 + this.maintaining = false + this.waiters = [] + // Idle connection errors must not become uncaught EventEmitter errors. + this.pool.on('error', () => {}) + } -/** - * Open (or return the already-open) singleton database. - * @returns {DatabaseSync} - */ -export function openDatabase({ dataDir, dbPath, migrationsDir } = {}) { - if (_db) return _db - _db = createDatabase({ dataDir, dbPath, migrationsDir }) - _dbPath = resolveDbPath({ dataDir, dbPath }) - return _db -} + get isTransaction() { + return transactions.getStore()?.db === this + } -/** - * Resolve the db a store should use: explicit db → global singleton → - * standalone connection rooted at dataDir (test isolation path). - */ -export function resolveStoreDb({ db, dataDir } = {}) { - if (db) return db - if (_db) return _db - return createDatabase({ dataDir }) -} + async query(sql, params = []) { + if (this.closed) throw new Error('database is closed') + const tx = transactions.getStore() + if (this.maintaining && maintenance.getStore() !== this && tx?.db !== this) { + throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) + } + this.active += 1 + try { + return await (tx?.db === this ? tx.client : this.pool).query(sql, params) + } catch (error) { + throw databaseError(error) + } finally { + this.active -= 1 + if (!this.active) for (const resolve of this.waiters.splice(0)) resolve() + } + } -/** Current db instance (throws when not opened). */ -export function getDb() { - if (!_db) throw new Error('database not opened — call openDatabase() first') - return _db -} + afterCommit(fn) { + const tx = transactions.getStore() + if (tx?.db === this) tx.commitCallbacks.push(fn) + else fn() + } -export function isDbOpen() { - return !!_db -} + prepare(sql) { + return { + get: async (...params) => (await this.query(sql, params)).rows[0], + all: async (...params) => (await this.query(sql, params)).rows, + run: async (...params) => { + const result = await this.query(sql, params) + return { changes: result.rowCount || 0, rows: result.rows } + }, + } + } -export function getDbPath() { - return _dbPath -} + async exec(sql) { + await this.query(sql) + } -export function closeDatabase() { - if (_db) { + async withMaintenance(fn) { + if (maintenance.getStore() === this) return fn() + if (this.maintaining) + throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) + this.maintaining = true try { - _db.close() - } catch {} + if (this.active) await new Promise((resolve) => this.waiters.push(resolve)) + return await maintenance.run(this, fn) + } finally { + this.maintaining = false + } } - _db = null - _dbPath = null -} -/** - * Versioned SQL migrations with checksum verification (sub2api-style). - * Files: migrations/NNN_name.sql, applied in filename order inside a - * transaction each. `schema_migrations` records version+checksum; a - * checksum mismatch on an already-applied migration is a hard error. - */ -export function applyMigrations(db, { migrationsDir } = {}) { - const dir = migrationsDir || MIGRATIONS_DIR - db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( - version TEXT PRIMARY KEY, - name TEXT, - checksum TEXT, - applied_at TEXT - )`) + async close() { + if (this.closing) return this.closing + if (this.closed) return + this.closing = (async () => { + let failure + try { + for (const queue of this.pendingWriteQueues || []) { + try { + await queue.flush() + } catch (error) { + failure ||= error + } + } + } finally { + this.closed = true + await this.pool.end() + } + if (failure) throw failure + })() + return this.closing + } +} - const files = fs.existsSync(dir) - ? fs - .readdirSync(dir) - .filter((f) => /^\d+.*\.sql$/.test(f)) - .sort() - : [] +export async function withTransaction(db, fn) { + const outer = transactions.getStore() + if (outer?.db === db) return fn(db) + if (db.closed) throw new Error('database is closed') + if (db.maintaining && maintenance.getStore() !== db) { + throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) + } + // Count the entire transaction, including time between individual queries. + db.active += 1 + let client + try { + client = await db.pool.connect() + await client.query('BEGIN') + const context = { db, client, commitCallbacks: [] } + const result = await transactions.run(context, () => fn(db)) + await client.query('COMMIT') + for (const callback of context.commitCallbacks) callback() + return result + } catch (error) { + if (client) await client.query('ROLLBACK').catch(() => {}) + throw error + } finally { + client?.release() + db.active -= 1 + if (!db.active) for (const resolve of db.waiters.splice(0)) resolve() + } +} - const appliedRows = db.prepare('SELECT version, checksum FROM schema_migrations').all() - const applied = new Map(appliedRows.map((r) => [r.version, r.checksum])) - const historical001Accepted = reconcileHistorical001ProxySchema(db, applied.get('001')) - const insert = db.prepare('INSERT INTO schema_migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)') +/** A recoverable import record must roll back both SQL and cache commit callbacks. */ +export async function withSavepoint(db, fn) { + const tx = transactions.getStore() + if (tx?.db !== db) return withTransaction(db, fn) + const mark = tx.commitCallbacks.length + const name = 'import_' + crypto.randomBytes(8).toString('hex') + await db.exec(`SAVEPOINT ${name}`) + try { + const result = await fn() + await db.exec(`RELEASE SAVEPOINT ${name}`) + return result + } catch (error) { + tx.commitCallbacks.splice(mark) + await db.exec(`ROLLBACK TO SAVEPOINT ${name}`) + await db.exec(`RELEASE SAVEPOINT ${name}`) + throw error + } +} - const results = [] - for (const file of files) { - const version = file.split('_')[0] - const sql = fs.readFileSync(path.join(dir, file), 'utf8') - const { canonical: checksum, variants } = migrationChecksums(sql) - if (applied.has(version)) { - const stored = applied.get(version) - const historicalAccepted = - version === '001' && historical001Accepted && HISTORICAL_001_PROXY_CHECKSUMS.has(stored) - if (!variants.has(stored) && !historicalAccepted) { - throw new Error(`migration checksum mismatch for ${file}: applied=${stored} current=${checksum}`) +export async function applyMigrations(db, { migrationsDir = MIGRATIONS_DIR } = {}) { + return withTransaction(db, async () => { + // Transaction-scoped lock is automatically released on connection failure. + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()), hashtext($1))', [`vm2api:${db.schema}`]) + if (db.schema !== 'public') await db.exec(`CREATE SCHEMA IF NOT EXISTS ${quoteIdentifier(db.schema)}`) + await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, name text NOT NULL, checksum text NOT NULL, applied_at text NOT NULL + )`) + const applied = new Map( + (await db.prepare('SELECT version, checksum FROM schema_migrations').all()).map((r) => [r.version, r.checksum]), + ) + const files = fs + .readdirSync(migrationsDir) + .filter((f) => /^\d+.*\.sql$/.test(f)) + .sort() + const known = new Set(files.map((f) => f.split('_')[0])) + if ([...applied.keys()].some((v) => !known.has(v))) + throw new Error('database contains an unknown PostgreSQL migration') + const results = [] + for (const file of files) { + const version = file.split('_')[0] + const sql = fs.readFileSync(path.join(migrationsDir, file), 'utf8').replaceAll('\r\n', '\n') + const checksum = crypto.createHash('sha256').update(sql).digest('hex') + if (applied.has(version)) { + if (applied.get(version) !== checksum) throw new Error(`migration checksum mismatch for ${file}`) + continue } - if (stored !== checksum) { - db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(checksum, version) - } - continue - } - db.exec('BEGIN') - try { - db.exec(sql) - insert.run(version, file, checksum, new Date().toISOString()) - db.exec('COMMIT') - } catch (e) { try { - db.exec('ROLLBACK') - } catch {} - throw new Error(`migration ${file} failed: ${e.message}`) + await db.exec(sql) + await db + .prepare('INSERT INTO schema_migrations (version,name,checksum,applied_at) VALUES ($1,$2,$3,$4)') + .run(version, file, checksum, new Date().toISOString()) + } catch (error) { + throw new Error(`migration ${file} failed (${error.code || 'migration_error'})`) + } + results.push(file) } - results.push(file) - } - return results + return results + }) } -/** Run fn inside a transaction (nested calls just run inline). */ -export function withTransaction(db, fn) { - if (db.isTransaction) return fn() - db.exec('BEGIN') +export async function createDatabase(options = {}) { + const db = new PostgresDatabase({ ...options, schema: options.schema || process.env.KIN_DATABASE_SCHEMA || 'public' }) try { - const out = fn() - db.exec('COMMIT') - return out - } catch (e) { - if (db.isTransaction) { - try { - db.exec('ROLLBACK') - } catch {} + const version = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) + if (version < 170000 || version >= 180000) + throw new Error('vm2api requires PostgreSQL 17 and PostgreSQL 17 backup clients') + await applyMigrations(db, options) + if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'kin.db')) && !options.allowLegacySource) { + const imported = await db.prepare("SELECT value FROM settings WHERE key = 'sqlite_import_done'").get() + if (!imported) throw new Error('existing SQLite data requires db:import:sqlite before PostgreSQL startup') } - throw e + return db + } catch (error) { + await db.close() + throw error } } -/** - * Online-consistent snapshot of the live database (WAL-safe). - * @returns {string} destination path - */ -export function vacuumInto(db, destPath) { - fs.mkdirSync(path.dirname(destPath), { recursive: true }) +export async function openDatabase(options = {}) { + if (singleton) return singleton + if (opening) return opening + opening = createDatabase(options) try { - fs.rmSync(destPath, { force: true }) - } catch {} - const esc = String(destPath).replace(/'/g, "''") - db.exec(`VACUUM INTO '${esc}'`) - try { - fs.chmodSync(destPath, 0o600) - } catch {} - return destPath + singleton = await opening + return singleton + } finally { + opening = null + } +} + +export function getDb() { + if (!singleton) throw new Error('database not opened — await openDatabase() first') + return singleton +} +export function isDbOpen() { + return !!singleton +} +export function resolveStoreDb({ db } = {}) { + return db || getDb() +} +export async function closeDatabase() { + if (opening) await opening + const db = singleton + singleton = null + if (db) await db.close() } diff --git a/src/lib/db/postgres-migrations/001_baseline.sql b/src/lib/db/postgres-migrations/001_baseline.sql new file mode 100644 index 000000000..5763dd16e --- /dev/null +++ b/src/lib/db/postgres-migrations/001_baseline.sql @@ -0,0 +1,646 @@ +-- PostgreSQL baseline corresponding to SQLite migrations 001–041. +-- SQLite history remains immutable for the offline importer. + +CREATE FUNCTION ascii_fold(text) RETURNS text +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT translate($1, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz') $$; + +CREATE TABLE "account_allocations" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "account_id" text COLLATE "C" NOT NULL, + "at" text COLLATE "C" NOT NULL, + "source" text COLLATE "C", + "util_5h" double precision, + "util_7d" double precision, + "status_5h" text COLLATE "C", + "status_7d" text COLLATE "C", + "claim" text COLLATE "C", + "tokens_in" bigint, + "tokens_out" bigint, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_alloc_account_at" ON "account_allocations" ("account_id", "at" DESC); + +CREATE TABLE "account_groups" ( + "account_id" text COLLATE "C" NOT NULL, + "group_id" bigint NOT NULL, + "priority" bigint NOT NULL DEFAULT 50, + "created_at" text COLLATE "C", + PRIMARY KEY ("account_id", "group_id") +); + +CREATE INDEX "idx_account_groups_priority" ON "account_groups" ("priority"); +CREATE INDEX "idx_account_groups_group" ON "account_groups" ("group_id"); + +CREATE TABLE "account_runtime" ( + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C" NOT NULL, + "status" text COLLATE "C" NOT NULL DEFAULT 'unknown', + "model_states_json" text COLLATE "C", + "credential_generation" bigint NOT NULL DEFAULT 0, + "refresh_status" text COLLATE "C", + "worker_heartbeat_at" bigint, + "worker_status_json" text COLLATE "C", + "updated_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("account_id") +); + +CREATE INDEX "idx_account_runtime_vm_new" ON "account_runtime" ("vm_id"); + +CREATE TABLE "account_runtime_states_legacy" ( + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C" NOT NULL, + "status" text COLLATE "C" NOT NULL DEFAULT 'unknown', + "priority" bigint NOT NULL DEFAULT 0, + "weight" bigint NOT NULL DEFAULT 1, + "cooldown_until" bigint, + "cooldown_reason" text COLLATE "C", + "model_states_json" text COLLATE "C", + "last_used_at" bigint, + "credential_generation" bigint NOT NULL DEFAULT 0, + "refresh_status" text COLLATE "C", + "worker_heartbeat_at" bigint, + "worker_status_json" text COLLATE "C", + "updated_at" text COLLATE "C" NOT NULL, + "rate_limited_at" bigint, + "rate_limit_reset_at" bigint, + "overload_until" bigint, + "session_window_start" bigint, + "session_window_end" bigint, + "session_window_status" text COLLATE "C", + PRIMARY KEY ("account_id") +); + +CREATE INDEX "idx_account_runtime_cooldown" ON "account_runtime_states_legacy" ("cooldown_until"); +CREATE INDEX "idx_account_runtime_vm" ON "account_runtime_states_legacy" ("vm_id"); + +CREATE TABLE "accounts" ( + "id" text COLLATE "C", + "vm_id" text COLLATE "C", + "email" text COLLATE "C", + "concurrency" bigint DEFAULT 2, + "requests" bigint DEFAULT 0, + "tokens_in" bigint DEFAULT 0, + "tokens_out" bigint DEFAULT 0, + "updated_at" text COLLATE "C", + "cache_read_tokens" bigint DEFAULT 0, + "cache_creation_tokens" bigint DEFAULT 0, + "concurrency_override" bigint NOT NULL DEFAULT 0, + "max_rpm" bigint NOT NULL DEFAULT 0, + "rpm_override" bigint NOT NULL DEFAULT 0, + "name" text COLLATE "C" NOT NULL DEFAULT '', + "platform" text COLLATE "C" NOT NULL DEFAULT 'claude', + "type" text COLLATE "C" NOT NULL DEFAULT 'oauth', + "credentials" text COLLATE "C" NOT NULL DEFAULT '{}', + "extra" text COLLATE "C" NOT NULL DEFAULT '{}', + "proxy_id" text COLLATE "C", + "load_factor" bigint, + "priority" bigint NOT NULL DEFAULT 50, + "rate_multiplier" double precision NOT NULL DEFAULT 1.0, + "status" text COLLATE "C" NOT NULL DEFAULT 'active', + "error_message" text COLLATE "C", + "last_used_at" text COLLATE "C", + "expires_at" text COLLATE "C", + "schedulable" bigint NOT NULL DEFAULT 1, + "rate_limited_at" text COLLATE "C", + "rate_limit_reset_at" text COLLATE "C", + "overload_until" text COLLATE "C", + "temp_unschedulable_until" text COLLATE "C", + "temp_unschedulable_reason" text COLLATE "C", + "session_window_start" text COLLATE "C", + "session_window_end" text COLLATE "C", + "session_window_status" text COLLATE "C", + "created_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_accounts_type" ON "accounts" ("type"); +CREATE INDEX "idx_accounts_deleted_at" ON "accounts" ("deleted_at"); +CREATE INDEX "idx_accounts_priority" ON "accounts" ("priority"); +CREATE INDEX "idx_accounts_schedulable" ON "accounts" ("schedulable"); +CREATE INDEX "idx_accounts_status" ON "accounts" ("status"); + +CREATE TABLE "accounts_legacy" ( + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C", + "email" text COLLATE "C", + "max_concurrency" bigint, + "requests" bigint, + "tokens_in" bigint, + "tokens_out" bigint, + "unified_json" text COLLATE "C", + "last_blocked_json" text COLLATE "C", + "last_cli_rate_limit_json" text COLLATE "C", + "updated_at" text COLLATE "C", + "cache_read_tokens" bigint, + "cache_creation_tokens" bigint, + "concurrency_override" bigint, + "max_rpm" bigint, + "rpm_override" bigint +); + + +CREATE TABLE "api_endpoint_keys" ( + "id" text COLLATE "C", + "endpoint_id" text COLLATE "C" NOT NULL, + "api_key" text COLLATE "C" NOT NULL, + "proxy_url" text COLLATE "C", + "disabled" bigint NOT NULL DEFAULT 0, + "cooldown_until" text COLLATE "C", + "created_at" text COLLATE "C", + "proxy_id" text COLLATE "C" NOT NULL DEFAULT '', + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_api_endpoint_keys_ep" ON "api_endpoint_keys" ("endpoint_id"); + +CREATE TABLE "api_endpoint_models" ( + "id" text COLLATE "C", + "endpoint_id" text COLLATE "C" NOT NULL, + "sort" bigint NOT NULL DEFAULT 0, + "upstream_name" text COLLATE "C" NOT NULL, + "alias" text COLLATE "C" NOT NULL, + "image" bigint NOT NULL DEFAULT 0, + "thinking_levels_json" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_api_endpoint_models_ep" ON "api_endpoint_models" ("endpoint_id"); + +CREATE TABLE "api_endpoints" ( + "id" text COLLATE "C", + "name" text COLLATE "C" NOT NULL, + "disabled" bigint NOT NULL DEFAULT 0, + "prefix" text COLLATE "C", + "base_url" text COLLATE "C" NOT NULL, + "headers_json" text COLLATE "C", + "disable_cooling" bigint NOT NULL DEFAULT 0, + "priority" bigint NOT NULL DEFAULT 0, + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "kind" text COLLATE "C" NOT NULL DEFAULT 'custom', + "protocol" text COLLATE "C" NOT NULL DEFAULT 'anthropic', + PRIMARY KEY ("id") +); + + +CREATE TABLE "api_keys" ( + "id" text COLLATE "C", + "name" text COLLATE "C", + "key" text COLLATE "C" NOT NULL, + "status" text COLLATE "C" DEFAULT 'active', + "max_concurrency" bigint DEFAULT 2, + "quota_requests" bigint DEFAULT 0, + "quota_requests_used" bigint DEFAULT 0, + "rpm" bigint DEFAULT 0, + "expires_at" text COLLATE "C", + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "last_used_at" text COLLATE "C", + "requests" bigint DEFAULT 0, + "tokens_in" bigint DEFAULT 0, + "tokens_out" bigint DEFAULT 0, + "key_hash" text COLLATE "C", + "key_prefix" text COLLATE "C", + "key_suffix" text COLLATE "C", + "cache_read_tokens" bigint DEFAULT 0, + "cache_creation_tokens" bigint DEFAULT 0, + "category" text COLLATE "C" NOT NULL DEFAULT 'oauth', + "key_secret" text COLLATE "C", + "user_id" text COLLATE "C", + "group_id" bigint, + "quota" double precision NOT NULL DEFAULT 0, + "quota_used" double precision NOT NULL DEFAULT 0, + "rate_limit_5h" double precision NOT NULL DEFAULT 0, + "rate_limit_1d" double precision NOT NULL DEFAULT 0, + "rate_limit_7d" double precision NOT NULL DEFAULT 0, + "usage_5h" double precision NOT NULL DEFAULT 0, + "usage_1d" double precision NOT NULL DEFAULT 0, + "usage_7d" double precision NOT NULL DEFAULT 0, + "window_5h_start" text COLLATE "C", + "window_1d_start" text COLLATE "C", + "window_7d_start" text COLLATE "C", + "ip_whitelist" text COLLATE "C", + "ip_blacklist" text COLLATE "C", + "deleted_at" text COLLATE "C", + "group_type" text COLLATE "C" NOT NULL DEFAULT 'all', + "allowed_vms" text COLLATE "C" NOT NULL DEFAULT '[]', + "vm_pool_id" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_api_keys_expires" ON "api_keys" ("expires_at"); +CREATE INDEX "idx_api_keys_deleted_at" ON "api_keys" ("deleted_at"); +CREATE INDEX "idx_api_keys_status" ON "api_keys" ("status"); +CREATE INDEX "idx_api_keys_group" ON "api_keys" ("group_id"); +CREATE INDEX "idx_api_keys_user" ON "api_keys" ("user_id"); +CREATE UNIQUE INDEX "idx_api_keys_key_hash" ON "api_keys" ("key_hash") WHERE key_hash IS NOT NULL; +CREATE UNIQUE INDEX "idx_api_keys_unique_6" ON "api_keys" ("key"); + +CREATE TABLE "backup_records" ( + "id" text COLLATE "C", + "created_at" text COLLATE "C" NOT NULL, + "kind" text COLLATE "C", + "status" text COLLATE "C" NOT NULL, + "file_path" text COLLATE "C", + "file_name" text COLLATE "C", + "size_bytes" bigint, + "sha256" text COLLATE "C", + "db_bytes" bigint, + "includes_json" text COLLATE "C", + "error" text COLLATE "C", + "note" text COLLATE "C", + PRIMARY KEY ("id") +); + + +CREATE TABLE "cluster_node_direct_links" ( + "node_id" text COLLATE "C", + "address" text COLLATE "C" NOT NULL, + "port" bigint NOT NULL, + "token" text COLLATE "C" NOT NULL, + "cert_sha256" text COLLATE "C", + "updated_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("node_id") +); + + +CREATE TABLE "cluster_nodes" ( + "id" text COLLATE "C", + "label" text COLLATE "C" NOT NULL, + "host" text COLLATE "C" NOT NULL, + "port" bigint NOT NULL DEFAULT 22, + "username" text COLLATE "C" NOT NULL, + "auth_type" text COLLATE "C" NOT NULL, + "password" text COLLATE "C", + "private_key" text COLLATE "C", + "passphrase" text COLLATE "C", + "host_key_alg" text COLLATE "C" NOT NULL, + "host_key_sha256" text COLLATE "C" NOT NULL, + "jump_node_id" text COLLATE "C", + "created_at" text COLLATE "C" NOT NULL, + "updated_at" text COLLATE "C" NOT NULL, + "platform" text COLLATE "C" NOT NULL DEFAULT 'linux', + PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "idx_cluster_nodes_endpoint" ON "cluster_nodes" ("host", "port", "username"); + +CREATE TABLE "groups" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "name" text COLLATE "C" NOT NULL, + "description" text COLLATE "C", + "rate_multiplier" double precision NOT NULL DEFAULT 1.0, + "is_exclusive" bigint NOT NULL DEFAULT 0, + "platform" text COLLATE "C" NOT NULL DEFAULT 'claude', + "rpm_limit" bigint NOT NULL DEFAULT 0, + "sort_order" bigint NOT NULL DEFAULT 0, + "status" text COLLATE "C" NOT NULL DEFAULT 'active', + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_groups_deleted_at" ON "groups" ("deleted_at"); +CREATE INDEX "idx_groups_status" ON "groups" ("status"); +CREATE UNIQUE INDEX "idx_groups_name_active" ON "groups" ("name") WHERE deleted_at IS NULL; + +CREATE TABLE "panel_users_legacy" ( + "id" text COLLATE "C", + "username" text COLLATE "C" NOT NULL, + "password_hash" text COLLATE "C" NOT NULL, + "role" text COLLATE "C" NOT NULL, + "enabled" bigint NOT NULL DEFAULT 1, + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "last_login_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "idx_panel_users_username" ON "panel_users_legacy" (ascii_fold("username")); + +CREATE TABLE "proxies" ( + "id" text COLLATE "C", + "protocol" text COLLATE "C" DEFAULT 'socks5', + "host" text COLLATE "C", + "port" bigint, + "username" text COLLATE "C", + "password" text COLLATE "C", + "raw" text COLLATE "C", + "status" text COLLATE "C" DEFAULT 'unknown', + "enabled" bigint DEFAULT 1, + "bound_vm_id" text COLLATE "C", + "consecutive_failures" bigint DEFAULT 0, + "latency_ms" bigint, + "last_probe_at" text COLLATE "C", + "last_error" text COLLATE "C", + "created_at" text COLLATE "C", + "name" text COLLATE "C" NOT NULL DEFAULT '', + "updated_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + "owner_user_id" text COLLATE "C", + "geo_ip" text COLLATE "C", + "geo_country" text COLLATE "C", + "geo_country_code" text COLLATE "C", + "geo_region" text COLLATE "C", + "geo_city" text COLLATE "C", + "geo_isp" text COLLATE "C", + "geo_timezone" text COLLATE "C", + "geo_checked_at" text COLLATE "C", + "geo_error" text COLLATE "C", + "label" text COLLATE "C", + "geo_v6_ip" text COLLATE "C", + "geo_v6_country" text COLLATE "C", + "geo_v6_country_code" text COLLATE "C", + "geo_v6_region" text COLLATE "C", + "geo_v6_city" text COLLATE "C", + "geo_v6_isp" text COLLATE "C", + "geo_v6_timezone" text COLLATE "C", + "geo_v6_checked_at" text COLLATE "C", + "geo_v6_error" text COLLATE "C", + "domain_forward" bigint NOT NULL DEFAULT 0, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_proxies_owner" ON "proxies" ("owner_user_id"); +CREATE INDEX "idx_proxies_deleted_at" ON "proxies" ("deleted_at"); +CREATE INDEX "idx_proxies_status" ON "proxies" ("status"); + +CREATE TABLE "redeem_codes" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "code" text COLLATE "C" NOT NULL, + "type" text COLLATE "C" NOT NULL DEFAULT 'balance', + "value" double precision NOT NULL DEFAULT 0, + "status" text COLLATE "C" NOT NULL DEFAULT 'unused', + "used_by" text COLLATE "C", + "used_at" text COLLATE "C", + "notes" text COLLATE "C", + "expires_at" text COLLATE "C", + "created_at" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_redeem_codes_used_by" ON "redeem_codes" ("used_by"); +CREATE INDEX "idx_redeem_codes_status" ON "redeem_codes" ("status"); +CREATE UNIQUE INDEX "idx_redeem_codes_unique_2" ON "redeem_codes" ("code"); + +CREATE TABLE "refusal_device_blocks" ( + "device_id" text COLLATE "C", + "first_seen_at" text COLLATE "C" NOT NULL, + "last_seen_at" text COLLATE "C" NOT NULL, + "hit_count" bigint NOT NULL DEFAULT 0, + "source_request_id" text COLLATE "C", + "fingerprint" text COLLATE "C", + "reason" text COLLATE "C", + PRIMARY KEY ("device_id") +); + +CREATE INDEX "idx_refusal_device_last_seen" ON "refusal_device_blocks" ("last_seen_at"); + +CREATE TABLE "refusal_guards" ( + "fingerprint" text COLLATE "C", + "model" text COLLATE "C" NOT NULL DEFAULT '', + "first_seen_at" text COLLATE "C" NOT NULL, + "last_seen_at" text COLLATE "C" NOT NULL, + "hit_count" bigint NOT NULL DEFAULT 0, + "source_request_id" text COLLATE "C", + "error_message" text COLLATE "C", + "preview" text COLLATE "C", + "expires_at" text COLLATE "C", + "signature" text COLLATE "C", + PRIMARY KEY ("fingerprint") +); + +CREATE INDEX "idx_refusal_guards_last_seen" ON "refusal_guards" ("last_seen_at"); +CREATE INDEX "idx_refusal_guards_model" ON "refusal_guards" ("model"); + +CREATE TABLE "request_attempts" ( + "id" bigint GENERATED BY DEFAULT AS IDENTITY, + "request_id" text COLLATE "C" NOT NULL, + "attempt_no" bigint NOT NULL, + "vm_id" text COLLATE "C", + "account_id" text COLLATE "C", + "model" text COLLATE "C", + "selection_reason" text COLLATE "C", + "started_at" text COLLATE "C" NOT NULL, + "completed_at" text COLLATE "C", + "upstream_status" bigint, + "error_scope" text COLLATE "C", + "action" text COLLATE "C", + "cooldown_until" bigint, + "downstream_committed" bigint NOT NULL DEFAULT 0, + "terminal_state" text COLLATE "C", + "usage_json" text COLLATE "C", + "wait_ms" bigint, + "ttft_ms" bigint, + "latency_ms" bigint, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_request_attempt_account" ON "request_attempts" ("account_id", "started_at" DESC); +CREATE INDEX "idx_request_attempt_request" ON "request_attempts" ("request_id", "attempt_no"); +CREATE UNIQUE INDEX "idx_request_attempt_unique" ON "request_attempts" ("request_id", "attempt_no"); + +CREATE TABLE "request_log_debug" ( + "request_id" text COLLATE "C", + "ts" text COLLATE "C" NOT NULL, + "record_json" text COLLATE "C" NOT NULL, + PRIMARY KEY ("request_id") +); + +CREATE INDEX "idx_reqdbg_ts" ON "request_log_debug" ("ts" DESC); + +CREATE TABLE "settings" ( + "key" text COLLATE "C", + "value" text COLLATE "C" NOT NULL, + "updated_at" text COLLATE "C", + PRIMARY KEY ("key") +); + + +CREATE TABLE "sticky_sessions" ( + "key" text COLLATE "C", + "account_id" text COLLATE "C", + "vm_id" text COLLATE "C", + "session_id" text COLLATE "C", + "bound_at" bigint, + "expires_at" bigint, + "hits" bigint DEFAULT 0, + "device_id" text COLLATE "C", + "generation" bigint NOT NULL DEFAULT 0, + "slot_index" bigint, + PRIMARY KEY ("key") +); + +CREATE INDEX "idx_sticky_device" ON "sticky_sessions" ("device_id"); +CREATE INDEX "idx_sticky_expires" ON "sticky_sessions" ("expires_at"); + +CREATE TABLE "usage_logs" ( + "id" text COLLATE "C", + "request_id" text COLLATE "C", + "created_at" text COLLATE "C" NOT NULL, + "log_mode" text COLLATE "C", + "method" text COLLATE "C", + "path" text COLLATE "C", + "protocol" text COLLATE "C", + "model" text COLLATE "C", + "stream" bigint, + "status" bigint, + "duration_ms" bigint, + "api_key_kind" text COLLATE "C", + "api_key_id" text COLLATE "C", + "vm_id" text COLLATE "C", + "account_id" text COLLATE "C", + "workspace" text COLLATE "C", + "input_tokens" bigint, + "output_tokens" bigint, + "error_code" text COLLATE "C", + "error_message" text COLLATE "C", + "user_agent" text COLLATE "C", + "ip_address" text COLLATE "C", + "has_tools" bigint, + "via" text COLLATE "C", + "cache_read_tokens" bigint, + "cache_creation_tokens" bigint, + "attempt_count" bigint, + "final_state" text COLLATE "C", + "final_account_id" text COLLATE "C", + "cache_creation_5m_tokens" bigint, + "cache_creation_1h_tokens" bigint, + "requested_model" text COLLATE "C", + "upstream_model" text COLLATE "C", + "model_mismatch" bigint, + "first_token_ms" bigint, + "stop_reason" text COLLATE "C", + "input_cost" double precision, + "output_cost" double precision, + "cache_read_cost" double precision, + "cache_creation_cost" double precision, + "total_cost" double precision, + "pricing_model" text COLLATE "C", + "api_key_presented" text COLLATE "C", + "user_id" text COLLATE "C", + "group_id" bigint, + "actual_cost" double precision, + "rate_multiplier" double precision, + "service_tier" text COLLATE "C", + "speed" text COLLATE "C", + "long_context" bigint, + "session_id" text COLLATE "C", + "reasoning_effort" text COLLATE "C", + "outbound_session_id" text COLLATE "C", + "intercept" text COLLATE "C", + "vm_owner_user_id" text COLLATE "C", + "web_search_requests" bigint NOT NULL DEFAULT 0, + "web_search_cost" double precision, + "price_source" text COLLATE "C", + "price_version" text COLLATE "C", + "inbound_device_id" text COLLATE "C", + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_usage_logs_inbound_device" ON "usage_logs" ("inbound_device_id", "created_at"); +CREATE INDEX "idx_usage_logs_vm_owner" ON "usage_logs" ("vm_owner_user_id", "created_at"); +CREATE INDEX "idx_usage_logs_outbound_session_created" ON "usage_logs" ("outbound_session_id", "created_at"); +CREATE INDEX "idx_usage_logs_session_created" ON "usage_logs" ("session_id", "created_at"); +CREATE INDEX "idx_usage_logs_group_created" ON "usage_logs" ("group_id", "created_at"); +CREATE INDEX "idx_usage_logs_user_created" ON "usage_logs" ("user_id", "created_at"); +CREATE INDEX "idx_reqlog_status" ON "usage_logs" ("status", "created_at" DESC); +CREATE INDEX "idx_reqlog_cost_ts" ON "usage_logs" ("created_at" DESC); +CREATE INDEX "idx_reqlog_account" ON "usage_logs" ("account_id", "created_at" DESC); +CREATE INDEX "idx_reqlog_requested_model" ON "usage_logs" ("requested_model"); +CREATE UNIQUE INDEX "idx_reqlog_reqid" ON "usage_logs" ("request_id"); +CREATE INDEX "idx_reqlog_vm" ON "usage_logs" ("vm_id", "created_at" DESC); +CREATE INDEX "idx_reqlog_key" ON "usage_logs" ("api_key_id", "created_at" DESC); +CREATE INDEX "idx_reqlog_ts" ON "usage_logs" ("created_at" DESC); + +CREATE TABLE "users" ( + "id" text COLLATE "C", + "email" text COLLATE "C" NOT NULL, + "username" text COLLATE "C" NOT NULL, + "password_hash" text COLLATE "C" NOT NULL, + "role" text COLLATE "C" NOT NULL DEFAULT 'user', + "balance" double precision NOT NULL DEFAULT 0, + "concurrency" bigint NOT NULL DEFAULT 5, + "status" text COLLATE "C" NOT NULL DEFAULT 'active', + "notes" text COLLATE "C" NOT NULL DEFAULT '', + "last_login_at" text COLLATE "C", + "last_active_at" text COLLATE "C", + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "deleted_at" text COLLATE "C", + "vm_create_quota" bigint NOT NULL DEFAULT 0, + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_users_deleted_at" ON "users" ("deleted_at"); +CREATE INDEX "idx_users_status" ON "users" ("status"); +CREATE UNIQUE INDEX "idx_users_email_active" ON "users" ("email") WHERE deleted_at IS NULL; +CREATE UNIQUE INDEX "idx_users_username_active" ON "users" (ascii_fold("username")) WHERE deleted_at IS NULL; + +CREATE TABLE "vm_pool_members" ( + "pool_id" text COLLATE "C" NOT NULL, + "vm_id" text COLLATE "C" NOT NULL, + "created_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("pool_id", "vm_id") +); + +CREATE UNIQUE INDEX "idx_vm_pool_members_vm" ON "vm_pool_members" ("vm_id"); + +CREATE TABLE "vm_pools" ( + "id" text COLLATE "C", + "name" text COLLATE "C" NOT NULL, + "enabled" bigint NOT NULL DEFAULT 1, + "created_at" text COLLATE "C" NOT NULL, + "updated_at" text COLLATE "C" NOT NULL, + PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "idx_vm_pools_name" ON "vm_pools" (ascii_fold("name")); + +CREATE TABLE "vms" ( + "id" text COLLATE "C", + "name" text COLLATE "C", + "status" text COLLATE "C", + "schedulable" bigint DEFAULT 1, + "email" text COLLATE "C", + "account_uuid" text COLLATE "C", + "org_uuid" text COLLATE "C", + "access_token" text COLLATE "C", + "refresh_token" text COLLATE "C", + "session_key" text COLLATE "C", + "oauth_expires_at" text COLLATE "C", + "oauth_source" text COLLATE "C", + "proxy_id" text COLLATE "C", + "claude_code_version" text COLLATE "C", + "timezone" text COLLATE "C", + "locale" text COLLATE "C", + "vm_json" text COLLATE "C" NOT NULL, + "encrypted" bigint DEFAULT 0, + "file_mtime_ms" bigint, + "synced_at" text COLLATE "C", + "created_at" text COLLATE "C", + "updated_at" text COLLATE "C", + "owner_user_id" text COLLATE "C", + "origin" text COLLATE "C" NOT NULL DEFAULT 'platform', + PRIMARY KEY ("id") +); + +CREATE INDEX "idx_vms_origin" ON "vms" ("origin"); +CREATE INDEX "idx_vms_owner" ON "vms" ("owner_user_id"); +CREATE INDEX "idx_vms_account" ON "vms" ("account_uuid"); + +CREATE TABLE thinking_signature_owners (key text COLLATE "C" PRIMARY KEY, account_id text COLLATE "C" NOT NULL, created_at bigint NOT NULL); +CREATE INDEX idx_thinking_owner_created ON thinking_signature_owners(created_at); + +INSERT INTO groups (id,name,description,sort_order,created_at,updated_at) VALUES +(1,'coding','Coding pool (default)',0,to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')), +(2,'other','Other traffic',1,to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')); +ALTER SEQUENCE groups_id_seq RESTART WITH 3; + +ALTER TABLE "cluster_node_direct_links" ADD FOREIGN KEY ("node_id") REFERENCES "cluster_nodes" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE "cluster_nodes" ADD FOREIGN KEY ("jump_node_id") REFERENCES "cluster_nodes" ("id") ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED; +ALTER TABLE "vm_pool_members" ADD FOREIGN KEY ("pool_id") REFERENCES "vm_pools" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; diff --git a/src/lib/db/sqlite-legacy.mjs b/src/lib/db/sqlite-legacy.mjs new file mode 100644 index 000000000..d62ae1679 --- /dev/null +++ b/src/lib/db/sqlite-legacy.mjs @@ -0,0 +1,273 @@ +/** Offline SQLite compatibility for source snapshots and historical migration fixtures only. */ +/** + * SQLite database layer — sub2api data-layer inspired, on node:sqlite. + * + * - Single-process gateway → embedded SQLite (WAL) instead of sub2api's + * PostgreSQL+Redis. Same patterns: versioned SQL migrations with SHA-256 + * checksum verification (sub2api migrations_runner), repository modules, + * settings table, backup records. + * - Zero new dependencies: uses the built-in `node:sqlite` DatabaseSync. + * + * Env: + * KIN_DB_PATH — override db file path (default /kin.db) + */ + +import fs from 'node:fs' +import path from 'node:path' +import crypto from 'node:crypto' +import { fileURLToPath } from 'node:url' +import { DatabaseSync } from 'node:sqlite' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const MIGRATIONS_DIR = path.join(__dirname, 'migrations') + +function sha256Hex(s) { + return crypto.createHash('sha256').update(s).digest('hex') +} + +/** LF-canonical hash plus CRLF/raw variants so Windows-applied checksums still match. */ +function migrationChecksums(sql) { + const lf = sql.replace(/\r\n/g, '\n') + const crlf = lf.replace(/\n/g, '\r\n') + return { + canonical: sha256Hex(lf), + variants: new Set([sha256Hex(sql), sha256Hex(lf), sha256Hex(crlf)]), + } +} + +// 001 was published briefly before the proxy pool switched to its persisted +// record shape. Its checksum cannot be treated as the current 001: migration +// 018 assumes the newer columns and otherwise fails after accepting the hash. +const HISTORICAL_001_PROXY_CHECKSUMS = new Set([ + '376c686118bbd8aaa6db6ea13fb2ff0afecc0c7464bc3d668743a55c9ccafdde', + 'd6769ea311718b5af91e47295f97c3bf2722ece6d26acee7e89df84b88599ae2', +]) + +function reconcileHistorical001ProxySchema(db, checksum) { + if (!HISTORICAL_001_PROXY_CHECKSUMS.has(checksum)) return false + const columns = new Set( + db + .prepare('PRAGMA table_info(proxies)') + .all() + .map((row) => row.name), + ) + const current = ['raw', 'enabled', 'consecutive_failures', 'last_error'].every((name) => columns.has(name)) + if (current && !columns.has('url')) return true + const historical = ['url', 'failures', 'last_probe_ok', 'disabled_reason', 'updated_at'].every((name) => + columns.has(name), + ) + if (!historical || columns.has('raw')) { + throw new Error('historical migration 001 proxy schema is not recognized') + } + + withTransaction(db, () => { + db.exec(` + ALTER TABLE proxies RENAME TO proxies_historical_001; + CREATE TABLE proxies ( + id TEXT PRIMARY KEY, + scheme TEXT DEFAULT 'socks5', + host TEXT, + port INTEGER, + username TEXT, + password TEXT, + raw TEXT, + status TEXT DEFAULT 'unknown', + enabled INTEGER DEFAULT 1, + bound_vm_id TEXT, + consecutive_failures INTEGER DEFAULT 0, + latency_ms INTEGER, + last_probe_at TEXT, + last_error TEXT, + created_at TEXT + ); + INSERT INTO proxies ( + id, scheme, host, port, username, password, raw, status, enabled, + bound_vm_id, consecutive_failures, latency_ms, last_probe_at, + last_error, created_at + ) + SELECT + id, scheme, host, port, username, password, url, + CASE + WHEN status IN ('unknown', 'ok', 'dead') THEN status + WHEN last_probe_ok = 1 THEN 'ok' + WHEN last_probe_ok = 0 THEN 'dead' + ELSE 'unknown' + END, + CASE WHEN status = 'disabled' THEN 0 ELSE 1 END, + bound_vm_id, COALESCE(failures, 0), latency_ms, last_probe_at, + disabled_reason, created_at + FROM proxies_historical_001; + DROP TABLE proxies_historical_001; + `) + }) + return true +} +let _db = null +let _dbPath = null + +export function resolveDbPath({ dataDir, dbPath } = {}) { + if (dbPath) return dbPath + if (process.env.KIN_DB_PATH) return process.env.KIN_DB_PATH + const dir = dataDir || path.join(process.cwd(), 'data') + return path.join(dir, 'kin.db') +} + +/** + * Standalone (non-singleton) open: PRAGMAs + migrations applied. + * Used by unit tests and by stores constructed without a global db. + * @returns {DatabaseSync} + */ +export function createDatabase({ dataDir, dbPath, migrationsDir } = {}) { + const file = resolveDbPath({ dataDir, dbPath }) + fs.mkdirSync(path.dirname(file), { recursive: true }) + const db = new DatabaseSync(file) + try { + fs.chmodSync(file, 0o600) + } catch {} + db.exec('PRAGMA journal_mode = WAL') + db.exec('PRAGMA synchronous = NORMAL') + db.exec('PRAGMA busy_timeout = 5000') + db.exec('PRAGMA foreign_keys = ON') + applyMigrations(db, { migrationsDir }) + return db +} + +/** + * Open (or return the already-open) singleton database. + * @returns {DatabaseSync} + */ +export function openDatabase({ dataDir, dbPath, migrationsDir } = {}) { + if (_db) return _db + _db = createDatabase({ dataDir, dbPath, migrationsDir }) + _dbPath = resolveDbPath({ dataDir, dbPath }) + return _db +} + +/** + * Resolve the db a store should use: explicit db → global singleton → + * standalone connection rooted at dataDir (test isolation path). + */ +export function resolveStoreDb({ db, dataDir } = {}) { + if (db) return db + if (_db) return _db + return createDatabase({ dataDir }) +} + +/** Current db instance (throws when not opened). */ +export function getDb() { + if (!_db) throw new Error('database not opened — call openDatabase() first') + return _db +} + +export function isDbOpen() { + return !!_db +} + +export function getDbPath() { + return _dbPath +} + +export function closeDatabase() { + if (_db) { + try { + _db.close() + } catch {} + } + _db = null + _dbPath = null +} + +/** + * Versioned SQL migrations with checksum verification (sub2api-style). + * Files: migrations/NNN_name.sql, applied in filename order inside a + * transaction each. `schema_migrations` records version+checksum; a + * checksum mismatch on an already-applied migration is a hard error. + */ +export function applyMigrations(db, { migrationsDir } = {}) { + const dir = migrationsDir || MIGRATIONS_DIR + db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT, + checksum TEXT, + applied_at TEXT + )`) + + const files = fs.existsSync(dir) + ? fs + .readdirSync(dir) + .filter((f) => /^\d+.*\.sql$/.test(f)) + .sort() + : [] + + const appliedRows = db.prepare('SELECT version, checksum FROM schema_migrations').all() + const applied = new Map(appliedRows.map((r) => [r.version, r.checksum])) + const historical001Accepted = reconcileHistorical001ProxySchema(db, applied.get('001')) + const insert = db.prepare('INSERT INTO schema_migrations (version, name, checksum, applied_at) VALUES (?, ?, ?, ?)') + + const results = [] + for (const file of files) { + const version = file.split('_')[0] + const sql = fs.readFileSync(path.join(dir, file), 'utf8') + const { canonical: checksum, variants } = migrationChecksums(sql) + if (applied.has(version)) { + const stored = applied.get(version) + const historicalAccepted = + version === '001' && historical001Accepted && HISTORICAL_001_PROXY_CHECKSUMS.has(stored) + if (!variants.has(stored) && !historicalAccepted) { + throw new Error(`migration checksum mismatch for ${file}: applied=${stored} current=${checksum}`) + } + if (stored !== checksum) { + db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(checksum, version) + } + continue + } + db.exec('BEGIN') + try { + db.exec(sql) + insert.run(version, file, checksum, new Date().toISOString()) + db.exec('COMMIT') + } catch (e) { + try { + db.exec('ROLLBACK') + } catch {} + throw new Error(`migration ${file} failed: ${e.message}`) + } + results.push(file) + } + return results +} + +/** Run fn inside a transaction (nested calls just run inline). */ +export function withTransaction(db, fn) { + if (db.isTransaction) return fn() + db.exec('BEGIN') + try { + const out = fn() + db.exec('COMMIT') + return out + } catch (e) { + if (db.isTransaction) { + try { + db.exec('ROLLBACK') + } catch {} + } + throw e + } +} + +/** + * Online-consistent snapshot of the live database (WAL-safe). + * @returns {string} destination path + */ +export function vacuumInto(db, destPath) { + fs.mkdirSync(path.dirname(destPath), { recursive: true }) + try { + fs.rmSync(destPath, { force: true }) + } catch {} + const esc = String(destPath).replace(/'/g, "''") + db.exec(`VACUUM INTO '${esc}'`) + try { + fs.chmodSync(destPath, 0o600) + } catch {} + return destPath +} diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index a2cd03de7..7453be3f6 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -1,417 +1,176 @@ -import { test, beforeEach, afterEach } from 'node:test' +import { test, after } from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' import crypto from 'node:crypto' -import { DatabaseSync } from 'node:sqlite' import { + createDatabase, openDatabase, closeDatabase, getDb, isDbOpen, - getDbPath, - applyMigrations, - resolveDbPath, - vacuumInto, withTransaction, + applyMigrations, } from '../../src/lib/db/database.mjs' -let tmp - -beforeEach(() => { - tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-db-test-')) -}) - -afterEach(() => { - closeDatabase() - fs.rmSync(tmp, { recursive: true, force: true }) - delete process.env.KIN_DB_PATH +const url = process.env.VM2API_TEST_DATABASE_URL +function options() { + assert.ok(url && /_test$/.test(new URL(url).pathname), 'dedicated PostgreSQL *_test database required') + return { connectionString: url, schema: 'db_' + crypto.randomBytes(8).toString('hex') } +} +const dbs = [] +async function database(extra = {}) { + const db = await createDatabase({ ...options(), ...extra }) + dbs.push(db) + return db +} +after(async () => { + await closeDatabase() + for (const db of dbs) { + if (!db.closed) await db.exec(`DROP SCHEMA IF EXISTS "${db.schema}" CASCADE`) + await db.close() + } }) -test('openDatabase creates db, applies migrations, WAL enabled', () => { - const db = openDatabase({ dataDir: tmp }) - assert.ok(isDbOpen()) - assert.equal(getDbPath(), path.join(tmp, 'kin.db')) - const mode = db.prepare('PRAGMA journal_mode').get() - assert.equal(String(Object.values(mode)[0]).toLowerCase(), 'wal') - const tables = db - .prepare("SELECT name FROM sqlite_master WHERE type='table'") - .all() - .map((r) => r.name) - for (const t of [ +test('PostgreSQL baseline includes business and historical tables and default groups', async () => { + const db = await database() + const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) + assert.equal(rows.length, 30) + for (const name of [ 'settings', + 'users', 'api_keys', 'accounts', - 'account_allocations', - 'sticky_sessions', - 'proxies', - 'vms', 'usage_logs', - 'request_log_debug', - 'backup_records', - 'account_runtime', - 'request_attempts', - 'schema_migrations', - 'api_endpoints', - 'api_endpoint_keys', - 'api_endpoint_models', - 'users', - 'groups', - 'account_groups', - 'redeem_codes', + 'thinking_signature_owners', + 'accounts_legacy', + 'panel_users_legacy', ]) { - assert.ok(tables.includes(t), `missing table ${t}`) + assert.ok( + rows.some((r) => r.table_name === name), + name, + ) } - const keyCols = db - .prepare('PRAGMA table_info(api_keys)') - .all() - .map((r) => r.name) - assert.ok(keyCols.includes('category'), 'api_keys missing category') -}) - -test('migration 005 adds protocol usage alignment columns', () => { - const db = openDatabase({ dataDir: tmp }) - const cols = (table) => - db - .prepare(`PRAGMA table_info(${table})`) - .all() - .map((r) => r.name) - const reqCols = cols('usage_logs') - for (const c of [ - 'cache_creation_5m_tokens', - 'cache_creation_1h_tokens', - 'requested_model', - 'upstream_model', - 'model_mismatch', - 'first_token_ms', - 'stop_reason', - 'input_cost', - 'output_cost', - 'cache_read_cost', - 'cache_creation_cost', - 'total_cost', - 'pricing_model', - 'service_tier', - 'speed', - 'long_context', - ]) - assert.ok(reqCols.includes(c), `usage_logs missing ${c}`) - for (const c of ['cache_read_tokens', 'cache_creation_tokens']) { - assert.ok(cols('accounts').includes(c), `accounts missing ${c}`) - assert.ok(cols('api_keys').includes(c), `api_keys missing ${c}`) - } - // scheduling window columns now live on accounts (sub2api SSOT merge) - const accCols = cols('accounts') - for (const c of [ - 'rate_limited_at', - 'rate_limit_reset_at', - 'overload_until', - 'session_window_start', - 'session_window_end', - 'session_window_status', - 'schedulable', - 'priority', - ]) - assert.ok(accCols.includes(c), `accounts missing ${c}`) -}) - -test('pre-005 database upgrades in place and keeps old rows readable', () => { - // Build a DB with migrations 001–004 only, insert a legacy row, then upgrade. - const migSrc = path.resolve('src/lib/db/migrations') - const oldDir = path.join(tmp, 'old-migs') - fs.mkdirSync(oldDir, { recursive: true }) - for (const f of fs.readdirSync(migSrc)) { - if (f < '005') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) - } - const dbPath = path.join(tmp, 'upgrade.db') - const legacy = new DatabaseSync(dbPath) - applyMigrations(legacy, { migrationsDir: oldDir }) - legacy - .prepare(` - INSERT INTO request_logs (id, request_id, ts, protocol, model, status, input_tokens, output_tokens) - VALUES ('log_old', 'rid_old', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 7, 2) - `) - .run(new Date().toISOString()) - legacy.close() - - process.env.KIN_DB_PATH = dbPath - const db = openDatabase({ dbPath }) - const row = db.prepare('SELECT * FROM usage_logs WHERE request_id = ?').get('rid_old') - assert.equal(row.input_tokens, 7) - // migration 017 normalizes NULL token counters to 0 - assert.equal(row.cache_creation_5m_tokens, 0) - assert.equal(row.stop_reason, null) - // new rows can use the new columns - db.prepare(` - INSERT INTO usage_logs (id, request_id, created_at, protocol, model, status, upstream_model, stop_reason, first_token_ms) - VALUES ('log_new', 'rid_new', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 'claude-sonnet-5', 'end_turn', 21) - `).run(new Date().toISOString()) - assert.equal( - db.prepare('SELECT stop_reason FROM usage_logs WHERE request_id = ?').get('rid_new').stop_reason, - 'end_turn', + assert.deepEqual( + (await db.prepare('SELECT id,name FROM groups ORDER BY id').all()).map((r) => [r.id, r.name]), + [ + [1, 'coding'], + [2, 'other'], + ], ) + assert.deepEqual(await applyMigrations(db), []) }) -test('openDatabase is idempotent singleton; getDb throws when closed', () => { - const a = openDatabase({ dataDir: tmp }) - const b = openDatabase({ dataDir: path.join(tmp, 'other') }) - assert.equal(a, b) - closeDatabase() - assert.throws(() => getDb(), /not opened/) -}) - -test('migrations are idempotent across reopen', () => { - openDatabase({ dataDir: tmp }) - closeDatabase() - const db = openDatabase({ dataDir: tmp }) - const rows = db.prepare('SELECT version FROM schema_migrations').all() - assert.equal(rows.length, new Set(rows.map((r) => r.version)).size) -}) - -test('migration 037 normalizes OpenAI input including NULL route fields; SUM(prompt) holds', () => { - const migSrc = path.resolve('src/lib/db/migrations') - const oldDir = path.join(tmp, 'pre-037') - fs.mkdirSync(oldDir, { recursive: true }) - for (const f of fs.readdirSync(migSrc)) { - if (f < '037') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) - } - const dbPath = path.join(tmp, 'bill037.db') - const db = new DatabaseSync(dbPath) - applyMigrations(db, { migrationsDir: oldDir }) - - const cacheModel = `LOWER(TRIM(COALESCE( - NULLIF(NULLIF(pricing_model, ''), 'unpriced'), NULLIF(upstream_model, ''), - NULLIF(model, ''), requested_model, '' - )))` - const openaiIds = [ - 'gpt-6-astra', - 'gpt-6.1-sol', - 'gpt-6-sol', - 'gpt-6-luna', - 'gpt-5.6-sol', - 'gpt-5.6-terra', - 'gpt-5.6-luna', - 'gpt-5.6', - 'gpt-5.5-pro', - 'gpt-5.5', - 'gpt-5.4-mini', - 'gpt-5.4-nano', - 'gpt-5.4-pro', - 'gpt-5.4', - 'gpt-5.3-codex', - 'gpt-5.2-pro', - 'gpt-5.2', - 'gpt-5.1', - 'gpt-5-mini', - 'gpt-5-nano', - 'gpt-5-pro', - 'gpt-5', - 'gpt-4.1-mini', - 'gpt-4.1-nano', - 'gpt-4.1', - 'gpt-4o-2024-05-13', - 'gpt-4o-mini', - 'gpt-4o', - 'o1-pro', - 'o1', - 'o3-pro', - 'o3-mini', - 'o3', - 'o4-mini', - 'gpt-4-turbo', - 'gpt-4', - 'gpt-3.5-turbo-instruct', - 'gpt-3.5-turbo-1106', - 'gpt-3.5-turbo', - 'gpt-5.1-codex-mini', - 'gpt-5.3-chat-latest', - 'gpt-5.6-cyber', - 'gpt-5.5-cyber', - 'chat-latest', - 'gpt-5-codex', - 'gpt-5.1-codex', - 'gpt-5.1-codex-max', - 'gpt-5-chat-latest', - 'gpt-5.1-chat-latest', - 'gpt-5.2-codex', - 'gpt-5.2-chat-latest', - ] - .map((id) => `'${id}'`) - .join(', ') - const oldPrompt = `CASE - WHEN via = 'api-kernel' AND protocol = 'anthropic.messages' AND stream = 0 - THEN IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) - WHEN via LIKE 'codex-%' - OR (${cacheModel} LIKE 'gpt%' AND ${cacheModel} NOT LIKE 'gpt-image%') - OR ${cacheModel} LIKE 'codex-%' - OR ${cacheModel} IN (${openaiIds}) - THEN IFNULL(input_tokens, 0) - ELSE IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) - END` - const newPrompt = 'IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' - - const ins = db.prepare(` - INSERT INTO usage_logs ( - id, request_id, created_at, model, protocol, stream, via, - input_tokens, cache_read_tokens, cache_creation_tokens, output_tokens, upstream_model - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - const at = '2026-09-01T00:00:00.000Z' - ins.run('n-via', 'r-via', at, 'gpt-5.4', 'anthropic.messages', 0, null, 100, 80, 0, 2, null) - ins.run('n-proto', 'r-proto', at, 'gpt-5.4', null, 0, 'go-worker-pool', 100, 80, 0, 2, null) - ins.run('n-stream', 'r-stream', at, 'gpt-5.4', 'anthropic.messages', null, 'go-worker-pool', 100, 80, 0, 2, null) - ins.run('n-all', 'r-all', at, 'gpt-5.4', null, null, null, 100, 80, 0, 2, null) - ins.run('keep-api', 'r-api', at, 'gpt-5.4', 'anthropic.messages', 0, 'api-kernel', 20, 80, 0, 2, 'gpt-5.4') - ins.run('keep-claude', 'r-claude', at, 'claude-sonnet-5', 'anthropic.messages', 0, null, 20, 80, 0, 2, null) - ins.run('codex', 'r-codex', at, 'gpt-5.4', 'openai.responses', 1, 'codex-kernel', 100000, 80000, 0, 2000, null) - - const before = db.prepare(`SELECT SUM(${oldPrompt}) AS n FROM usage_logs`).get().n - applyMigrations(db, { migrationsDir: migSrc }) - const after = db.prepare(`SELECT SUM(${newPrompt}) AS n FROM usage_logs`).get().n - assert.equal(after, before) - - const cols = db - .prepare('PRAGMA table_info(usage_logs)') - .all() - .map((r) => r.name) - for (const c of ['web_search_requests', 'web_search_cost', 'price_source', 'price_version']) { - assert.ok(cols.includes(c), `missing ${c}`) - } - - const get = (id) => db.prepare('SELECT input_tokens, cache_read_tokens FROM usage_logs WHERE id = ?').get(id) - assert.equal(get('n-via').input_tokens, 20) - assert.equal(get('n-proto').input_tokens, 20) - assert.equal(get('n-stream').input_tokens, 20) - assert.equal(get('n-all').input_tokens, 20) - assert.equal(get('keep-api').input_tokens, 20) - assert.equal(get('keep-claude').input_tokens, 20) - assert.equal(get('codex').input_tokens, 20000) - // Earlier prunes were never recorded; existing rows only vouch for history from the oldest one. - const pruned = db.prepare("SELECT value FROM settings WHERE key = 'usage_logs_pruned_through'").get() - assert.equal(JSON.parse(pruned.value), at) - db.close() -}) - -test('checksum mismatch on applied migration throws', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') - const db = new DatabaseSync(path.join(tmp, 'x.db')) - applyMigrations(db, { migrationsDir: migDir }) - fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT, b TEXT);') - assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /checksum mismatch/) - db.close() -}) - -test('CRLF-applied checksum of the same SQL is accepted and rewritten', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - const lf = 'CREATE TABLE t1 (a TEXT);\n' - const crlf = lf.replace(/\n/g, '\r\n') - fs.writeFileSync(path.join(migDir, '001_a.sql'), lf) - const db = new DatabaseSync(path.join(tmp, 'x.db')) - applyMigrations(db, { migrationsDir: migDir }) - const crlfHash = crypto.createHash('sha256').update(crlf).digest('hex') - db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(crlfHash, '001') - assert.doesNotThrow(() => applyMigrations(db, { migrationsDir: migDir })) - const stored = db.prepare('SELECT checksum FROM schema_migrations WHERE version = ?').get('001').checksum - assert.notEqual(stored, crlfHash) - assert.equal(stored.length, 64) - db.close() -}) - -test('new migration files apply in order and record checksums', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') - const db = new DatabaseSync(path.join(tmp, 'x.db')) - let applied = applyMigrations(db, { migrationsDir: migDir }) - assert.deepEqual(applied, ['001_a.sql']) - fs.writeFileSync(path.join(migDir, '002_b.sql'), 'ALTER TABLE t1 ADD COLUMN b TEXT;') - applied = applyMigrations(db, { migrationsDir: migDir }) - assert.deepEqual(applied, ['002_b.sql']) - const rows = db.prepare('SELECT version, checksum FROM schema_migrations ORDER BY version').all() - assert.equal(rows.length, 2) - assert.ok(rows.every((r) => /^[0-9a-f]{64}$/.test(r.checksum))) - db.close() -}) - -test('failed migration rolls back and is not recorded', () => { - const migDir = path.join(tmp, 'migs') - fs.mkdirSync(migDir, { recursive: true }) - fs.writeFileSync(path.join(migDir, '001_bad.sql'), 'CREATE TABLE ok1 (a TEXT); THIS IS NOT SQL;') - const db = new DatabaseSync(path.join(tmp, 'x.db')) - assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /001_bad\.sql failed/) - const tables = db - .prepare("SELECT name FROM sqlite_master WHERE type='table'") - .all() - .map((r) => r.name) - assert.ok(!tables.includes('ok1'), 'rollback should drop partial table') - assert.equal(db.prepare('SELECT COUNT(*) c FROM schema_migrations').get().c, 0) - db.close() -}) - -test('KIN_DB_PATH env overrides path', () => { - const custom = path.join(tmp, 'custom', 'my.db') - process.env.KIN_DB_PATH = custom - assert.equal(resolveDbPath({ dataDir: tmp }), custom) - openDatabase({ dataDir: tmp }) - assert.equal(getDbPath(), custom) - assert.ok(fs.existsSync(custom)) -}) - -test('vacuumInto produces a consistent snapshot', () => { - const db = openDatabase({ dataDir: tmp }) - db.prepare('INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?)').run( - 'k1', - '"v1"', - new Date().toISOString(), +test('transactions await callbacks and rollback writes on the same connection', async () => { + const db = await database() + await withTransaction(db, async () => { + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('outer', '1') + await withTransaction(db, async () => { + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('nested', '2') + }) + }) + await assert.rejects( + withTransaction(db, async () => { + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('rolled', '3') + await new Promise((r) => setImmediate(r)) + throw new Error('rollback marker') + }), + /rollback marker/, + ) + assert.deepEqual( + (await db.prepare('SELECT key FROM settings ORDER BY key').all()).map((r) => r.key), + ['nested', 'outer'], ) - const dest = path.join(tmp, 'backup', 'snap.db') - vacuumInto(db, dest) - const snap = new DatabaseSync(dest) - const row = snap.prepare('SELECT value FROM settings WHERE key = ?').get('k1') - assert.equal(row.value, '"v1"') - snap.close() }) -test('withTransaction commits and rolls back', () => { - const db = openDatabase({ dataDir: tmp }) - withTransaction(db, () => { - db.prepare("INSERT INTO settings (key, value) VALUES ('a', '1')").run() - }) - assert.ok(db.prepare("SELECT * FROM settings WHERE key='a'").get()) - assert.throws( - () => - withTransaction(db, () => { - db.prepare("INSERT INTO settings (key, value) VALUES ('b', '2')").run() - throw new Error('boom') +test('concurrent transactions do not share a connection or rollback each other', async () => { + const db = await database() + await Promise.all([ + withTransaction(db, async () => { + await db.prepare("INSERT INTO settings(key,value) VALUES('keep','1')").run() + }), + assert.rejects( + withTransaction(db, async () => { + await db.prepare("INSERT INTO settings(key,value) VALUES('drop','2')").run() + throw new Error('fail') }), - /boom/, + /fail/, + ), + ]) + assert.ok(await db.prepare("SELECT * FROM settings WHERE key='keep'").get()) + assert.equal(await db.prepare("SELECT * FROM settings WHERE key='drop'").get(), undefined) +}) + +test('concurrent initialization serializes migrations and checksums are enforced', async () => { + const opts = options() + const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) + dbs.push(a, b) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 1) + await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() + await assert.rejects(applyMigrations(b), /checksum mismatch/) +}) + +test('failed migrations rollback DDL and migration records', async () => { + const dir = fs.mkdtempSync('/tmp/vm2api-pg-migrations-') + fs.writeFileSync(dir + '/002_bad.sql', 'CREATE TABLE partial_write(id text); INVALID SQL;') + const db = await database() + await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /unknown PostgreSQL migration/) + fs.copyFileSync( + new URL('../../src/lib/db/postgres-migrations/001_baseline.sql', import.meta.url), + dir + '/001_baseline.sql', + ) + await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /002_bad.sql failed/) + assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 1) + fs.rmSync(dir, { recursive: true, force: true }) +}) + +test('big integers preserve precision and errors exclude SQL parameter values', async () => { + const db = await database() + const row = await db.prepare('SELECT $1::bigint AS large, $2::bigint AS small').get('9007199254740993', 42) + assert.equal(row.large, '9007199254740993') + assert.equal(row.small, 42) + await db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('sensitive-key', 'sensitive-value') + await assert.rejects( + db.prepare('INSERT INTO settings(key,value) VALUES($1,$2)').run('sensitive-key', 'sensitive-value'), + (error) => error.code === '23505' && !error.message.includes('sensitive'), ) - assert.equal(db.prepare("SELECT * FROM settings WHERE key='b'").get(), undefined) }) -test('withTransaction only runs inline for a real outer transaction', () => { - const db = openDatabase({ dataDir: tmp }) - db.exec('BEGIN') - withTransaction(db, () => { - db.prepare("INSERT INTO settings (key, value) VALUES ('nested', '1')").run() +test('singleton opens once and closes explicitly', async () => { + await closeDatabase() + assert.throws(getDb, /not opened/) + const opts = options() + const [a, b] = await Promise.all([openDatabase(opts), openDatabase(opts)]) + assert.equal(a, b) + assert.equal(getDb(), a) + assert.equal(isDbOpen(), true) + await a.exec(`DROP SCHEMA "${a.schema}" CASCADE`) + await closeDatabase() + assert.equal(isDbOpen(), false) +}) + +test('maintenance drains transactions and rejects unrelated writes', async () => { + const db = await database() + let release, started + const ready = new Promise((r) => { + started = r }) - assert.equal(db.isTransaction, true) - db.exec('ROLLBACK') - assert.equal(db.prepare("SELECT * FROM settings WHERE key='nested'").get(), undefined) - - const closed = new DatabaseSync(':memory:') - closed.close() - let called = false - assert.throws( - () => - withTransaction(closed, () => { - called = true - }), - /not open|closed/i, - ) - assert.equal(called, false) + const transaction = withTransaction(db, async () => { + started() + await new Promise((r) => { + release = r + }) + await db.prepare("INSERT INTO settings(key,value) VALUES('before','1')").run() + }) + await ready + const maintaining = db.withMaintenance(async () => { + await db.prepare("INSERT INTO settings(key,value) VALUES('maintenance','1')").run() + }) + await assert.rejects(db.prepare("INSERT INTO settings(key,value) VALUES('blocked','1')").run(), { + code: 'database_maintenance', + }) + release() + await transaction + await maintaining + assert.equal((await db.prepare('SELECT count(*) AS n FROM settings').get()).n, 2) }) diff --git a/test/unit/db-migration-sub2api.test.mjs b/test/unit/db-migration-sub2api.test.mjs index 9d3338698..a31e7d007 100644 --- a/test/unit/db-migration-sub2api.test.mjs +++ b/test/unit/db-migration-sub2api.test.mjs @@ -4,7 +4,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { createDatabase } from '../../src/lib/db/sqlite-legacy.mjs' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const MIGRATIONS = path.join(__dirname, '../../src/lib/db/migrations') diff --git a/test/unit/sqlite-legacy.test.mjs b/test/unit/sqlite-legacy.test.mjs new file mode 100644 index 000000000..8258be4df --- /dev/null +++ b/test/unit/sqlite-legacy.test.mjs @@ -0,0 +1,417 @@ +import { test, beforeEach, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import crypto from 'node:crypto' +import { DatabaseSync } from 'node:sqlite' +import { + openDatabase, + closeDatabase, + getDb, + isDbOpen, + getDbPath, + applyMigrations, + resolveDbPath, + vacuumInto, + withTransaction, +} from '../../src/lib/db/sqlite-legacy.mjs' + +let tmp + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-db-test-')) +}) + +afterEach(() => { + closeDatabase() + fs.rmSync(tmp, { recursive: true, force: true }) + delete process.env.KIN_DB_PATH +}) + +test('openDatabase creates db, applies migrations, WAL enabled', () => { + const db = openDatabase({ dataDir: tmp }) + assert.ok(isDbOpen()) + assert.equal(getDbPath(), path.join(tmp, 'kin.db')) + const mode = db.prepare('PRAGMA journal_mode').get() + assert.equal(String(Object.values(mode)[0]).toLowerCase(), 'wal') + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type='table'") + .all() + .map((r) => r.name) + for (const t of [ + 'settings', + 'api_keys', + 'accounts', + 'account_allocations', + 'sticky_sessions', + 'proxies', + 'vms', + 'usage_logs', + 'request_log_debug', + 'backup_records', + 'account_runtime', + 'request_attempts', + 'schema_migrations', + 'api_endpoints', + 'api_endpoint_keys', + 'api_endpoint_models', + 'users', + 'groups', + 'account_groups', + 'redeem_codes', + ]) { + assert.ok(tables.includes(t), `missing table ${t}`) + } + const keyCols = db + .prepare('PRAGMA table_info(api_keys)') + .all() + .map((r) => r.name) + assert.ok(keyCols.includes('category'), 'api_keys missing category') +}) + +test('migration 005 adds protocol usage alignment columns', () => { + const db = openDatabase({ dataDir: tmp }) + const cols = (table) => + db + .prepare(`PRAGMA table_info(${table})`) + .all() + .map((r) => r.name) + const reqCols = cols('usage_logs') + for (const c of [ + 'cache_creation_5m_tokens', + 'cache_creation_1h_tokens', + 'requested_model', + 'upstream_model', + 'model_mismatch', + 'first_token_ms', + 'stop_reason', + 'input_cost', + 'output_cost', + 'cache_read_cost', + 'cache_creation_cost', + 'total_cost', + 'pricing_model', + 'service_tier', + 'speed', + 'long_context', + ]) + assert.ok(reqCols.includes(c), `usage_logs missing ${c}`) + for (const c of ['cache_read_tokens', 'cache_creation_tokens']) { + assert.ok(cols('accounts').includes(c), `accounts missing ${c}`) + assert.ok(cols('api_keys').includes(c), `api_keys missing ${c}`) + } + // scheduling window columns now live on accounts (sub2api SSOT merge) + const accCols = cols('accounts') + for (const c of [ + 'rate_limited_at', + 'rate_limit_reset_at', + 'overload_until', + 'session_window_start', + 'session_window_end', + 'session_window_status', + 'schedulable', + 'priority', + ]) + assert.ok(accCols.includes(c), `accounts missing ${c}`) +}) + +test('pre-005 database upgrades in place and keeps old rows readable', () => { + // Build a DB with migrations 001–004 only, insert a legacy row, then upgrade. + const migSrc = path.resolve('src/lib/db/migrations') + const oldDir = path.join(tmp, 'old-migs') + fs.mkdirSync(oldDir, { recursive: true }) + for (const f of fs.readdirSync(migSrc)) { + if (f < '005') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) + } + const dbPath = path.join(tmp, 'upgrade.db') + const legacy = new DatabaseSync(dbPath) + applyMigrations(legacy, { migrationsDir: oldDir }) + legacy + .prepare(` + INSERT INTO request_logs (id, request_id, ts, protocol, model, status, input_tokens, output_tokens) + VALUES ('log_old', 'rid_old', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 7, 2) + `) + .run(new Date().toISOString()) + legacy.close() + + process.env.KIN_DB_PATH = dbPath + const db = openDatabase({ dbPath }) + const row = db.prepare('SELECT * FROM usage_logs WHERE request_id = ?').get('rid_old') + assert.equal(row.input_tokens, 7) + // migration 017 normalizes NULL token counters to 0 + assert.equal(row.cache_creation_5m_tokens, 0) + assert.equal(row.stop_reason, null) + // new rows can use the new columns + db.prepare(` + INSERT INTO usage_logs (id, request_id, created_at, protocol, model, status, upstream_model, stop_reason, first_token_ms) + VALUES ('log_new', 'rid_new', ?, 'anthropic.messages', 'claude-sonnet-5', 200, 'claude-sonnet-5', 'end_turn', 21) + `).run(new Date().toISOString()) + assert.equal( + db.prepare('SELECT stop_reason FROM usage_logs WHERE request_id = ?').get('rid_new').stop_reason, + 'end_turn', + ) +}) + +test('openDatabase is idempotent singleton; getDb throws when closed', () => { + const a = openDatabase({ dataDir: tmp }) + const b = openDatabase({ dataDir: path.join(tmp, 'other') }) + assert.equal(a, b) + closeDatabase() + assert.throws(() => getDb(), /not opened/) +}) + +test('migrations are idempotent across reopen', () => { + openDatabase({ dataDir: tmp }) + closeDatabase() + const db = openDatabase({ dataDir: tmp }) + const rows = db.prepare('SELECT version FROM schema_migrations').all() + assert.equal(rows.length, new Set(rows.map((r) => r.version)).size) +}) + +test('migration 037 normalizes OpenAI input including NULL route fields; SUM(prompt) holds', () => { + const migSrc = path.resolve('src/lib/db/migrations') + const oldDir = path.join(tmp, 'pre-037') + fs.mkdirSync(oldDir, { recursive: true }) + for (const f of fs.readdirSync(migSrc)) { + if (f < '037') fs.copyFileSync(path.join(migSrc, f), path.join(oldDir, f)) + } + const dbPath = path.join(tmp, 'bill037.db') + const db = new DatabaseSync(dbPath) + applyMigrations(db, { migrationsDir: oldDir }) + + const cacheModel = `LOWER(TRIM(COALESCE( + NULLIF(NULLIF(pricing_model, ''), 'unpriced'), NULLIF(upstream_model, ''), + NULLIF(model, ''), requested_model, '' + )))` + const openaiIds = [ + 'gpt-6-astra', + 'gpt-6.1-sol', + 'gpt-6-sol', + 'gpt-6-luna', + 'gpt-5.6-sol', + 'gpt-5.6-terra', + 'gpt-5.6-luna', + 'gpt-5.6', + 'gpt-5.5-pro', + 'gpt-5.5', + 'gpt-5.4-mini', + 'gpt-5.4-nano', + 'gpt-5.4-pro', + 'gpt-5.4', + 'gpt-5.3-codex', + 'gpt-5.2-pro', + 'gpt-5.2', + 'gpt-5.1', + 'gpt-5-mini', + 'gpt-5-nano', + 'gpt-5-pro', + 'gpt-5', + 'gpt-4.1-mini', + 'gpt-4.1-nano', + 'gpt-4.1', + 'gpt-4o-2024-05-13', + 'gpt-4o-mini', + 'gpt-4o', + 'o1-pro', + 'o1', + 'o3-pro', + 'o3-mini', + 'o3', + 'o4-mini', + 'gpt-4-turbo', + 'gpt-4', + 'gpt-3.5-turbo-instruct', + 'gpt-3.5-turbo-1106', + 'gpt-3.5-turbo', + 'gpt-5.1-codex-mini', + 'gpt-5.3-chat-latest', + 'gpt-5.6-cyber', + 'gpt-5.5-cyber', + 'chat-latest', + 'gpt-5-codex', + 'gpt-5.1-codex', + 'gpt-5.1-codex-max', + 'gpt-5-chat-latest', + 'gpt-5.1-chat-latest', + 'gpt-5.2-codex', + 'gpt-5.2-chat-latest', + ] + .map((id) => `'${id}'`) + .join(', ') + const oldPrompt = `CASE + WHEN via = 'api-kernel' AND protocol = 'anthropic.messages' AND stream = 0 + THEN IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) + WHEN via LIKE 'codex-%' + OR (${cacheModel} LIKE 'gpt%' AND ${cacheModel} NOT LIKE 'gpt-image%') + OR ${cacheModel} LIKE 'codex-%' + OR ${cacheModel} IN (${openaiIds}) + THEN IFNULL(input_tokens, 0) + ELSE IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0) + END` + const newPrompt = 'IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' + + const ins = db.prepare(` + INSERT INTO usage_logs ( + id, request_id, created_at, model, protocol, stream, via, + input_tokens, cache_read_tokens, cache_creation_tokens, output_tokens, upstream_model + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + const at = '2026-09-01T00:00:00.000Z' + ins.run('n-via', 'r-via', at, 'gpt-5.4', 'anthropic.messages', 0, null, 100, 80, 0, 2, null) + ins.run('n-proto', 'r-proto', at, 'gpt-5.4', null, 0, 'go-worker-pool', 100, 80, 0, 2, null) + ins.run('n-stream', 'r-stream', at, 'gpt-5.4', 'anthropic.messages', null, 'go-worker-pool', 100, 80, 0, 2, null) + ins.run('n-all', 'r-all', at, 'gpt-5.4', null, null, null, 100, 80, 0, 2, null) + ins.run('keep-api', 'r-api', at, 'gpt-5.4', 'anthropic.messages', 0, 'api-kernel', 20, 80, 0, 2, 'gpt-5.4') + ins.run('keep-claude', 'r-claude', at, 'claude-sonnet-5', 'anthropic.messages', 0, null, 20, 80, 0, 2, null) + ins.run('codex', 'r-codex', at, 'gpt-5.4', 'openai.responses', 1, 'codex-kernel', 100000, 80000, 0, 2000, null) + + const before = db.prepare(`SELECT SUM(${oldPrompt}) AS n FROM usage_logs`).get().n + applyMigrations(db, { migrationsDir: migSrc }) + const after = db.prepare(`SELECT SUM(${newPrompt}) AS n FROM usage_logs`).get().n + assert.equal(after, before) + + const cols = db + .prepare('PRAGMA table_info(usage_logs)') + .all() + .map((r) => r.name) + for (const c of ['web_search_requests', 'web_search_cost', 'price_source', 'price_version']) { + assert.ok(cols.includes(c), `missing ${c}`) + } + + const get = (id) => db.prepare('SELECT input_tokens, cache_read_tokens FROM usage_logs WHERE id = ?').get(id) + assert.equal(get('n-via').input_tokens, 20) + assert.equal(get('n-proto').input_tokens, 20) + assert.equal(get('n-stream').input_tokens, 20) + assert.equal(get('n-all').input_tokens, 20) + assert.equal(get('keep-api').input_tokens, 20) + assert.equal(get('keep-claude').input_tokens, 20) + assert.equal(get('codex').input_tokens, 20000) + // Earlier prunes were never recorded; existing rows only vouch for history from the oldest one. + const pruned = db.prepare("SELECT value FROM settings WHERE key = 'usage_logs_pruned_through'").get() + assert.equal(JSON.parse(pruned.value), at) + db.close() +}) + +test('checksum mismatch on applied migration throws', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') + const db = new DatabaseSync(path.join(tmp, 'x.db')) + applyMigrations(db, { migrationsDir: migDir }) + fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT, b TEXT);') + assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /checksum mismatch/) + db.close() +}) + +test('CRLF-applied checksum of the same SQL is accepted and rewritten', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + const lf = 'CREATE TABLE t1 (a TEXT);\n' + const crlf = lf.replace(/\n/g, '\r\n') + fs.writeFileSync(path.join(migDir, '001_a.sql'), lf) + const db = new DatabaseSync(path.join(tmp, 'x.db')) + applyMigrations(db, { migrationsDir: migDir }) + const crlfHash = crypto.createHash('sha256').update(crlf).digest('hex') + db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(crlfHash, '001') + assert.doesNotThrow(() => applyMigrations(db, { migrationsDir: migDir })) + const stored = db.prepare('SELECT checksum FROM schema_migrations WHERE version = ?').get('001').checksum + assert.notEqual(stored, crlfHash) + assert.equal(stored.length, 64) + db.close() +}) + +test('new migration files apply in order and record checksums', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + fs.writeFileSync(path.join(migDir, '001_a.sql'), 'CREATE TABLE t1 (a TEXT);') + const db = new DatabaseSync(path.join(tmp, 'x.db')) + let applied = applyMigrations(db, { migrationsDir: migDir }) + assert.deepEqual(applied, ['001_a.sql']) + fs.writeFileSync(path.join(migDir, '002_b.sql'), 'ALTER TABLE t1 ADD COLUMN b TEXT;') + applied = applyMigrations(db, { migrationsDir: migDir }) + assert.deepEqual(applied, ['002_b.sql']) + const rows = db.prepare('SELECT version, checksum FROM schema_migrations ORDER BY version').all() + assert.equal(rows.length, 2) + assert.ok(rows.every((r) => /^[0-9a-f]{64}$/.test(r.checksum))) + db.close() +}) + +test('failed migration rolls back and is not recorded', () => { + const migDir = path.join(tmp, 'migs') + fs.mkdirSync(migDir, { recursive: true }) + fs.writeFileSync(path.join(migDir, '001_bad.sql'), 'CREATE TABLE ok1 (a TEXT); THIS IS NOT SQL;') + const db = new DatabaseSync(path.join(tmp, 'x.db')) + assert.throws(() => applyMigrations(db, { migrationsDir: migDir }), /001_bad\.sql failed/) + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type='table'") + .all() + .map((r) => r.name) + assert.ok(!tables.includes('ok1'), 'rollback should drop partial table') + assert.equal(db.prepare('SELECT COUNT(*) c FROM schema_migrations').get().c, 0) + db.close() +}) + +test('KIN_DB_PATH env overrides path', () => { + const custom = path.join(tmp, 'custom', 'my.db') + process.env.KIN_DB_PATH = custom + assert.equal(resolveDbPath({ dataDir: tmp }), custom) + openDatabase({ dataDir: tmp }) + assert.equal(getDbPath(), custom) + assert.ok(fs.existsSync(custom)) +}) + +test('vacuumInto produces a consistent snapshot', () => { + const db = openDatabase({ dataDir: tmp }) + db.prepare('INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?)').run( + 'k1', + '"v1"', + new Date().toISOString(), + ) + const dest = path.join(tmp, 'backup', 'snap.db') + vacuumInto(db, dest) + const snap = new DatabaseSync(dest) + const row = snap.prepare('SELECT value FROM settings WHERE key = ?').get('k1') + assert.equal(row.value, '"v1"') + snap.close() +}) + +test('withTransaction commits and rolls back', () => { + const db = openDatabase({ dataDir: tmp }) + withTransaction(db, () => { + db.prepare("INSERT INTO settings (key, value) VALUES ('a', '1')").run() + }) + assert.ok(db.prepare("SELECT * FROM settings WHERE key='a'").get()) + assert.throws( + () => + withTransaction(db, () => { + db.prepare("INSERT INTO settings (key, value) VALUES ('b', '2')").run() + throw new Error('boom') + }), + /boom/, + ) + assert.equal(db.prepare("SELECT * FROM settings WHERE key='b'").get(), undefined) +}) + +test('withTransaction only runs inline for a real outer transaction', () => { + const db = openDatabase({ dataDir: tmp }) + db.exec('BEGIN') + withTransaction(db, () => { + db.prepare("INSERT INTO settings (key, value) VALUES ('nested', '1')").run() + }) + assert.equal(db.isTransaction, true) + db.exec('ROLLBACK') + assert.equal(db.prepare("SELECT * FROM settings WHERE key='nested'").get(), undefined) + + const closed = new DatabaseSync(':memory:') + closed.close() + let called = false + assert.throws( + () => + withTransaction(closed, () => { + called = true + }), + /not open|closed/i, + ) + assert.equal(called, false) +}) From 016efae88927c774282cd29aa01346b06a6f5bcc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:25:39 +0800 Subject: [PATCH 003/473] test(cluster): expose reconnect race during asynchronous node edits --- test/unit/cluster-node-edit.test.mjs | 172 +++++++++++++++++---------- 1 file changed, 107 insertions(+), 65 deletions(-) diff --git a/test/unit/cluster-node-edit.test.mjs b/test/unit/cluster-node-edit.test.mjs index 3a1af006a..342be9757 100644 --- a/test/unit/cluster-node-edit.test.mjs +++ b/test/unit/cluster-node-edit.test.mjs @@ -1,9 +1,8 @@ import test from 'node:test' import assert from 'node:assert/strict' -import fs from 'node:fs' import http from 'node:http' import net from 'node:net' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import ssh2 from 'ssh2' import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' import { ClusterError, connectSsh, hostKeyDigest, SshLink } from '../../src/lib/cluster/ssh-link.mjs' @@ -11,9 +10,6 @@ import { dockerPing, sshDocker } from '../../src/lib/cluster/docker-remote.mjs' import { ClusterNodesRepo } from '../../src/lib/db/repos/cluster-nodes-repo.mjs' const { Server, utils } = ssh2 -const schema = ['026_cluster_nodes.sql', '039_cluster_node_direct_links.sql'] - .map((name) => fs.readFileSync(new URL(`../../src/lib/db/migrations/${name}`, import.meta.url), 'utf8')) - .join('\n') const key = () => utils.generateKeyPairSync('ecdsa', { bits: 256 }) const defer = () => { let resolve @@ -109,10 +105,9 @@ async function sshHost(t, options = {}) { async function fixture(t, opts = {}) { const remote = await sshHost(t) - const db = new DatabaseSync(':memory:') - db.exec(schema) + const db = getDb() const repo = new ClusterNodesRepo(db) - repo.insert( + await repo.insert( { id: 'node-a', label: 'A', @@ -134,12 +129,13 @@ async function fixture(t, opts = {}) { assertNodeEditable: opts.assertNodeEditable, logger: { log() {}, warn() {} }, }) - const link = new SshLink({ id: 'node-a', resolveTarget: () => manager._targetFor('node-a') }) + await manager.reload() + const link = new SshLink({ id: 'node-a', resolveTarget: async () => await manager._targetFor('node-a') }) manager.links.set('node-a', link) if (opts.ready !== false) link.adopt(await connectSsh(await manager._targetFor('node-a'))) t.after(async () => { await manager.stop() - db.close() + await db.close() }) return { ...remote, @@ -148,7 +144,7 @@ async function fixture(t, opts = {}) { manager, link, vms, - patch: (change) => ({ expected_updated_at: repo.get('node-a').updated_at, ...change }), + patch: async (change) => ({ expected_updated_at: (await repo.get('node-a')).updated_at, ...change }), } } @@ -198,18 +194,24 @@ test('rename changes only metadata, even when the node has VMs or jump dependent }, }) slot(f) - f.repo.insert({ ...f.repo.get('node-a'), id: 'node-b', port: f.port + 1, jump_node_id: 'node-a', password: 'pw' }) + await f.repo.insert({ + ...(await f.repo.get('node-a')), + id: 'node-b', + port: f.port + 1, + jump_node_id: 'node-a', + password: 'pw', + }) const client = f.link.client const authCount = f.state.auth.length - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const getSecrets = f.repo.getSecrets.bind(f.repo) f.repo.getSecrets = () => { throw new Error('stored credentials cannot be decrypted') } - const probe = await f.manager.probeUpdate('node-a', f.patch({ label: 'Renamed' })) + const probe = await f.manager.probeUpdate('node-a', await f.patch({ label: 'Renamed' })) assert.equal(probe.vm_count, 1) assert.equal(probe.dependent_count, 1) - const result = await f.manager.update('node-a', f.patch({ label: 'Renamed' })) + const result = await f.manager.update('node-a', await f.patch({ label: 'Renamed' })) assert.equal(result.label, 'Renamed') assert.equal(result.id, before.id) assert.equal(result.created_at, before.created_at) @@ -217,7 +219,7 @@ test('rename changes only metadata, even when the node has VMs or jump dependent assert.equal(f.state.auth.length, authCount) assert.deepEqual(f.state.requests, []) assert.equal(JSON.stringify(result).includes('old-password'), false) - assert.equal(getSecrets('node-a').password, 'old-password') + assert.equal((await getSecrets('node-a')).password, 'old-password') }) test('successful credential rotation revalidates on save, reuses the link and clears old shell tickets', async (t) => { @@ -225,7 +227,7 @@ test('successful credential rotation revalidates on save, reuses the link and cl const client = f.link.client const relayClient = () => (f.link.state === 'ready' ? f.link.client : null) const ticket = f.manager.issueShellTicket('node-a').ticket - const patch = f.patch({ password: 'new-password' }) + const patch = await f.patch({ password: 'new-password' }) const preview = await f.manager.probeUpdate('node-a', patch) assert.deepEqual(preview.docker, { ok: true, error: null }) assert.equal(preview.host_key.sha256, f.sha256) @@ -236,7 +238,7 @@ test('successful credential rotation revalidates on save, reuses the link and cl assert.ok(f.state.auth.length > authCount, 'save opens its own verified candidate') assert.equal(f.manager.links.get('node-a'), f.link) assert.notEqual(relayClient(), client) - assert.equal(f.repo.getSecrets('node-a').password, 'new-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'new-password') assert.equal(result.link.state, 'ready') assert.equal(f.manager.consumeShellTicket(ticket, 'node-a'), false) assert.equal(await dockerPing(sshDocker(relayClient())), true) @@ -245,13 +247,13 @@ test('successful credential rotation revalidates on save, reuses the link and cl test('a successful preview is not a credential authorization for a later save', async (t) => { const f = await fixture(t) - const patch = f.patch({ password: 'new-password' }) + const patch = await f.patch({ password: 'new-password' }) await f.manager.probeUpdate('node-a', patch) f.state.passwords.delete('new-password') const client = f.link.client await assert.rejects(f.manager.update('node-a', patch), { code: 'auth_failed' }) assert.equal(f.link.client, client) - assert.equal(f.repo.getSecrets('node-a').password, 'old-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') }) test('resubmitted saved credentials or equivalent endpoints are authenticated instead of trusting cached health', async (t) => { @@ -262,14 +264,14 @@ test('resubmitted saved credentials or equivalent endpoints are authenticated in f.state.passwords.delete('old-password') // A previously successful observation must not authorize a new request. f.manager.health.set('node-a', { docker: { ok: true, error: null } }) - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const client = f.link.client for (const change of [{ password: 'old-password' }, { host: ' 127.0.0.1 ' }, { port: String(f.port) }]) { const count = f.state.auth.length - await assert.rejects(f.manager.probeUpdate('node-a', f.patch(change)), { code: 'auth_failed' }) - await assert.rejects(f.manager.update('node-a', f.patch(change)), { code: 'auth_failed' }) + await assert.rejects(f.manager.probeUpdate('node-a', await f.patch(change)), { code: 'auth_failed' }) + await assert.rejects(f.manager.update('node-a', await f.patch(change)), { code: 'auth_failed' }) assert.ok(f.state.auth.length > count) - assert.deepEqual(f.repo.get('node-a'), before) + assert.deepEqual(await f.repo.get('node-a'), before) assert.equal(f.link.client, client) } }) @@ -280,20 +282,20 @@ test('password to key rotation clears the saved password and remains usable', as const f = await fixture(t) const userKey = key() f.state.publicKey = Buffer.from(userKey.public.split(' ')[1], 'base64') - await f.manager.update('node-a', f.patch({ auth_type: 'key', private_key: userKey.private })) - assert.equal(f.repo.get('node-a').auth_type, 'key') - assert.equal(f.repo.getSecrets('node-a').password, null) - assert.equal(f.repo.getSecrets('node-a').privateKey, userKey.private.trim()) + await f.manager.update('node-a', await f.patch({ auth_type: 'key', private_key: userKey.private })) + assert.equal((await f.repo.get('node-a')).auth_type, 'key') + assert.equal((await f.repo.getSecrets('node-a')).password, null) + assert.equal((await f.repo.getSecrets('node-a')).privateKey, userKey.private.trim()) assert.equal(f.link.state, 'ready') }) test('candidate authentication failure leaves saved credentials and old connection intact', async (t) => { const f = await fixture(t) - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'wrong' })), { code: 'auth_failed' }) - assert.deepEqual(f.repo.get('node-a'), before) - assert.equal(f.repo.getSecrets('node-a').password, 'old-password') + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'wrong' })), { code: 'auth_failed' }) + assert.deepEqual(await f.repo.get('node-a'), before) + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') assert.equal(f.link.client, client) await dockerPing(sshDocker(client)) }) @@ -302,30 +304,34 @@ test('changed endpoints must have the saved host key before credentials are offe const f = await fixture(t) const impostor = await sshHost(t) const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ port: impostor.port })), { code: 'host_key_mismatch' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ port: impostor.port })), { + code: 'host_key_mismatch', + }) assert.deepEqual(impostor.state.auth, []) - assert.equal(f.repo.get('node-a').port, f.port) + assert.equal((await f.repo.get('node-a')).port, f.port) assert.equal(f.link.client, client) const sameHost = await sshHost(t, { hostKey: f.hostKey }) - await f.manager.update('node-a', f.patch({ port: sameHost.port })) - assert.equal(f.repo.get('node-a').port, sameHost.port) + await f.manager.update('node-a', await f.patch({ port: sameHost.port })) + assert.equal((await f.repo.get('node-a')).port, sameHost.port) }) test('empty nodes can change username and save without Docker, but existing slots require Docker', async (t) => { const f = await fixture(t) f.state.username = 'nextuser' f.state.docker = false - const patch = f.patch({ username: 'nextuser' }) + const patch = await f.patch({ username: 'nextuser' }) const probe = await f.manager.probeUpdate('node-a', patch) assert.equal(probe.docker.ok, false) await f.manager.update('node-a', patch) - assert.equal(f.repo.get('node-a').username, 'nextuser') + assert.equal((await f.repo.get('node-a')).username, 'nextuser') slot(f) const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_runtime_mismatch', }) - await assert.rejects(f.manager.update('node-a', f.patch({ username: 'third' })), { code: 'node_username_in_use' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ username: 'third' })), { + code: 'node_username_in_use', + }) assert.equal(f.link.client, client) }) @@ -335,13 +341,13 @@ test('VM identity and Docker/KVM mount roots are verified without changing conta const f = await fixture(t) const vm = slot(f, runtime) const snapshot = JSON.stringify(vm) - await f.manager.update('node-a', f.patch({ password: 'new-password' })) + await f.manager.update('node-a', await f.patch({ password: 'new-password' })) assert.equal(JSON.stringify(vm), snapshot) assert.ok(f.state.requests.every(([method]) => method === 'GET')) assert.ok(f.state.commands.every((command) => command.startsWith('sh -c ') || command.startsWith('test -d '))) f.state.containers.get('container-a').Mounts[0].Source = '/elsewhere/run' const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'old-password' })), { + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'old-password' })), { code: 'node_runtime_mismatch', }) assert.equal(f.link.client, client) @@ -360,12 +366,12 @@ test('changed HOME, UID/GID, missing container or directories reject an attached if (change === 'gid') f.state.gid = 999 if (change === 'container') f.state.containers.clear() if (change === 'directories') f.state.directories = false - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') const client = f.link.client - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_runtime_mismatch', }) - assert.deepEqual(f.repo.get('node-a'), before) + assert.deepEqual(await f.repo.get('node-a'), before) assert.equal(f.link.client, client) }) } @@ -374,38 +380,38 @@ test('changed HOME, UID/GID, missing container or directories reject an attached test('a disconnected node can recover using its pinned identity and recorded runtime', async (t) => { const f = await fixture(t, { ready: false }) slot(f) - await f.manager.update('node-a', f.patch({ password: 'new-password' })) + await f.manager.update('node-a', await f.patch({ password: 'new-password' })) assert.equal(f.link.state, 'ready') - assert.equal(f.repo.getSecrets('node-a').password, 'new-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'new-password') }) test('connection updates reject jump dependents, duplicate endpoints and busy installs before connecting', async (t) => { const f = await fixture(t) - const next = { ...f.repo.get('node-a'), id: 'node-b', port: f.port + 1, password: 'pw' } - f.repo.insert({ ...next, jump_node_id: 'node-a' }) + const next = { ...(await f.repo.get('node-a')), id: 'node-b', port: f.port + 1, password: 'pw' } + await f.repo.insert({ ...next, jump_node_id: 'node-a' }) const count = f.state.auth.length - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { code: 'node_in_use' }) - f.repo.remove('node-b') - f.repo.insert(next) - await assert.rejects(f.manager.update('node-a', f.patch({ port: next.port })), { code: 'node_exists' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_in_use' }) + await f.repo.remove('node-b') + await f.repo.insert(next) + await assert.rejects(f.manager.update('node-a', await f.patch({ port: next.port })), { code: 'node_exists' }) f.manager.installs.set('node-a', { status: 'running' }) - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), { code: 'node_busy' }) + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), { code: 'node_busy' }) assert.equal(f.state.auth.length, count) }) test('stale edits and database failures do not replace a working link', async (t) => { const f = await fixture(t) - const stale = f.patch({ password: 'new-password' }) - await f.manager.update('node-a', f.patch({ label: 'new label' })) + const stale = await f.patch({ password: 'new-password' }) + await f.manager.update('node-a', await f.patch({ label: 'new label' })) await assert.rejects(f.manager.probeUpdate('node-a', stale), { code: 'node_edit_conflict' }) await assert.rejects(f.manager.update('node-a', stale), { code: 'node_edit_conflict' }) const client = f.link.client - const before = f.repo.get('node-a') + const before = await f.repo.get('node-a') f.repo.update = () => { throw new Error('write failed') } - await assert.rejects(f.manager.update('node-a', f.patch({ password: 'new-password' })), /write failed/) - assert.deepEqual(f.repo.get('node-a'), before) + await assert.rejects(f.manager.update('node-a', await f.patch({ password: 'new-password' })), /write failed/) + assert.deepEqual(await f.repo.get('node-a'), before) assert.equal(f.link.client, client) await dockerPing(sshDocker(client)) }) @@ -424,14 +430,14 @@ test('the final guard rejects a node that becomes busy while its candidate is ch await release.promise } const client = f.link.client - const pending = f.manager.update('node-a', f.patch({ password: 'new-password' })) + const pending = f.manager.update('node-a', await f.patch({ password: 'new-password' })) const rejected = assert.rejects(pending, { code: 'node_busy' }) await entered.promise busy = true release.resolve() await rejected assert.equal(f.link.client, client) - assert.equal(f.repo.getSecrets('node-a').password, 'old-password') + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') }) test('reconnect, removal and another save during verification cannot resurrect stale configuration', async (t) => { @@ -444,17 +450,53 @@ test('reconnect, removal and another save during verification cannot resurrect s entered.resolve() await release.promise } - const pending = f.manager.update('node-a', f.patch({ password: 'new-password' })) + const pending = f.manager.update('node-a', await f.patch({ password: 'new-password' })) const rejected = assert.rejects(pending, { code: action === 'remove' ? 'node_not_found' : 'node_edit_conflict' }) await entered.promise if (action === 'reconnect') f.manager.reconnect('node-a') if (action === 'remove') await f.manager.remove('node-a') - if (action === 'rename') await f.manager.update('node-a', f.patch({ label: 'concurrent' })) + if (action === 'rename') await f.manager.update('node-a', await f.patch({ label: 'concurrent' })) if (action === 'placement') f.vms.push({ id: 'new-vm' }) release.resolve() await rejected - if (action !== 'remove') assert.equal(f.repo.getSecrets('node-a').password, 'old-password') - else assert.equal(f.repo.get('node-a'), null) + if (action !== 'remove') assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') + else assert.equal(await f.repo.get('node-a'), null) }) } }) + +test('reconnect during the database compare-and-swap rolls back credentials and closes the candidate', async (t) => { + const f = await fixture(t) + const before = await f.repo.get('node-a') + const entered = defer(), + release = defer(), + closed = defer() + const update = f.repo.update.bind(f.repo) + f.repo.update = async (...args) => { + const saved = await update(...args) + entered.resolve() + await release.promise + return saved + } + const probe = f.manager._probeEdit.bind(f.manager) + f.manager._probeEdit = async (...args) => { + const candidate = await probe(...args) + candidate.client.once('close', closed.resolve) + return candidate + } + const pending = f.manager.update('node-a', await f.patch({ password: 'new-password' })) + const rejected = assert.rejects(pending, { code: 'node_edit_conflict' }) + await entered.promise + f.manager.reconnect('node-a') + release.resolve() + await rejected + await Promise.race([ + closed.promise, + new Promise((_resolve, reject) => { + const timer = setTimeout(() => reject(new Error('candidate was not closed')), 1500) + timer.unref?.() + }), + ]) + assert.deepEqual(await f.repo.get('node-a'), before) + assert.equal((await f.repo.getSecrets('node-a')).password, 'old-password') +}) From 3225b07381b5a076e5582dd527a64e1c628dd422 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:31:14 +0800 Subject: [PATCH 004/473] feat(db): migrate repositories and account stores to PostgreSQL --- src/lib/admin/api-endpoints.mjs | 172 +++++---- src/lib/admin/api-keys.mjs | 228 ++++++----- src/lib/admin/panel-users.mjs | 146 +++---- src/lib/db/repos/account-runtime-repo.mjs | 355 ++++++++++-------- src/lib/db/repos/accounts-repo.mjs | 178 ++++----- src/lib/db/repos/api-endpoints-repo.mjs | 176 +++++---- src/lib/db/repos/api-keys-repo.mjs | 131 ++++--- src/lib/db/repos/backup-repo.mjs | 38 +- src/lib/db/repos/cluster-nodes-repo.mjs | 92 ++--- src/lib/db/repos/groups-repo.mjs | 34 +- src/lib/db/repos/proxies-repo.mjs | 22 +- src/lib/db/repos/redeem-codes-repo.mjs | 45 ++- .../db/repos/refusal-device-blocks-repo.mjs | 44 +-- src/lib/db/repos/refusal-guards-repo.mjs | 59 ++- src/lib/db/repos/request-attempts-repo.mjs | 34 +- src/lib/db/repos/settings-repo.mjs | 31 +- src/lib/db/repos/sticky-repo.mjs | 52 +-- src/lib/db/repos/vm-pools-repo.mjs | 110 +++--- src/lib/db/repos/vms-repo.mjs | 47 ++- test/unit/api-endpoints.test.mjs | 24 +- test/unit/api-keys.test.mjs | 229 ++++++----- test/unit/cluster-node-edit-repo.test.mjs | 140 +++---- test/unit/panel-users.test.mjs | 200 ++++++---- test/unit/pool-runtime-repos.test.mjs | 100 ++--- test/unit/refusal-guard.test.mjs | 54 +-- test/unit/refusal-similarity.test.mjs | 46 +-- test/unit/tenant-billing.test.mjs | 46 +-- test/unit/vm-pools.test.mjs | 48 +-- 28 files changed, 1534 insertions(+), 1347 deletions(-) diff --git a/src/lib/admin/api-endpoints.mjs b/src/lib/admin/api-endpoints.mjs index 3a33215bd..efc257f3b 100644 --- a/src/lib/admin/api-endpoints.mjs +++ b/src/lib/admin/api-endpoints.mjs @@ -1,5 +1,5 @@ import crypto from 'node:crypto' -import { resolveStoreDb } from '../db/database.mjs' +import { resolveStoreDb, withTransaction } from '../db/database.mjs' import { ApiEndpointsRepo } from '../db/repos/api-endpoints-repo.mjs' import { mergeModelLists, @@ -101,93 +101,99 @@ export class ApiEndpointStore { this.repo = new ApiEndpointsRepo(db) } - list({ reveal = false } = {}) { - return this.repo.listHydrated().map((ep) => publicEndpointView(ep, { reveal })) + async list({ reveal = false } = {}) { + return (await this.repo.listHydrated()).map((ep) => publicEndpointView(ep, { reveal })) } - listRaw() { - return this.repo.listHydrated() + async listRaw() { + return await this.repo.listHydrated() } - getRaw(id) { - return this.repo.getHydrated(id) + async getRaw(id) { + return await this.repo.getHydrated(id) } - create(input = {}) { - const preset = resolvePreset(input.kind, input) - const base_url = preset.base_url - if (!/^https?:\/\//i.test(base_url)) { - throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) - } - const rec = { - id: nid('ep_'), - name: - String(preset.name || 'api') - .trim() - .slice(0, 80) || 'api', - disabled: !!input.disabled, - prefix: String(input.prefix || '').trim(), - base_url, - kind: preset.kind, - protocol: preset.protocol, - headers: headersWithAuthScheme( - input.headers && typeof input.headers === 'object' ? input.headers : {}, - input.auth_scheme || input.authScheme, - ), - disable_cooling: !!input.disable_cooling, - priority: Number(input.priority) || 0, - created_at: nowIso(), - updated_at: nowIso(), - } - const stored = this.repo.insertEndpoint(rec) - const models = Array.isArray(input.models) ? input.models : [] - if (models.length) this.replaceModels(stored.id, models) - const keys = Array.isArray(input.api_key_entries) ? input.api_key_entries : [] - for (const k of keys) this.addKey(stored.id, k) - return this.repo.getHydrated(stored.id) + async create(input = {}) { + return withTransaction(this.db, async () => { + const preset = resolvePreset(input.kind, input) + const base_url = preset.base_url + if (!/^https?:\/\//i.test(base_url)) { + throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) + } + const rec = { + id: nid('ep_'), + name: + String(preset.name || 'api') + .trim() + .slice(0, 80) || 'api', + disabled: !!input.disabled, + prefix: String(input.prefix || '').trim(), + base_url, + kind: preset.kind, + protocol: preset.protocol, + headers: headersWithAuthScheme( + input.headers && typeof input.headers === 'object' ? input.headers : {}, + input.auth_scheme || input.authScheme, + ), + disable_cooling: !!input.disable_cooling, + priority: Number(input.priority) || 0, + created_at: nowIso(), + updated_at: nowIso(), + } + const stored = await this.repo.insertEndpoint(rec) + const models = Array.isArray(input.models) ? input.models : [] + if (models.length) await this.replaceModels(stored.id, models) + const keys = Array.isArray(input.api_key_entries) ? input.api_key_entries : [] + for (const k of keys) await this.addKey(stored.id, k) + return await this.repo.getHydrated(stored.id) + }) } - update(id, patch = {}) { - const rec = this.repo.getHydrated(id) - if (!rec) return null - if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 80) || rec.name - if (patch.disabled != null) rec.disabled = !!patch.disabled - if (patch.prefix != null) rec.prefix = String(patch.prefix).trim() - if (patch.kind != null || patch.protocol != null || patch.base_url != null) { - const preset = resolvePreset(patch.kind ?? rec.kind, { - ...rec, - ...patch, - base_url: patch.base_url != null ? patch.base_url : rec.base_url, - }) - rec.kind = preset.kind - rec.protocol = preset.protocol - rec.base_url = preset.base_url - if (!/^https?:\/\//i.test(rec.base_url)) { - throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) + async update(id, patch = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM api_endpoints WHERE id = $1 FOR UPDATE').get(id) + + const rec = await this.repo.getHydrated(id) + if (!rec) return null + if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 80) || rec.name + if (patch.disabled != null) rec.disabled = !!patch.disabled + if (patch.prefix != null) rec.prefix = String(patch.prefix).trim() + if (patch.kind != null || patch.protocol != null || patch.base_url != null) { + const preset = resolvePreset(patch.kind ?? rec.kind, { + ...rec, + ...patch, + base_url: patch.base_url != null ? patch.base_url : rec.base_url, + }) + rec.kind = preset.kind + rec.protocol = preset.protocol + rec.base_url = preset.base_url + if (!/^https?:\/\//i.test(rec.base_url)) { + throw Object.assign(new Error('base_url must be http(s)'), { code: 'invalid_base_url' }) + } } - } - if (patch.headers != null && typeof patch.headers === 'object') rec.headers = patch.headers - if (patch.auth_scheme != null || patch.authScheme != null) { - rec.headers = headersWithAuthScheme(rec.headers, patch.auth_scheme || patch.authScheme) - } - if (patch.disable_cooling != null) rec.disable_cooling = !!patch.disable_cooling - if (patch.priority != null) rec.priority = Number(patch.priority) || 0 - rec.updated_at = nowIso() - this.repo.updateEndpoint(rec) - if (Array.isArray(patch.models)) this.replaceModels(id, patch.models) - return this.repo.getHydrated(id) + if (patch.headers != null && typeof patch.headers === 'object') rec.headers = patch.headers + if (patch.auth_scheme != null || patch.authScheme != null) { + rec.headers = headersWithAuthScheme(rec.headers, patch.auth_scheme || patch.authScheme) + } + if (patch.disable_cooling != null) rec.disable_cooling = !!patch.disable_cooling + if (patch.priority != null) rec.priority = Number(patch.priority) || 0 + rec.updated_at = nowIso() + await this.repo.updateEndpoint(rec) + if (Array.isArray(patch.models)) await this.replaceModels(id, patch.models) + return await this.repo.getHydrated(id) + }) } - remove(id) { - return this.repo.removeEndpoint(id) + async remove(id) { + return await this.repo.removeEndpoint(id) } - addKey(endpointId, input = {}) { - const ep = this.repo.getHydrated(endpointId) + async addKey(endpointId, input = {}) { + const ep = await this.repo.getHydrated(endpointId) if (!ep) throw Object.assign(new Error('endpoint not found'), { code: 'endpoint_not_found' }) const api_key = String(input.api_key || '').trim() if (api_key.length < 4) throw Object.assign(new Error('api_key too short'), { code: 'key_too_short' }) - return this.repo.insertKey({ + return await this.repo.insertKey({ id: nid('ek_'), endpoint_id: endpointId, api_key, @@ -199,15 +205,15 @@ export class ApiEndpointStore { }) } - removeKey(id) { - return this.repo.removeKey(id) + async removeKey(id) { + return await this.repo.removeKey(id) } - updateKey(id, patch = {}) { - return this.repo.updateKey(id, patch) + async updateKey(id, patch = {}) { + return await this.repo.updateKey(id, patch) } - replaceModels(endpointId, models = []) { + async replaceModels(endpointId, models = []) { const rows = models .map((m, i) => ({ id: m.id || nid('em_'), @@ -218,17 +224,17 @@ export class ApiEndpointStore { thinking: m.thinking, })) .filter((m) => m.name && m.alias) - return this.repo.replaceModels(endpointId, rows) + return await this.repo.replaceModels(endpointId, rows) } - setKeyCooldown(id, untilIso) { - this.repo.setKeyCooldown(id, untilIso) + async setKeyCooldown(id, untilIso) { + await this.repo.setKeyCooldown(id, untilIso) } - mergeFetchedModels(endpointId, fetched = []) { - const ep = this.repo.getHydrated(endpointId) + async mergeFetchedModels(endpointId, fetched = []) { + const ep = await this.repo.getHydrated(endpointId) if (!ep) return null - return this.replaceModels(endpointId, mergeModelLists(ep.models, fetched)) + return await this.replaceModels(endpointId, mergeModelLists(ep.models, fetched)) } } diff --git a/src/lib/admin/api-keys.mjs b/src/lib/admin/api-keys.mjs index 61db2f12a..a03a1f909 100644 --- a/src/lib/admin/api-keys.mjs +++ b/src/lib/admin/api-keys.mjs @@ -1,5 +1,5 @@ /** - * Multi API-key store (sub2api-inspired, SQLite-backed). + * Multi API-key store (sub2api-inspired, PostgreSQL-backed). * * Fields aligned to sub2api basics: * name, status, expires_at, quota (+ used), rate windows → rpm, @@ -18,7 +18,7 @@ import { keyScopeFromRecord, normalizeKeyScope } from './key-scope.mjs' import { VmPoolsRepo } from '../db/repos/vm-pools-repo.mjs' import crypto from 'node:crypto' -import { resolveStoreDb } from '../db/database.mjs' +import { resolveStoreDb, withTransaction } from '../db/database.mjs' import { ApiKeysRepo } from '../db/repos/api-keys-repo.mjs' import { maybeEncrypt, maybeDecrypt } from '../db/secure.mjs' import { calculateCost, normalizeUsage } from './pricing.mjs' @@ -139,10 +139,10 @@ export class ApiKeyStore { } /** sub2api: an api key of a disabled/deleted user must not authenticate. */ - _ownerActive(rec) { + async _ownerActive(rec) { if (!rec?.user_id) return true // legacy unowned keys keep working try { - const owner = this.users.getById(rec.user_id) + const owner = await this.users.getById(rec.user_id) // soft-deleted owner is invisible here → key is revoked with them return !!owner && owner.status === 'active' } catch { @@ -160,14 +160,14 @@ export class ApiKeyStore { this.users = new UsersRepo(db) } - list({ reveal = false } = {}) { + async list({ reveal = false } = {}) { const names = new Map() try { - for (const pool of new VmPoolsRepo(this.db).list()) names.set(pool.id, pool.name) + for (const pool of await new VmPoolsRepo(this.db).list()) names.set(pool.id, pool.name) } catch { // A database opened before migration 034 still lists keys. } - return this.repo.list().map((k) => { + return (await this.repo.list()).map((k) => { const v = publicKeyView(k, { reveal }) v.inflight = this.inflight.get(k.id) || 0 if (v.vm_pool_id) v.vm_pool_name = names.get(v.vm_pool_id) || null @@ -175,34 +175,38 @@ export class ApiKeyStore { }) } - getById(id) { - return this.repo.getById(id) + async getById(id) { + return await this.repo.getById(id) } - authenticate(token) { + async authenticate(token) { if (!token) return { ok: false, reason: 'missing' } const hash = this._hash(token) - const indexed = this.repo.getByHash(hash) + const indexed = await this.repo.getByHash(hash) if (indexed) { - if (!this._ownerActive(indexed)) return { ok: false, reason: 'user_disabled' } + if (!(await this._ownerActive(indexed))) return { ok: false, reason: 'user_disabled' } return { ok: true, record: indexed } } // One-time migration for legacy plaintext rows. - for (const rec of this.repo.list()) { + for (const rec of await this.repo.list()) { if (String(rec.key || '').startsWith(HASH_MARKER)) continue if (timingSafeEqualStr(token, rec.key)) { - rec.key_hash = hash - rec.key_prefix = String(token).slice(0, 10) - rec.key_suffix = String(token).slice(-4) - rec.key_secret = maybeEncrypt(token) - rec.key = HASH_MARKER + hash - return { ok: true, record: this.repo.update(rec) } + return withTransaction(this.db, async () => { + const current = await this.repo.getById(rec.id, { forUpdate: true }) + if (!current || !timingSafeEqualStr(token, current.key)) return { ok: false, reason: 'invalid' } + current.key_hash = hash + current.key_prefix = String(token).slice(0, 10) + current.key_suffix = String(token).slice(-4) + current.key_secret = maybeEncrypt(token) + current.key = HASH_MARKER + hash + return { ok: true, record: await this.repo.update(current) } + }) } } return { ok: false, reason: 'invalid' } } - create(input = {}) { + async create(input = {}) { const name = String(input.name || 'default') .trim() @@ -213,7 +217,7 @@ export class ApiKeyStore { throw Object.assign(new Error('key has invalid characters'), { code: 'key_invalid' }) } const keyHash = this._hash(key) - if (this.repo.getByHash(keyHash) || this.repo.getByKey(key)) { + if ((await this.repo.getByHash(keyHash)) || (await this.repo.getByKey(key))) { throw Object.assign(new Error('key already exists'), { code: 'key_exists' }) } @@ -256,65 +260,67 @@ export class ApiKeyStore { if (rec.expires_at && Number.isNaN(Date.parse(rec.expires_at))) { throw Object.assign(new Error('invalid expires_at'), { code: 'invalid_expires_at' }) } - const stored = this.repo.insert(rec) + const stored = await this.repo.insert(rec) return { ...stored, key, _plain_key: key } } - update(id, patch = {}) { - const rec = this.repo.getById(id) - if (!rec) return null - if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 100) || rec.name - if (patch.status != null) { - const st = String(patch.status).toLowerCase() - if (!['active', 'disabled'].includes(st)) { - throw Object.assign(new Error('invalid status'), { code: 'invalid_status' }) + async update(id, patch = {}) { + return withTransaction(this.db, async () => { + const rec = await this.repo.getById(id, { forUpdate: true }) + if (!rec) return null + if (patch.name != null) rec.name = String(patch.name).trim().slice(0, 100) || rec.name + if (patch.status != null) { + const st = String(patch.status).toLowerCase() + if (!['active', 'disabled'].includes(st)) { + throw Object.assign(new Error('invalid status'), { code: 'invalid_status' }) + } + rec.status = st } - rec.status = st - } - if (patch.max_concurrency != null) - rec.max_concurrency = clampInt(patch.max_concurrency, 0, 128, rec.max_concurrency) - if (patch.quota_requests != null) rec.quota_requests = clampInt(patch.quota_requests, 0, 1e9, rec.quota_requests) - if (patch.quota != null) rec.quota = Math.max(0, Number(patch.quota) || 0) - if (patch.rate_limit_5h != null) rec.rate_limit_5h = Math.max(0, Number(patch.rate_limit_5h) || 0) - if (patch.rate_limit_1d != null) rec.rate_limit_1d = Math.max(0, Number(patch.rate_limit_1d) || 0) - if (patch.rate_limit_7d != null) rec.rate_limit_7d = Math.max(0, Number(patch.rate_limit_7d) || 0) - if (patch.group_id != null) rec.group_id = Number(patch.group_id) || 1 - if (patch.user_id != null) rec.user_id = patch.user_id - if (patch.rpm != null) rec.rpm = clampInt(patch.rpm, 0, 1e6, rec.rpm) - if (patch.expires_at === null) rec.expires_at = null - else if (patch.expires_at != null) { - const iso = new Date(patch.expires_at).toISOString() - if (Number.isNaN(Date.parse(iso))) - throw Object.assign(new Error('invalid expires_at'), { code: 'invalid_expires_at' }) - rec.expires_at = iso - } - if (patch.reset_quota === true) { - rec.quota_requests_used = 0 - rec.quota_used = 0 - } - // recover from quota_exhausted whenever spend is back under the cap - if ( - rec.status === 'quota_exhausted' && - !((rec.quota ?? 0) > 0 && (rec.quota_used ?? 0) >= rec.quota) && - !(rec.quota_requests > 0 && (rec.quota_requests_used ?? 0) >= rec.quota_requests) - ) { - rec.status = 'active' - } - if (patch.category != null) { - rec.category = String(patch.category).toLowerCase() === 'api' ? 'api' : 'oauth' - } - const scope = normalizeKeyScope(patch, { partial: true, current: rec }) - if (scope) Object.assign(rec, storedKeyScope(scope)) - rec.updated_at = nowIso() - return this.repo.update(rec) + if (patch.max_concurrency != null) + rec.max_concurrency = clampInt(patch.max_concurrency, 0, 128, rec.max_concurrency) + if (patch.quota_requests != null) rec.quota_requests = clampInt(patch.quota_requests, 0, 1e9, rec.quota_requests) + if (patch.quota != null) rec.quota = Math.max(0, Number(patch.quota) || 0) + if (patch.rate_limit_5h != null) rec.rate_limit_5h = Math.max(0, Number(patch.rate_limit_5h) || 0) + if (patch.rate_limit_1d != null) rec.rate_limit_1d = Math.max(0, Number(patch.rate_limit_1d) || 0) + if (patch.rate_limit_7d != null) rec.rate_limit_7d = Math.max(0, Number(patch.rate_limit_7d) || 0) + if (patch.group_id != null) rec.group_id = Number(patch.group_id) || 1 + if (patch.user_id != null) rec.user_id = patch.user_id + if (patch.rpm != null) rec.rpm = clampInt(patch.rpm, 0, 1e6, rec.rpm) + if (patch.expires_at === null) rec.expires_at = null + else if (patch.expires_at != null) { + const iso = new Date(patch.expires_at).toISOString() + if (Number.isNaN(Date.parse(iso))) + throw Object.assign(new Error('invalid expires_at'), { code: 'invalid_expires_at' }) + rec.expires_at = iso + } + if (patch.reset_quota === true) { + rec.quota_requests_used = 0 + rec.quota_used = 0 + } + // recover from quota_exhausted whenever spend is back under the cap + if ( + rec.status === 'quota_exhausted' && + !((rec.quota ?? 0) > 0 && (rec.quota_used ?? 0) >= rec.quota) && + !(rec.quota_requests > 0 && (rec.quota_requests_used ?? 0) >= rec.quota_requests) + ) { + rec.status = 'active' + } + if (patch.category != null) { + rec.category = String(patch.category).toLowerCase() === 'api' ? 'api' : 'oauth' + } + const scope = normalizeKeyScope(patch, { partial: true, current: rec }) + if (scope) Object.assign(rec, storedKeyScope(scope)) + rec.updated_at = nowIso() + return await this.repo.update(rec) + }) } /** * Plaintext of a managed key for panel copy. * @returns {null | { ok: false, code: string } | { ok: true, id, name, key }} */ - reveal(id) { - const rec = this.repo.getById(id) + async reveal(id) { + const rec = await this.repo.getById(id, { forUpdate: true }) if (!rec) return null const key = plainApiKey(rec) if (!key) return { ok: false, code: 'key_not_recoverable' } @@ -322,26 +328,28 @@ export class ApiKeyStore { } /** Issue a fresh plaintext on the same record; the old value stops working. */ - rotate(id) { - const rec = this.repo.getById(id) - if (!rec) return null - const key = generateApiKey() - const keyHash = this._hash(key) - if (this.repo.getByHash(keyHash)) { - throw Object.assign(new Error('key already exists'), { code: 'key_exists' }) - } - rec.key = HASH_MARKER + keyHash - rec.key_hash = keyHash - rec.key_prefix = key.slice(0, 10) - rec.key_suffix = key.slice(-4) - rec.key_secret = maybeEncrypt(key) - rec.updated_at = nowIso() - const stored = this.repo.update(rec) - return { ...stored, key, _plain_key: key } + async rotate(id) { + return withTransaction(this.db, async () => { + const rec = await this.repo.getById(id, { forUpdate: true }) + if (!rec) return null + const key = generateApiKey() + const keyHash = this._hash(key) + if (await this.repo.getByHash(keyHash)) { + throw Object.assign(new Error('key already exists'), { code: 'key_exists' }) + } + rec.key = HASH_MARKER + keyHash + rec.key_hash = keyHash + rec.key_prefix = key.slice(0, 10) + rec.key_suffix = key.slice(-4) + rec.key_secret = maybeEncrypt(key) + rec.updated_at = nowIso() + const stored = await this.repo.update(rec) + return { ...stored, key, _plain_key: key } + }) } - remove(id) { - const removed = this.repo.remove(id) + async remove(id) { + const removed = await this.repo.remove(id) if (!removed) return false this.inflight.delete(id) this.rpmBuckets.delete(id) @@ -365,7 +373,9 @@ export class ApiKeyStore { * Pre-flight gate for a managed key. Does not acquire concurrency. * @returns {{ ok: true } | { ok: false, code, message, status }} */ - canAccept(rec, now = Date.now()) { + async canAccept(rec, now = Date.now(), { reserve = false } = {}) { + const owner = rec?.user_id ? await this.users.getById(rec.user_id) : null + const ownedKeys = rec?.user_id ? await this.repo.list() : [] if (!rec) return { ok: false, code: 'invalid_api_key', message: 'Invalid credentials', status: 401 } if (rec.status !== 'active') { if (rec.status === 'quota_exhausted') { @@ -439,17 +449,10 @@ export class ApiKeyStore { // sub2api user-level cap: sum of inflight across all of the owner's keys. // The operator (admin) is never capped: its traffic is the platform's own. if (rec.user_id) { - const owner = (() => { - try { - return this.users.getById(rec.user_id) - } catch { - return null - } - })() const cap = owner?.role === 'admin' ? 0 : Number(owner?.concurrency) || 0 if (cap > 0) { let total = 0 - for (const k of this.repo.list()) { + for (const k of ownedKeys) { if (k.user_id === rec.user_id) total += this.inflight.get(k.id) || 0 } if (total >= cap) { @@ -463,18 +466,17 @@ export class ApiKeyStore { } } } + if (reserve) { + this.inflight.set(rec.id, (this.inflight.get(rec.id) || 0) + 1) + const arr = this.rpmBuckets.get(rec.id) || [] + arr.push(now) + this.rpmBuckets.set(rec.id, arr) + } return { ok: true } } - acquire(rec, now = Date.now()) { - const gate = this.canAccept(rec, now) - if (!gate.ok) return gate - const id = rec.id - this.inflight.set(id, (this.inflight.get(id) || 0) + 1) - const arr = this.rpmBuckets.get(id) || [] - arr.push(now) - this.rpmBuckets.set(id, arr) - return { ok: true } + async acquire(rec, now = Date.now()) { + return this.canAccept(rec, now, { reserve: true }) } release(recOrId) { @@ -485,9 +487,9 @@ export class ApiKeyStore { else this.inflight.set(id, n) } - recordUsage(recOrId, usage = {}, { model = null, rateMultiplier = 1, officialCost } = {}) { + async recordUsage(recOrId, usage = {}, { model = null, rateMultiplier = 1, officialCost } = {}) { const id = typeof recOrId === 'string' ? recOrId : recOrId?.id - if (!this.repo.getById(id)) return null + if (!(await this.repo.getById(id))) return null const normalized = normalizeUsage(usage) let billed = officialCost if (billed == null) billed = usage.total_cost @@ -501,22 +503,18 @@ export class ApiKeyStore { // key quota/windows track actual (group-rated) spend, sub2api-style const rate = Number.isFinite(Number(rateMultiplier)) ? Number(rateMultiplier) : 1 const cost = billed * rate - const rec = this.repo.recordUsage(id, { + const rec = await this.repo.recordUsage(id, { tokens_in: Number(normalized.input_tokens) || Number(normalized.tokens_in) || 0, tokens_out: Number(normalized.output_tokens) || Number(normalized.tokens_out) || 0, cache_read_tokens: Number(normalized.cache_read_input_tokens ?? normalized.cache_read_tokens) || 0, cache_creation_tokens: Number(normalized.cache_creation_input_tokens ?? normalized.cache_creation_tokens) || 0, cost, }) - // sub2api: the billing write flips the key when the USD quota runs out - if (rec && rec.status === 'active' && (rec.quota ?? 0) > 0 && (rec.quota_used ?? 0) >= rec.quota) { - this.repo.update({ ...rec, status: 'quota_exhausted', updated_at: nowIso() }) - } return rec } - snapshot() { - const keys = this.list({ reveal: false }) + async snapshot() { + const keys = await this.list({ reveal: false }) return { total: keys.length, active: keys.filter((k) => k.status === 'active').length, diff --git a/src/lib/admin/panel-users.mjs b/src/lib/admin/panel-users.mjs index 90a0787a1..f71725ff0 100644 --- a/src/lib/admin/panel-users.mjs +++ b/src/lib/admin/panel-users.mjs @@ -6,7 +6,7 @@ */ import crypto from 'node:crypto' -import { resolveStoreDb } from '../db/database.mjs' +import { resolveStoreDb, withTransaction } from '../db/database.mjs' import { UsersRepo } from '../db/repos/users-repo.mjs' import { clampVmCreateQuota } from './resource-owner.mjs' import { reclaimAssignedVms } from '../vm/vm-registry.mjs' @@ -129,30 +129,30 @@ export class PanelUserStore { this.repo = new UsersRepo(db) } - list() { - return this.repo.list().map(publicUserView) + async list() { + return (await this.repo.list()).map(publicUserView) } - getById(id) { - return this.repo.getById(id) + async getById(id) { + return await this.repo.getById(id) } - getByUsername(username) { - return this.repo.getByUsername(username) + async getByUsername(username) { + return await this.repo.getByUsername(username) } - countEnabledAdmins() { - return this.repo.countEnabledRole('admin') + async countEnabledAdmins() { + return await this.repo.countEnabledRole('admin') } - bootstrapFromEnv({ username, password } = {}) { + async bootstrapFromEnv({ username, password } = {}) { const name = normalizeUsername(username) if (!name || !password) return null - const existing = this.repo.getByUsername(name) + const existing = await this.repo.getByUsername(name) if (existing) return publicUserView(existing) // Import the already accepted env credential; new passwords still go through assertPassword. return publicUserView( - this.repo.insert({ + await this.repo.insert({ id: crypto.randomUUID(), username: assertUsername(name), password_hash: hashPassword(password), @@ -166,18 +166,18 @@ export class PanelUserStore { ) } - authenticate(username, password) { - const rec = this.repo.getByUsername(username) + async authenticate(username, password) { + const rec = await this.repo.getByUsername(username) if (!rec || rec.enabled === false) return null if (!verifyPassword(password, rec.password_hash)) return null const next = { ...rec, last_login_at: nowIso(), updated_at: rec.updated_at } - this.repo.update(next) + await this.repo.touchLogin(rec.id, next.last_login_at) return publicUserView(next) } - create({ username, password, role = 'user', enabled = true, vm_create_quota = 0 } = {}) { + async create({ username, password, role = 'user', enabled = true, vm_create_quota = 0 } = {}) { const name = assertUsername(username) - if (this.repo.getByUsername(name)) { + if (await this.repo.getByUsername(name)) { const err = new Error('用户名已存在') err.code = 'username_exists' throw err @@ -193,66 +193,78 @@ export class PanelUserStore { updated_at: nowIso(), last_login_at: null, } - return this.repo.insert(rec) + return await this.repo.insert(rec) } - update(id, patch = {}, { actorId = null } = {}) { - const rec = this.repo.getById(id) - if (!rec) return null - const next = { ...rec } + async update(id, patch = {}, { actorId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db + .prepare("SELECT id FROM users WHERE role = 'admin' AND deleted_at IS NULL ORDER BY id FOR UPDATE") + .all() - if (patch.role != null) { - const role = assertRole(patch.role) - if (rec.role === 'admin' && role !== 'admin' && this.countEnabledAdmins() <= 1) { - const err = new Error('不能取消最后一个管理员') - err.code = 'last_admin' - throw err - } - next.role = role - } + const rec = await this.repo.getById(id, { forUpdate: true }) + if (!rec) return null + const next = { ...rec } - if (patch.enabled != null) { - const enabled = !!patch.enabled - if (rec.role === 'admin' && rec.enabled && !enabled && this.countEnabledAdmins() <= 1) { - const err = new Error('不能停用最后一个管理员') - err.code = 'last_admin' - throw err + if (patch.role != null) { + const role = assertRole(patch.role) + if (rec.role === 'admin' && role !== 'admin' && (await this.countEnabledAdmins()) <= 1) { + const err = new Error('不能取消最后一个管理员') + err.code = 'last_admin' + throw err + } + next.role = role } - if (actorId && actorId === rec.id && !enabled) { - const err = new Error('不能停用当前登录账号') - err.code = 'self_disable' - throw err + + if (patch.enabled != null) { + const enabled = !!patch.enabled + if (rec.role === 'admin' && rec.enabled && !enabled && (await this.countEnabledAdmins()) <= 1) { + const err = new Error('不能停用最后一个管理员') + err.code = 'last_admin' + throw err + } + if (actorId && actorId === rec.id && !enabled) { + const err = new Error('不能停用当前登录账号') + err.code = 'self_disable' + throw err + } + next.enabled = enabled } - next.enabled = enabled - } - if (patch.password != null && String(patch.password) !== '') { - next.password_hash = hashPassword(assertPassword(patch.password)) - } + if (patch.password != null && String(patch.password) !== '') { + next.password_hash = hashPassword(assertPassword(patch.password)) + } - if (patch.vm_create_quota != null) { - next.vm_create_quota = clampVmCreateQuota(patch.vm_create_quota, rec.vm_create_quota || 0) - } + if (patch.vm_create_quota != null) { + next.vm_create_quota = clampVmCreateQuota(patch.vm_create_quota, rec.vm_create_quota || 0) + } - next.updated_at = nowIso() - return this.repo.update(next) + next.updated_at = nowIso() + return await this.repo.update(next) + }) } - remove(id, { actorId = null } = {}) { - const rec = this.repo.getById(id) - if (!rec) return false - if (actorId && actorId === rec.id) { - const err = new Error('不能删除当前登录账号') - err.code = 'self_delete' - throw err - } - if (rec.role === 'admin' && rec.enabled && this.countEnabledAdmins() <= 1) { - const err = new Error('不能删除最后一个管理员') - err.code = 'last_admin' - throw err - } - // Validate deletion first; a failed slot write must leave the user available for retry. - if (this.projectRoot) reclaimAssignedVms(this.projectRoot, new Set([rec.id])) - return this.repo.remove(id) + async remove(id, { actorId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db + .prepare("SELECT id FROM users WHERE role = 'admin' AND deleted_at IS NULL ORDER BY id FOR UPDATE") + .all() + + const rec = await this.repo.getById(id) + if (!rec) return false + if (actorId && actorId === rec.id) { + const err = new Error('不能删除当前登录账号') + err.code = 'self_delete' + throw err + } + if (rec.role === 'admin' && rec.enabled && (await this.countEnabledAdmins()) <= 1) { + const err = new Error('不能删除最后一个管理员') + err.code = 'last_admin' + throw err + } + // Validate deletion first; a failed slot write must leave the user available for retry. + if (this.projectRoot) reclaimAssignedVms(this.projectRoot, new Set([rec.id])) + return await this.repo.remove(id) + }) } } diff --git a/src/lib/db/repos/account-runtime-repo.mjs b/src/lib/db/repos/account-runtime-repo.mjs index 2c08371d2..41b43eee3 100644 --- a/src/lib/db/repos/account-runtime-repo.mjs +++ b/src/lib/db/repos/account-runtime-repo.mjs @@ -10,7 +10,7 @@ * with ms-epoch numbers; conversion happens here. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' function parse(value, fallback = null) { if (value == null) return fallback @@ -63,14 +63,14 @@ function mergeState(rt, acc) { export class AccountRuntimeRepo { constructor(db = getDb()) { this.db = db - this._getRt = db.prepare('SELECT * FROM account_runtime WHERE account_id = ?') + this._getRt = db.prepare('SELECT * FROM account_runtime WHERE account_id = $1') this._getAcc = db.prepare(` SELECT id, vm_id, priority, load_factor, last_used_at, temp_unschedulable_until, temp_unschedulable_reason, rate_limited_at, rate_limit_reset_at, overload_until, session_window_start, session_window_end, session_window_status, updated_at - FROM accounts WHERE id = ? + FROM accounts WHERE id = $1 `) this._listIds = db.prepare(` SELECT account_id FROM account_runtime @@ -78,28 +78,30 @@ export class AccountRuntimeRepo { ORDER BY 1 `) this._ensureAcc = db.prepare(` - INSERT OR IGNORE INTO accounts (id, vm_id, name, extra, created_at, updated_at) - VALUES (?, ?, ?, '{}', ?, ?) + INSERT INTO accounts (id, vm_id, name, extra, created_at, updated_at) + VALUES ($1, $2, $3, '{}', $4, $5) + ON CONFLICT(id) DO NOTHING `) this._linkDefaultGroup = db.prepare(` - INSERT OR IGNORE INTO account_groups (account_id, group_id, priority, created_at) - VALUES (?, 1, 50, ?) + INSERT INTO account_groups (account_id, group_id, priority, created_at) + VALUES ($1, 1, 50, $2) + ON CONFLICT(account_id, group_id) DO NOTHING `) this._updateAcc = db.prepare(` UPDATE accounts SET - vm_id = ?, - priority = ?, load_factor = ?, last_used_at = ?, - temp_unschedulable_until = ?, temp_unschedulable_reason = ?, - rate_limited_at = ?, rate_limit_reset_at = ?, overload_until = ?, - session_window_start = ?, session_window_end = ?, session_window_status = ?, - updated_at = ? - WHERE id = ? + vm_id = $1, + priority = $2, load_factor = $3, last_used_at = $4, + temp_unschedulable_until = $5, temp_unschedulable_reason = $6, + rate_limited_at = $7, rate_limit_reset_at = $8, overload_until = $9, + session_window_start = $10, session_window_end = $11, session_window_status = $12, + updated_at = $13 + WHERE id = $14 `) this._upsertRt = db.prepare(` INSERT INTO account_runtime ( account_id, vm_id, status, model_states_json, credential_generation, refresh_status, worker_heartbeat_at, worker_status_json, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT(account_id) DO UPDATE SET vm_id = excluded.vm_id, status = excluded.status, @@ -110,65 +112,67 @@ export class AccountRuntimeRepo { worker_status_json = excluded.worker_status_json, updated_at = excluded.updated_at `) - this._removeRt = db.prepare('DELETE FROM account_runtime WHERE account_id = ?') + this._removeRt = db.prepare('DELETE FROM account_runtime WHERE account_id = $1') } - get(accountId) { - const rt = this._getRt.get(accountId) - const acc = this._getAcc.get(accountId) + async get(accountId) { + const rt = await this._getRt.get(accountId) + const acc = await this._getAcc.get(accountId) if (!rt && !acc) return null return mergeState(rt, acc) } - list() { - return this._listIds - .all() - .map((r) => this.get(r.account_id ?? r.id)) - .filter(Boolean) + async list() { + const ids = await this._listIds.all() + return (await Promise.all(ids.map((r) => this.get(r.account_id ?? r.id)))).filter(Boolean) } - upsert(input = {}) { - if (!input.account_id || !input.vm_id) return null - const current = this.get(input.account_id) || {} - const next = { - ...current, - ...input, - status: input.status ?? current.status ?? 'unknown', - priority: Number(input.priority ?? current.priority ?? 0), - weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), - model_states: input.model_states ?? current.model_states ?? {}, - updated_at: new Date().toISOString(), - } - this._ensureAcc.run(next.account_id, next.vm_id, next.account_id, next.updated_at, next.updated_at) - this._linkDefaultGroup.run(next.account_id, next.updated_at) - this._updateAcc.run( - next.vm_id, - next.priority, - next.weight, - toIso(next.last_used_at), - toIso(next.cooldown_until), - next.cooldown_reason ?? null, - toIso(next.rate_limited_at), - toIso(next.rate_limit_reset_at), - toIso(next.overload_until), - toIso(next.session_window_start), - toIso(next.session_window_end), - next.session_window_status ?? null, - next.updated_at, - next.account_id, - ) - this._upsertRt.run( - next.account_id, - next.vm_id, - next.status, - JSON.stringify(next.model_states || {}), - next.credential_generation ?? 0, - next.refresh_status ?? null, - next.worker_heartbeat_at ?? null, - next.worker_status != null ? JSON.stringify(next.worker_status) : null, - next.updated_at, - ) - return this.get(next.account_id) + async upsert(input = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(input.account_id)) + + if (!input.account_id || !input.vm_id) return null + const current = (await this.get(input.account_id)) || {} + const next = { + ...current, + ...input, + status: input.status ?? current.status ?? 'unknown', + priority: Number(input.priority ?? current.priority ?? 0), + weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), + model_states: input.model_states ?? current.model_states ?? {}, + updated_at: new Date().toISOString(), + } + await this._ensureAcc.run(next.account_id, next.vm_id, next.account_id, next.updated_at, next.updated_at) + await this._linkDefaultGroup.run(next.account_id, next.updated_at) + await this._updateAcc.run( + next.vm_id, + next.priority, + next.weight, + toIso(next.last_used_at), + toIso(next.cooldown_until), + next.cooldown_reason ?? null, + toIso(next.rate_limited_at), + toIso(next.rate_limit_reset_at), + toIso(next.overload_until), + toIso(next.session_window_start), + toIso(next.session_window_end), + next.session_window_status ?? null, + next.updated_at, + next.account_id, + ) + await this._upsertRt.run( + next.account_id, + next.vm_id, + next.status, + JSON.stringify(next.model_states || {}), + next.credential_generation ?? 0, + next.refresh_status ?? null, + next.worker_heartbeat_at ?? null, + next.worker_status != null ? JSON.stringify(next.worker_status) : null, + next.updated_at, + ) + return await this.get(next.account_id) + }) } /** @@ -176,7 +180,7 @@ export class AccountRuntimeRepo { * (sub2api account.rate_limited_at / rate_limit_reset_at / overload_until / * session_window_* counterpart). Only provided fields are updated. */ - updateWindow( + async updateWindow( accountId, { vmId = null, @@ -188,50 +192,58 @@ export class AccountRuntimeRepo { sessionWindowStatus, } = {}, ) { - const current = this.get(accountId) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, - } - if (!current.vm_id && vmId) current.vm_id = vmId - if (!current.vm_id) return null - if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt - if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt - if (overloadUntil !== undefined) current.overload_until = overloadUntil - if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart - if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd - if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus - return this.upsert(current) + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = (await this.get(accountId)) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, + } + if (!current.vm_id && vmId) current.vm_id = vmId + if (!current.vm_id) return null + if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt + if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt + if (overloadUntil !== undefined) current.overload_until = overloadUntil + if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart + if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd + if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus + return await this.upsert(current) + }) } - markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { - const current = this.get(accountId) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, - } - if (model) { - current.model_states = { ...(current.model_states || {}) } - current.model_states[model] = { - status, - cooldown_until: Number(until) || null, - reason: reason || null, - updated_at: new Date().toISOString(), + async markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = (await this.get(accountId)) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, } - } else { - current.status = status - current.cooldown_until = Number(until) || null - current.cooldown_reason = reason || null - // Structured mirror of the protocol-level limit (sub2api account columns). - // overload_until is only the 529 hard column (RateLimitService.handle529). - // A 502 failover also uses reason provider_overloaded for 15s; writing it - // here paints every hopped VM as 过载冷却 and removes them from the pool. - if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { - current.rate_limited_at = Date.now() - current.rate_limit_reset_at = Number(until) || null + if (model) { + current.model_states = { ...(current.model_states || {}) } + current.model_states[model] = { + status, + cooldown_until: Number(until) || null, + reason: reason || null, + updated_at: new Date().toISOString(), + } + } else { + current.status = status + current.cooldown_until = Number(until) || null + current.cooldown_reason = reason || null + // Structured mirror of the protocol-level limit (sub2api account columns). + // overload_until is only the 529 hard column (RateLimitService.handle529). + // A 502 failover also uses reason provider_overloaded for 15s; writing it + // here paints every hopped VM as 过载冷却 and removes them from the pool. + if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { + current.rate_limited_at = Date.now() + current.rate_limit_reset_at = Number(until) || null + } } - } - return this.upsert(current) + return await this.upsert(current) + }) } /** Stale-access 401 / 403 permission — not quota, Fable, or 429. */ @@ -253,77 +265,94 @@ export class AccountRuntimeRepo { } /** Worker fresh / refresh success: drop only auth leftover so quota cools stay. */ - clearAuthCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current || !this.isAuthCooldown(current)) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, + async clearAuthCooldown(accountId, { vmId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = await this.get(accountId) + if (!current || !this.isAuthCooldown(current)) return false + return !!(await this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + })) }) } /** Live login / import: drop leftover oauth_revoked disable. Quota cools stay. */ - clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current || !this.isGrantRevokeCooldown(current)) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, + async clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = await this.get(accountId) + if (!current || !this.isGrantRevokeCooldown(current)) return false + return !!(await this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + })) }) } /** Operator / panel: drop account + per-model cooldown leftovers. */ - clearAccountCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - rate_limit_reset_at: null, - overload_until: null, - model_states: {}, + async clearAccountCooldown(accountId, { vmId = null } = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + + const current = await this.get(accountId) + if (!current) return false + return !!(await this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + rate_limit_reset_at: null, + overload_until: null, + model_states: {}, + })) }) } - clearExpired(now = Date.now()) { - for (const state of this.list()) { - if (!state.vm_id) continue - let changed = false - if (state.cooldown_until && state.cooldown_until <= now) { - state.status = 'ready' - state.cooldown_until = null - state.cooldown_reason = null - changed = true - } - if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { - state.rate_limit_reset_at = null - changed = true - } - if (state.overload_until && state.overload_until <= now) { - state.overload_until = null - changed = true - } - const models = { ...(state.model_states || {}) } - for (const [model, modelState] of Object.entries(models)) { - if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { - delete models[model] + async clearExpired(now = Date.now()) { + for (const id of await this._listIds.all()) { + await withTransaction(this.db, async () => { + const accountId = id.account_id ?? id.id + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + const state = await this.get(accountId) + if (!state?.vm_id) return + let changed = false + if (state.cooldown_until && state.cooldown_until <= now) { + state.status = 'ready' + state.cooldown_until = null + state.cooldown_reason = null changed = true } - } - if (changed) this.upsert({ ...state, model_states: models }) + if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { + state.rate_limit_reset_at = null + changed = true + } + if (state.overload_until && state.overload_until <= now) { + state.overload_until = null + changed = true + } + const models = { ...(state.model_states || {}) } + for (const [model, modelState] of Object.entries(models)) { + if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { + delete models[model] + changed = true + } + } + if (changed) await this.upsert({ ...state, model_states: models }) + }) } } - remove(accountId) { - return this._removeRt.run(accountId).changes > 0 + async remove(accountId) { + return (await this._removeRt.run(accountId)).changes > 0 } } diff --git a/src/lib/db/repos/accounts-repo.mjs b/src/lib/db/repos/accounts-repo.mjs index e33c9df10..7d74c7f02 100644 --- a/src/lib/db/repos/accounts-repo.mjs +++ b/src/lib/db/repos/accounts-repo.mjs @@ -8,7 +8,7 @@ * The unified/blocked/rate-limit snapshots live inside the `extra` JSON. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' const MAX_ALLOCATIONS_PER_ACCOUNT = 50 @@ -70,7 +70,7 @@ function buildExtra(acc) { export class AccountsRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM accounts WHERE id = ?') + this._get = db.prepare('SELECT * FROM accounts WHERE id = $1') this._list = db.prepare('SELECT * FROM accounts ORDER BY id') this._insert = db.prepare(` INSERT INTO accounts (id, vm_id, email, name, platform, type, status, @@ -79,77 +79,80 @@ export class AccountsRepo { requests, tokens_in, tokens_out, cache_read_tokens, cache_creation_tokens, extra, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) `) this._update = db.prepare(` - UPDATE accounts SET vm_id = ?, email = ?, name = ?, type = ?, - concurrency = ?, concurrency_override = ?, - max_rpm = ?, rpm_override = ?, - requests = ?, tokens_in = ?, tokens_out = ?, - cache_read_tokens = ?, cache_creation_tokens = ?, - extra = ?, updated_at = ? - WHERE id = ? + UPDATE accounts SET vm_id = $1, email = $2, name = $3, type = $4, + concurrency = $5, concurrency_override = $6, + max_rpm = $7, rpm_override = $8, + requests = $9, tokens_in = $10, tokens_out = $11, + cache_read_tokens = $12, cache_creation_tokens = $13, + extra = $14, updated_at = $15 + WHERE id = $16 `) this._insertAlloc = db.prepare(` INSERT INTO account_allocations (account_id, at, source, util_5h, util_7d, status_5h, status_7d, claim, tokens_in, tokens_out) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) `) this._trimAlloc = db.prepare(` - DELETE FROM account_allocations WHERE account_id = ? AND id NOT IN ( - SELECT id FROM account_allocations WHERE account_id = ? ORDER BY id DESC LIMIT ? + DELETE FROM account_allocations WHERE account_id = $1 AND id NOT IN ( + SELECT id FROM account_allocations WHERE account_id = $2 ORDER BY id DESC LIMIT $3 ) `) this._recentAlloc = db.prepare(` - SELECT * FROM account_allocations WHERE account_id = ? ORDER BY id DESC LIMIT ? + SELECT * FROM account_allocations WHERE account_id = $1 ORDER BY id DESC LIMIT $2 `) - this._removeAllocs = db.prepare('DELETE FROM account_allocations WHERE account_id = ?') - this._removeGroups = db.prepare('DELETE FROM account_groups WHERE account_id = ?') - this._remove = db.prepare('DELETE FROM accounts WHERE id = ?') + this._removeAllocs = db.prepare('DELETE FROM account_allocations WHERE account_id = $1') + this._removeGroups = db.prepare('DELETE FROM account_groups WHERE account_id = $1') + this._remove = db.prepare('DELETE FROM accounts WHERE id = $1') this._linkGroup = db.prepare(` - INSERT OR IGNORE INTO account_groups (account_id, group_id, priority, created_at) - VALUES (?, ?, ?, ?) + INSERT INTO account_groups (account_id, group_id, priority, created_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT(account_id, group_id) DO NOTHING `) } - get(accountId) { - return rowToAccount(this._get.get(accountId)) + async get(accountId) { + return rowToAccount(await this._get.get(accountId)) } - list() { - return this._list.all().map(rowToAccount) + async list() { + return (await this._list.all()).map(rowToAccount) } /** Insert if missing, returns stored account. New accounts join the coding group (id=1). */ - insert(acc) { - const now = new Date().toISOString() - this._insert.run( - acc.account_id, - acc.vm_id ?? null, - acc.email ?? null, - acc.name ?? acc.email ?? acc.account_id, - acc.platform ?? 'claude', - acc.type ?? 'oauth', - acc.status ?? 'active', - acc.max_concurrency ?? 2, - acc.concurrency_override ? 1 : 0, - acc.max_rpm ?? 0, - acc.rpm_override ? 1 : 0, - acc.requests ?? 0, - acc.tokens_in ?? 0, - acc.tokens_out ?? 0, - acc.cache_read_tokens ?? 0, - acc.cache_creation_tokens ?? 0, - buildExtra(acc), - acc.created_at ?? now, - acc.updated_at ?? now, - ) - this._linkGroup.run(acc.account_id, acc.group_id ?? 1, acc.group_priority ?? 50, now) - return this.get(acc.account_id) + async insert(acc) { + return withTransaction(this.db, async () => { + const now = new Date().toISOString() + await this._insert.run( + acc.account_id, + acc.vm_id ?? null, + acc.email ?? null, + acc.name ?? acc.email ?? acc.account_id, + acc.platform ?? 'claude', + acc.type ?? 'oauth', + acc.status ?? 'active', + acc.max_concurrency ?? 2, + acc.concurrency_override ? 1 : 0, + acc.max_rpm ?? 0, + acc.rpm_override ? 1 : 0, + acc.requests ?? 0, + acc.tokens_in ?? 0, + acc.tokens_out ?? 0, + acc.cache_read_tokens ?? 0, + acc.cache_creation_tokens ?? 0, + buildExtra(acc), + acc.created_at ?? now, + acc.updated_at ?? now, + ) + await this._linkGroup.run(acc.account_id, acc.group_id ?? 1, acc.group_priority ?? 50, now) + return await this.get(acc.account_id) + }) } /** Full-row save of a mutated account object. */ - save(acc) { - this._update.run( + async save(acc) { + await this._update.run( acc.vm_id ?? null, acc.email ?? null, acc.name ?? acc.email ?? acc.account_id, @@ -167,50 +170,51 @@ export class AccountsRepo { new Date().toISOString(), acc.account_id, ) - return this.get(acc.account_id) + return await this.get(acc.account_id) } - remove(accountId) { - this._removeAllocs.run(accountId) - this._removeGroups.run(accountId) - const info = this._remove.run(accountId) - return info.changes > 0 + async remove(accountId) { + return withTransaction(this.db, async () => { + await this._removeAllocs.run(accountId) + await this._removeGroups.run(accountId) + const info = await this._remove.run(accountId) + return info.changes > 0 + }) } - addAllocation(accountId, alloc, { max = MAX_ALLOCATIONS_PER_ACCOUNT } = {}) { - this._insertAlloc.run( - accountId, - alloc.at || new Date().toISOString(), - alloc.source ?? null, - alloc.util_5h ?? null, - alloc.util_7d ?? null, - alloc.status_5h ?? null, - alloc.status_7d ?? null, - alloc.claim ?? null, - alloc.tokens_in ?? null, - alloc.tokens_out ?? null, - ) - this._trimAlloc.run(accountId, accountId, max) + async addAllocation(accountId, alloc, { max = MAX_ALLOCATIONS_PER_ACCOUNT } = {}) { + return withTransaction(this.db, async () => { + await this._insertAlloc.run( + accountId, + alloc.at || new Date().toISOString(), + alloc.source ?? null, + alloc.util_5h ?? null, + alloc.util_7d ?? null, + alloc.status_5h ?? null, + alloc.status_7d ?? null, + alloc.claim ?? null, + alloc.tokens_in ?? null, + alloc.tokens_out ?? null, + ) + await this._trimAlloc.run(accountId, accountId, max) + }) } - recentAllocations(accountId, limit = 5) { - return this._recentAlloc - .all(accountId, limit) - .reverse() - .map((r) => ({ - at: r.at, - source: r.source ?? undefined, - util_5h: r.util_5h, - util_7d: r.util_7d, - status_5h: r.status_5h ?? undefined, - status_7d: r.status_7d ?? undefined, - claim: r.claim, - tokens_in: r.tokens_in, - tokens_out: r.tokens_out, - })) + async recentAllocations(accountId, limit = 5) { + return (await this._recentAlloc.all(accountId, limit)).reverse().map((r) => ({ + at: r.at, + source: r.source ?? undefined, + util_5h: r.util_5h, + util_7d: r.util_7d, + status_5h: r.status_5h ?? undefined, + status_7d: r.status_7d ?? undefined, + claim: r.claim, + tokens_in: r.tokens_in, + tokens_out: r.tokens_out, + })) } - allocationCount(accountId) { - return this.db.prepare('SELECT COUNT(*) c FROM account_allocations WHERE account_id = ?').get(accountId).c + async allocationCount(accountId) { + return (await this.db.prepare('SELECT COUNT(*) c FROM account_allocations WHERE account_id = $1').get(accountId)).c } } diff --git a/src/lib/db/repos/api-endpoints-repo.mjs b/src/lib/db/repos/api-endpoints-repo.mjs index 50c79322f..7deea21c9 100644 --- a/src/lib/db/repos/api-endpoints-repo.mjs +++ b/src/lib/db/repos/api-endpoints-repo.mjs @@ -82,43 +82,43 @@ export class ApiEndpointsRepo { constructor(db = getDb()) { this.db = db this._listEp = db.prepare('SELECT * FROM api_endpoints ORDER BY priority DESC, created_at, id') - this._getEp = db.prepare('SELECT * FROM api_endpoints WHERE id = ?') + this._getEp = db.prepare('SELECT * FROM api_endpoints WHERE id = $1') this._insEp = db.prepare( - `INSERT INTO api_endpoints (${EP_COLS.join(',')}) VALUES (${EP_COLS.map(() => '?').join(',')})`, + `INSERT INTO api_endpoints (${EP_COLS.join(',')}) VALUES (${EP_COLS.map((_, i) => '$' + (i + 1)).join(',')})`, ) - this._delEp = db.prepare('DELETE FROM api_endpoints WHERE id = ?') - this._listKeys = db.prepare('SELECT * FROM api_endpoint_keys WHERE endpoint_id = ? ORDER BY created_at, id') - this._getKey = db.prepare('SELECT * FROM api_endpoint_keys WHERE id = ?') + this._delEp = db.prepare('DELETE FROM api_endpoints WHERE id = $1') + this._listKeys = db.prepare('SELECT * FROM api_endpoint_keys WHERE endpoint_id = $1 ORDER BY created_at, id') + this._getKey = db.prepare('SELECT * FROM api_endpoint_keys WHERE id = $1') this._insKey = db.prepare( - `INSERT INTO api_endpoint_keys (${KEY_COLS.join(',')}) VALUES (${KEY_COLS.map(() => '?').join(',')})`, + `INSERT INTO api_endpoint_keys (${KEY_COLS.join(',')}) VALUES (${KEY_COLS.map((_, i) => '$' + (i + 1)).join(',')})`, ) - this._delKey = db.prepare('DELETE FROM api_endpoint_keys WHERE id = ?') - this._delKeysEp = db.prepare('DELETE FROM api_endpoint_keys WHERE endpoint_id = ?') - this._listModels = db.prepare('SELECT * FROM api_endpoint_models WHERE endpoint_id = ? ORDER BY sort, id') + this._delKey = db.prepare('DELETE FROM api_endpoint_keys WHERE id = $1') + this._delKeysEp = db.prepare('DELETE FROM api_endpoint_keys WHERE endpoint_id = $1') + this._listModels = db.prepare('SELECT * FROM api_endpoint_models WHERE endpoint_id = $1 ORDER BY sort, id') this._insModel = db.prepare( - `INSERT INTO api_endpoint_models (${MODEL_COLS.join(',')}) VALUES (${MODEL_COLS.map(() => '?').join(',')})`, + `INSERT INTO api_endpoint_models (${MODEL_COLS.join(',')}) VALUES (${MODEL_COLS.map((_, i) => '$' + (i + 1)).join(',')})`, ) - this._delModelsEp = db.prepare('DELETE FROM api_endpoint_models WHERE endpoint_id = ?') - this._cool = db.prepare('UPDATE api_endpoint_keys SET cooldown_until = ? WHERE id = ?') + this._delModelsEp = db.prepare('DELETE FROM api_endpoint_models WHERE endpoint_id = $1') + this._cool = db.prepare('UPDATE api_endpoint_keys SET cooldown_until = $1 WHERE id = $2') this._updKey = db.prepare( - 'UPDATE api_endpoint_keys SET api_key = ?, proxy_url = ?, proxy_id = ?, disabled = ?, cooldown_until = ? WHERE id = ?', + 'UPDATE api_endpoint_keys SET api_key = $1, proxy_url = $2, proxy_id = $3, disabled = $4, cooldown_until = $5 WHERE id = $6', ) } - listHydrated() { - return this._listEp.all().map((row) => this.hydrate(rowEp(row))) + async listHydrated() { + return Promise.all((await this._listEp.all()).map((row) => this.hydrate(rowEp(row)))) } - getHydrated(id) { - return this.hydrate(rowEp(this._getEp.get(id))) + async getHydrated(id) { + return await this.hydrate(rowEp(await this._getEp.get(id))) } - hydrate(ep) { + async hydrate(ep) { if (!ep) return null return { ...ep, - api_key_entries: this._listKeys.all(ep.id).map(rowKey), - models: this._listModels.all(ep.id).map(rowModel), + api_key_entries: (await this._listKeys.all(ep.id)).map(rowKey), + models: (await this._listModels.all(ep.id)).map(rowModel), } } @@ -139,31 +139,33 @@ export class ApiEndpointsRepo { } } - insertEndpoint(rec) { + async insertEndpoint(rec) { const vals = this._epValues(rec) - this._insEp.run(...EP_COLS.map((c) => vals[c] ?? null)) - return this.getHydrated(rec.id) + await this._insEp.run(...EP_COLS.map((c) => vals[c] ?? null)) + return await this.getHydrated(rec.id) } - updateEndpoint(rec) { + async updateEndpoint(rec) { const vals = this._epValues(rec) const cols = EP_COLS.filter((c) => c !== 'id') - this.db - .prepare(`UPDATE api_endpoints SET ${cols.map((c) => `${c} = ?`).join(', ')} WHERE id = ?`) + await this.db + .prepare( + `UPDATE api_endpoints SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')} WHERE id = $${cols.length + 1}`, + ) .run(...cols.map((c) => vals[c] ?? null), rec.id) - return this.getHydrated(rec.id) + return await this.getHydrated(rec.id) } - removeEndpoint(id) { - return withTransaction(this.db, () => { - this._delKeysEp.run(id) - this._delModelsEp.run(id) - return this._delEp.run(id).changes > 0 + async removeEndpoint(id) { + return await withTransaction(this.db, async () => { + await this._delKeysEp.run(id) + await this._delModelsEp.run(id) + return (await this._delEp.run(id)).changes > 0 }) } - insertKey(rec) { - this._insKey.run( + async insertKey(rec) { + await this._insKey.run( rec.id, rec.endpoint_id, rec.api_key, @@ -173,58 +175,68 @@ export class ApiEndpointsRepo { rec.cooldown_until || null, rec.created_at, ) - return rowKey(this._getKey.get(rec.id)) - } - - removeKey(id) { - return this._delKey.run(id).changes > 0 - } - - updateKey(id, patch = {}) { - const rec = rowKey(this._getKey.get(id)) - if (!rec) return null - if (patch.api_key != null) { - const next = String(patch.api_key).trim() - if (next.length >= 4) rec.api_key = next - } - if (patch.proxy_url != null) rec.proxy_url = String(patch.proxy_url).trim() - if (patch.proxy_id != null) rec.proxy_id = String(patch.proxy_id).trim() - if (patch.disabled != null) rec.disabled = !!patch.disabled - if (patch.cooldown_until !== undefined) rec.cooldown_until = patch.cooldown_until || null - this._updKey.run( - rec.api_key, - rec.proxy_url || '', - rec.proxy_id || '', - bool01(rec.disabled), - rec.cooldown_until, - rec.id, - ) + return rowKey(await this._getKey.get(rec.id)) + } + + async removeKey(id) { + return (await this._delKey.run(id)).changes > 0 + } + + async updateKey(id, patch = {}) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM api_endpoint_keys WHERE id = $1 FOR UPDATE').get(id) + + const rec = rowKey(await this._getKey.get(id)) + if (!rec) return null + if (patch.api_key != null) { + const next = String(patch.api_key).trim() + if (next.length >= 4) rec.api_key = next + } + if (patch.proxy_url != null) rec.proxy_url = String(patch.proxy_url).trim() + if (patch.proxy_id != null) rec.proxy_id = String(patch.proxy_id).trim() + if (patch.disabled != null) rec.disabled = !!patch.disabled + if (patch.cooldown_until !== undefined) rec.cooldown_until = patch.cooldown_until || null + await this._updKey.run( + rec.api_key, + rec.proxy_url || '', + rec.proxy_id || '', + bool01(rec.disabled), + rec.cooldown_until, + rec.id, + ) + }) } - setKeyCooldown(id, untilIso) { - this._cool.run(untilIso, id) - } - - replaceModels(endpointId, models = []) { - withTransaction(this.db, () => { - this._delModelsEp.run(endpointId) - models.forEach((m, i) => { - const levels = Array.isArray(m.thinking?.levels) - ? m.thinking.levels - : Array.isArray(m.thinking_levels) - ? m.thinking_levels - : null - this._insModel.run( - m.id, - endpointId, - Number(m.sort ?? i), - m.name || m.upstream_name, - m.alias || m.name, - bool01(m.image), - levels ? JSON.stringify(levels) : null, + async setKeyCooldown(id, untilIso) { + await this._cool.run(untilIso, id) + } + + async replaceModels(endpointId, models = []) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM api_endpoints WHERE id = $1 FOR UPDATE').get(endpointId) + + await withTransaction(this.db, async () => { + await this._delModelsEp.run(endpointId) + await Promise.all( + models.map(async (m, i) => { + const levels = Array.isArray(m.thinking?.levels) + ? m.thinking.levels + : Array.isArray(m.thinking_levels) + ? m.thinking_levels + : null + await this._insModel.run( + m.id, + endpointId, + Number(m.sort ?? i), + m.name || m.upstream_name, + m.alias || m.name, + bool01(m.image), + levels ? JSON.stringify(levels) : null, + ) + }), ) }) + return await this.getHydrated(endpointId) }) - return this.getHydrated(endpointId) } } diff --git a/src/lib/db/repos/api-keys-repo.mjs b/src/lib/db/repos/api-keys-repo.mjs index 7ef6750a1..d3aafd340 100644 --- a/src/lib/db/repos/api-keys-repo.mjs +++ b/src/lib/db/repos/api-keys-repo.mjs @@ -7,7 +7,7 @@ * is quota_requests / quota_requests_used. Soft delete via deleted_at. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' const COLUMNS = [ 'id', @@ -95,63 +95,69 @@ export class ApiKeysRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare('SELECT * FROM api_keys WHERE deleted_at IS NULL ORDER BY created_at') - this._get = db.prepare('SELECT * FROM api_keys WHERE id = ? AND deleted_at IS NULL') - this._getByKey = db.prepare('SELECT * FROM api_keys WHERE key = ? AND deleted_at IS NULL') - this._getByHash = db.prepare('SELECT * FROM api_keys WHERE key_hash = ? AND deleted_at IS NULL') + this._get = db.prepare('SELECT * FROM api_keys WHERE id = $1 AND deleted_at IS NULL') + this._getByKey = db.prepare('SELECT * FROM api_keys WHERE key = $1 AND deleted_at IS NULL') + this._getByHash = db.prepare('SELECT * FROM api_keys WHERE key_hash = $1 AND deleted_at IS NULL') this._insert = db.prepare(` INSERT INTO api_keys (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) - this._delete = db.prepare('DELETE FROM api_keys WHERE id = ?') + this._delete = db.prepare('DELETE FROM api_keys WHERE id = $1') this._recordUsage = db.prepare(` UPDATE api_keys SET requests = requests + 1, quota_requests_used = quota_requests_used + 1, - tokens_in = tokens_in + ?, - tokens_out = tokens_out + ?, - cache_read_tokens = COALESCE(cache_read_tokens, 0) + ?, - cache_creation_tokens = COALESCE(cache_creation_tokens, 0) + ?, - quota_used = quota_used + ?, - usage_5h = ?, window_5h_start = ?, - usage_1d = ?, window_1d_start = ?, - usage_7d = ?, window_7d_start = ?, - last_used_at = ?, - updated_at = ? - WHERE id = ? + tokens_in = tokens_in + $1, + tokens_out = tokens_out + $2, + cache_read_tokens = COALESCE(cache_read_tokens, 0) + $3, + cache_creation_tokens = COALESCE(cache_creation_tokens, 0) + $4, + quota_used = quota_used + $5, + status = CASE WHEN status = 'active' AND quota > 0 AND quota_used + $5 >= quota THEN 'quota_exhausted' ELSE status END, + usage_5h = $6, window_5h_start = $7, + usage_1d = $8, window_1d_start = $9, + usage_7d = $10, window_7d_start = $11, + last_used_at = $12, + updated_at = $13 + WHERE id = $14 `) } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getById(id) { - return rowToRec(this._get.get(id)) + async getById(id, { forUpdate = false } = {}) { + return rowToRec( + await (forUpdate + ? this.db.prepare('SELECT * FROM api_keys WHERE id = $1 AND deleted_at IS NULL FOR UPDATE') + : this._get + ).get(id), + ) } - getByKey(key) { - return rowToRec(this._getByKey.get(key)) + async getByKey(key) { + return rowToRec(await this._getByKey.get(key)) } - getByHash(hash) { - return rowToRec(this._getByHash.get(hash)) + async getByHash(hash) { + return rowToRec(await this._getByHash.get(hash)) } - insert(rec) { - this._insert.run(...COLUMNS.map((c) => toValue(rec, c))) - return this.getById(rec.id) + async insert(rec) { + await this._insert.run(...COLUMNS.map((c) => toValue(rec, c))) + return await this.getById(rec.id) } /** Full-row update from a record object (id immutable). */ - update(rec) { + async update(rec) { const cols = COLUMNS.filter((c) => c !== 'id') - const sql = `UPDATE api_keys SET ${cols.map((c) => `${c} = ?`).join(', ')} WHERE id = ?` - this.db.prepare(sql).run(...cols.map((c) => toValue(rec, c)), rec.id) - return this.getById(rec.id) + const sql = `UPDATE api_keys SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')} WHERE id = $${cols.length + 1}` + await this.db.prepare(sql).run(...cols.map((c) => toValue(rec, c)), rec.id) + return await this.getById(rec.id) } - remove(id) { - const info = this._delete.run(id) + async remove(id) { + const info = await this._delete.run(id) return info.changes > 0 } @@ -167,33 +173,38 @@ export class ApiKeysRepo { return { usage: (Number(rec[key.usage]) || 0) + cost, start: rec[key.start] } } - recordUsage(id, { tokens_in = 0, tokens_out = 0, cache_read_tokens = 0, cache_creation_tokens = 0, cost = 0 } = {}) { - const rec = this.getById(id) - if (!rec) return null - const nowMs = Date.now() - const now = new Date(nowMs).toISOString() - const usd = Number(cost) || 0 - const w = WINDOWS.map((win) => this._rollWindow(rec, win, usd, nowMs)) - this._recordUsage.run( - Number(tokens_in) || 0, - Number(tokens_out) || 0, - Number(cache_read_tokens) || 0, - Number(cache_creation_tokens) || 0, - usd, - w[0].usage, - w[0].start, - w[1].usage, - w[1].start, - w[2].usage, - w[2].start, - now, - now, - id, - ) - return this.getById(id) + async recordUsage( + id, + { tokens_in = 0, tokens_out = 0, cache_read_tokens = 0, cache_creation_tokens = 0, cost = 0 } = {}, + ) { + return withTransaction(this.db, async () => { + const rec = await this.getById(id, { forUpdate: true }) + if (!rec) return null + const nowMs = Date.now() + const now = new Date(nowMs).toISOString() + const usd = Number(cost) || 0 + const w = WINDOWS.map((win) => this._rollWindow(rec, win, usd, nowMs)) + await this._recordUsage.run( + Number(tokens_in) || 0, + Number(tokens_out) || 0, + Number(cache_read_tokens) || 0, + Number(cache_creation_tokens) || 0, + usd, + w[0].usage, + w[0].start, + w[1].usage, + w[1].start, + w[2].usage, + w[2].start, + now, + now, + id, + ) + return await this.getById(id) + }) } - count() { - return this.db.prepare('SELECT COUNT(*) c FROM api_keys WHERE deleted_at IS NULL').get().c + async count() { + return (await this.db.prepare('SELECT COUNT(*) c FROM api_keys WHERE deleted_at IS NULL').get()).c } } diff --git a/src/lib/db/repos/backup-repo.mjs b/src/lib/db/repos/backup-repo.mjs index df476a726..6b2be9502 100644 --- a/src/lib/db/repos/backup-repo.mjs +++ b/src/lib/db/repos/backup-repo.mjs @@ -34,49 +34,49 @@ export class BackupRepo { this.db = db this._insert = db.prepare(` INSERT INTO backup_records (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) - this._get = db.prepare('SELECT * FROM backup_records WHERE id = ?') - this._list = db.prepare('SELECT * FROM backup_records ORDER BY created_at DESC LIMIT ?') - this._remove = db.prepare('DELETE FROM backup_records WHERE id = ?') + this._get = db.prepare('SELECT * FROM backup_records WHERE id = $1') + this._list = db.prepare('SELECT * FROM backup_records ORDER BY created_at DESC LIMIT $1') + this._remove = db.prepare('DELETE FROM backup_records WHERE id = $1') } - insert(rec) { - this._insert.run( + async insert(rec) { + await this._insert.run( ...COLUMNS.map((c) => { if (c === 'includes_json') return rec.includes ? JSON.stringify(rec.includes) : (rec.includes_json ?? null) return rec[c] ?? null }), ) - return this.get(rec.id) + return await this.get(rec.id) } - get(id) { - return fromRow(this._get.get(id)) + async get(id) { + return fromRow(await this._get.get(id)) } - list({ limit = 100 } = {}) { - return this._list.all(Math.max(1, Math.min(500, Number(limit) || 100))).map(fromRow) + async list({ limit = 100 } = {}) { + return (await this._list.all(Math.max(1, Math.min(500, Number(limit) || 100)))).map(fromRow) } - remove(id) { - return this._remove.run(id).changes > 0 + async remove(id) { + return (await this._remove.run(id)).changes > 0 } - lastSuccessful({ kinds = null } = {}) { - const rows = this.list({ limit: 200 }) + async lastSuccessful({ kinds = null } = {}) { + const rows = await this.list({ limit: 200 }) return rows.find((r) => r.status === 'ok' && (!kinds || kinds.includes(r.kind))) || null } /** Successful non-pre_restore backups beyond `keep` newest. */ - beyondRetention(keep) { - const rows = this.list({ limit: 500 }).filter((r) => r.status === 'ok' && r.kind !== 'pre_restore') + async beyondRetention(keep) { + const rows = (await this.list({ limit: 500 })).filter((r) => r.status === 'ok' && r.kind !== 'pre_restore') return rows.slice(Math.max(0, keep)) } /** pre_restore backups beyond `keep` newest. */ - preRestoreBeyond(keep = 3) { - const rows = this.list({ limit: 500 }).filter((r) => r.status === 'ok' && r.kind === 'pre_restore') + async preRestoreBeyond(keep = 3) { + const rows = (await this.list({ limit: 500 })).filter((r) => r.status === 'ok' && r.kind === 'pre_restore') return rows.slice(Math.max(0, keep)) } } diff --git a/src/lib/db/repos/cluster-nodes-repo.mjs b/src/lib/db/repos/cluster-nodes-repo.mjs index 5a792166b..13966d6f4 100644 --- a/src/lib/db/repos/cluster-nodes-repo.mjs +++ b/src/lib/db/repos/cluster-nodes-repo.mjs @@ -31,49 +31,49 @@ export class ClusterNodesRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare(`SELECT ${PUBLIC_COLUMNS.join(', ')} FROM cluster_nodes ORDER BY created_at, id`) - this._get = db.prepare(`SELECT ${PUBLIC_COLUMNS.join(', ')} FROM cluster_nodes WHERE id = ?`) - this._secrets = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?') - this._findEndpoint = db.prepare('SELECT id FROM cluster_nodes WHERE host = ? AND port = ? AND username = ?') - this._dependents = db.prepare('SELECT id FROM cluster_nodes WHERE jump_node_id = ?') + this._get = db.prepare(`SELECT ${PUBLIC_COLUMNS.join(', ')} FROM cluster_nodes WHERE id = $1`) + this._secrets = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + this._findEndpoint = db.prepare('SELECT id FROM cluster_nodes WHERE host = $1 AND port = $2 AND username = $3') + this._dependents = db.prepare('SELECT id FROM cluster_nodes WHERE jump_node_id = $1') this._insert = db.prepare(` INSERT INTO cluster_nodes (id, label, host, port, username, auth_type, password, private_key, passphrase, host_key_alg, host_key_sha256, jump_node_id, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) `) this._update = db.prepare(` - UPDATE cluster_nodes SET label = ?, host = ?, port = ?, username = ?, auth_type = ?, - password = ?, private_key = ?, passphrase = ?, updated_at = ? - WHERE id = ? AND updated_at = ? + UPDATE cluster_nodes SET label = $1, host = $2, port = $3, username = $4, auth_type = $5, + password = $6, private_key = $7, passphrase = $8, updated_at = $9 + WHERE id = $10 AND updated_at = $11 RETURNING ${PUBLIC_COLUMNS.join(', ')} `) this._updateLabel = db.prepare(` - UPDATE cluster_nodes SET label = ?, updated_at = ? - WHERE id = ? AND updated_at = ? + UPDATE cluster_nodes SET label = $1, updated_at = $2 + WHERE id = $3 AND updated_at = $4 RETURNING ${PUBLIC_COLUMNS.join(', ')} `) - this._remove = db.prepare('DELETE FROM cluster_nodes WHERE id = ?') + this._remove = db.prepare('DELETE FROM cluster_nodes WHERE id = $1') this._stmts = new Map() } - list() { - return this._list.all().map((row) => ({ ...row })) + async list() { + return (await this._list.all()).map((row) => ({ ...row })) } - get(id) { - const row = this._get.get(id) + async get(id) { + const row = await this._get.get(id) return row ? { ...row } : null } - findByEndpoint(host, port, username) { - return this._findEndpoint.get(host, port, username)?.id || null + async findByEndpoint(host, port, username) { + return (await this._findEndpoint.get(host, port, username))?.id || null } - dependentsOf(id) { - return this._dependents.all(id).map((row) => row.id) + async dependentsOf(id) { + return (await this._dependents.all(id)).map((row) => row.id) } - getSecrets(id) { - const row = this._secrets.get(id) + async getSecrets(id) { + const row = await this._secrets.get(id) if (!row) return null return { password: maybeDecrypt(row.password), @@ -82,8 +82,8 @@ export class ClusterNodesRepo { } } - insert(node, now = new Date().toISOString()) { - this._insert.run( + async insert(node, now = new Date().toISOString()) { + await this._insert.run( node.id, node.label, node.host, @@ -99,14 +99,14 @@ export class ClusterNodesRepo { now, now, ) - return this.get(node.id) + return await this.get(node.id) } /** Replace normalized connection fields only if the caller's version is still current. */ - update(id, node, expectedUpdatedAt) { + async update(id, node, expectedUpdatedAt) { const now = nextUpdatedAt(expectedUpdatedAt) if (!now) return null - const row = this._update.get( + const row = await this._update.get( node.label, node.host, node.port, @@ -123,15 +123,15 @@ export class ClusterNodesRepo { } /** Rename without reading, decrypting or rewriting stored credentials. */ - updateLabel(id, label, expectedUpdatedAt) { + async updateLabel(id, label, expectedUpdatedAt) { const now = nextUpdatedAt(expectedUpdatedAt) if (!now) return null - const row = this._updateLabel.get(label, now, id, expectedUpdatedAt) + const row = await this._updateLabel.get(label, now, id, expectedUpdatedAt) return row ? { ...row } : null } - remove(id) { - return this._remove.run(id).changes > 0 + async remove(id) { + return (await this._remove.run(id)).changes > 0 } // Prepared on first use: callers that only load 026 (older fixtures) never touch this table. @@ -145,44 +145,44 @@ export class ClusterNodesRepo { } /** 'linux' or 'darwin'; null when the node is gone. Detected over SSH, never taken from the form. */ - platform(id) { - return this._stmt('SELECT platform FROM cluster_nodes WHERE id = ?').get(id)?.platform ?? null + async platform(id) { + return (await this._stmt('SELECT platform FROM cluster_nodes WHERE id = $1').get(id))?.platform ?? null } /** Not a connection edit: leaves updated_at alone so a pending node edit does not conflict. */ - setPlatform(id, platform) { - return this._stmt('UPDATE cluster_nodes SET platform = ? WHERE id = ?').run(platform, id).changes > 0 + async setPlatform(id, platform) { + return (await this._stmt('UPDATE cluster_nodes SET platform = $1 WHERE id = $2').run(platform, id)).changes > 0 } /** node_id → { address, port }; the gate token stays out of listings. */ - directLinks() { - const rows = this._stmt('SELECT node_id, address, port FROM cluster_node_direct_links').all() + async directLinks() { + const rows = await this._stmt('SELECT node_id, address, port FROM cluster_node_direct_links').all() return new Map(rows.map((row) => [row.node_id, { address: row.address, port: row.port }])) } - getDirectLink(id) { - const row = this._stmt( - 'SELECT address, port, token, cert_sha256 FROM cluster_node_direct_links WHERE node_id = ?', + async getDirectLink(id) { + const row = await this._stmt( + 'SELECT address, port, token, cert_sha256 FROM cluster_node_direct_links WHERE node_id = $1', ).get(id) return row ? { address: row.address, port: row.port, token: maybeDecrypt(row.token), certSha256: row.cert_sha256 || null } : null } - setDirectLink(id, { address, port, token }, now = new Date().toISOString()) { - this._stmt(` - INSERT INTO cluster_node_direct_links (node_id, address, port, token, updated_at) VALUES (?, ?, ?, ?, ?) + async setDirectLink(id, { address, port, token }, now = new Date().toISOString()) { + await this._stmt(` + INSERT INTO cluster_node_direct_links (node_id, address, port, token, updated_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT(node_id) DO UPDATE SET address = excluded.address, port = excluded.port, token = excluded.token, updated_at = excluded.updated_at `).run(id, address, port, maybeEncrypt(token), now) } /** The pin follows the node's gate, not the address: an address edit keeps it. */ - setDirectLinkCert(id, certSha256) { - this._stmt('UPDATE cluster_node_direct_links SET cert_sha256 = ? WHERE node_id = ?').run(certSha256, id) + async setDirectLinkCert(id, certSha256) { + await this._stmt('UPDATE cluster_node_direct_links SET cert_sha256 = $1 WHERE node_id = $2').run(certSha256, id) } - clearDirectLink(id) { - return this._stmt('DELETE FROM cluster_node_direct_links WHERE node_id = ?').run(id).changes > 0 + async clearDirectLink(id) { + return (await this._stmt('DELETE FROM cluster_node_direct_links WHERE node_id = $1').run(id)).changes > 0 } } diff --git a/src/lib/db/repos/groups-repo.mjs b/src/lib/db/repos/groups-repo.mjs index 4fc65f4c3..08d1e1f25 100644 --- a/src/lib/db/repos/groups-repo.mjs +++ b/src/lib/db/repos/groups-repo.mjs @@ -32,33 +32,33 @@ export class GroupsRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare('SELECT * FROM groups WHERE deleted_at IS NULL ORDER BY sort_order, id') - this._get = db.prepare('SELECT * FROM groups WHERE id = ? AND deleted_at IS NULL') - this._getByName = db.prepare('SELECT * FROM groups WHERE name = ? AND deleted_at IS NULL') + this._get = db.prepare('SELECT * FROM groups WHERE id = $1 AND deleted_at IS NULL') + this._getByName = db.prepare('SELECT * FROM groups WHERE name = $1 AND deleted_at IS NULL') } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getById(id) { - return rowToRec(this._get.get(id)) + async getById(id) { + return rowToRec(await this._get.get(id)) } - getByName(name) { - return rowToRec(this._getByName.get(String(name || '').trim())) + async getByName(name) { + return rowToRec(await this._getByName.get(String(name || '').trim())) } /** The coding group — where keys/accounts land unless told otherwise. */ - getDefault() { - return this.getById(DEFAULT_GROUP_ID) + async getDefault() { + return await this.getById(DEFAULT_GROUP_ID) } /** Patch editable attributes only (name/platform/id are fixed). */ - update(id, patch = {}) { + async update(id, patch = {}) { const cols = EDITABLE.filter((c) => patch[c] !== undefined) - if (!cols.length) return this.getById(id) - const sql = `UPDATE groups SET ${cols.map((c) => `${c} = ?`).join(', ')}, updated_at = ? WHERE id = ? AND deleted_at IS NULL` - this.db.prepare(sql).run( + if (!cols.length) return await this.getById(id) + const sql = `UPDATE groups SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')}, updated_at = $${cols.length + 1} WHERE id = $${cols.length + 2} AND deleted_at IS NULL` + await this.db.prepare(sql).run( ...cols.map((c) => { if (c === 'is_exclusive') return patch[c] ? 1 : 0 return patch[c] @@ -66,12 +66,12 @@ export class GroupsRepo { new Date().toISOString(), id, ) - return this.getById(id) + return await this.getById(id) } /** Effective billing multiplier for a group id (missing group → 1). */ - rateMultiplier(id) { - const g = this.getById(id ?? DEFAULT_GROUP_ID) + async rateMultiplier(id) { + const g = await this.getById(id ?? DEFAULT_GROUP_ID) return g ? g.rate_multiplier : 1 } } diff --git a/src/lib/db/repos/proxies-repo.mjs b/src/lib/db/repos/proxies-repo.mjs index ea6fdfaaf..1af1bb305 100644 --- a/src/lib/db/repos/proxies-repo.mjs +++ b/src/lib/db/repos/proxies-repo.mjs @@ -64,19 +64,19 @@ export class ProxiesRepo { this._clear = db.prepare('DELETE FROM proxies') this._insert = db.prepare(` INSERT INTO proxies (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) } - loadAll() { - return this._all.all().map(rowToProxy) + async loadAll() { + return (await this._all.all()).map(rowToProxy) } - replaceAll(proxies = []) { - withTransaction(this.db, () => { - this._clear.run() + async replaceAll(proxies = []) { + await withTransaction(this.db, async () => { + await this._clear.run() for (const p of proxies) { - this._insert.run( + await this._insert.run( ...COLUMNS.map((c) => { if (c === 'enabled') return p.enabled === false ? 0 : 1 if (c === 'domain_forward') return p.domain_forward ? 1 : 0 @@ -89,11 +89,11 @@ export class ProxiesRepo { }) } - getConfig(fallback = null) { - return this.settings.get(CONFIG_KEY, fallback) + async getConfig(fallback = null) { + return await this.settings.get(CONFIG_KEY, fallback) } - setConfig(config) { - return this.settings.set(CONFIG_KEY, config) + async setConfig(config) { + return await this.settings.set(CONFIG_KEY, config) } } diff --git a/src/lib/db/repos/redeem-codes-repo.mjs b/src/lib/db/repos/redeem-codes-repo.mjs index 2daa6890c..17e7a8d76 100644 --- a/src/lib/db/repos/redeem-codes-repo.mjs +++ b/src/lib/db/repos/redeem-codes-repo.mjs @@ -33,58 +33,57 @@ export class RedeemCodesRepo { this.db = db this.users = new UsersRepo(db) this._list = db.prepare('SELECT * FROM redeem_codes ORDER BY id DESC') - this._get = db.prepare('SELECT * FROM redeem_codes WHERE id = ?') - this._getByCode = db.prepare('SELECT * FROM redeem_codes WHERE code = ?') + this._get = db.prepare('SELECT * FROM redeem_codes WHERE id = $1') + this._getByCode = db.prepare('SELECT * FROM redeem_codes WHERE code = $1') this._insert = db.prepare(` INSERT INTO redeem_codes (code, type, value, status, notes, expires_at, created_at) - VALUES (?, 'balance', ?, 'unused', ?, ?, ?) + VALUES ($1, 'balance', $2, 'unused', $3, $4, $5) RETURNING * `) this._markUsed = db.prepare(` - UPDATE redeem_codes SET status = 'used', used_by = ?, used_at = ? - WHERE id = ? AND status = 'unused' + UPDATE redeem_codes SET status = 'used', used_by = $1, used_at = $2 + WHERE id = $3 AND status = 'unused' `) - this._delete = db.prepare("DELETE FROM redeem_codes WHERE id = ? AND status = 'unused'") + this._delete = db.prepare("DELETE FROM redeem_codes WHERE id = $1 AND status = 'unused'") } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getByCode(code) { - return rowToRec(this._getByCode.get(String(code || '').trim())) + async getByCode(code) { + return rowToRec(await this._getByCode.get(String(code || '').trim())) } - create({ value, notes = null, expires_at = null, code = null } = {}) { + async create({ value, notes = null, expires_at = null, code = null } = {}) { const v = Number(value) if (!Number.isFinite(v) || v <= 0) return null const c = code || generateRedeemCode() - const info = this._insert.run(c, v, notes, expires_at, new Date().toISOString()) - return rowToRec(this._get.get(info.lastInsertRowid)) + return rowToRec(await this._insert.get(c, v, notes, expires_at, new Date().toISOString())) } /** Unused codes only; used codes stay as the audit ledger. */ - remove(id) { - return this._delete.run(id).changes > 0 + async remove(id) { + return (await this._delete.run(id)).changes > 0 } /** * Redeem a code for a user: optimistic status flip + balance credit in one * transaction. Returns {ok, code?, error?}. */ - redeem(code, userId) { - const rec = this.getByCode(code) + async redeem(code, userId) { + const rec = await this.getByCode(code) if (!rec) return { ok: false, error: 'code_not_found' } if (rec.status !== 'unused') return { ok: false, error: 'code_used' } if (rec.expires_at && Date.parse(rec.expires_at) < Date.now()) { return { ok: false, error: 'code_expired' } } - const user = this.users.getById(userId) - if (!user) return { ok: false, error: 'user_not_found' } - return withTransaction(this.db, () => { - const info = this._markUsed.run(userId, new Date().toISOString(), rec.id) + return await withTransaction(this.db, async () => { + const user = await this.users.getById(userId, { forUpdate: true }) + if (!user) return { ok: false, error: 'user_not_found' } + const info = await this._markUsed.run(userId, new Date().toISOString(), rec.id) if (!info.changes) return { ok: false, error: 'code_used' } - this.users.addBalance(userId, rec.value) - return { ok: true, code: rowToRec(this._get.get(rec.id)) } + await this.users.addBalance(userId, rec.value) + return { ok: true, code: rowToRec(await this._get.get(rec.id)) } }) } } diff --git a/src/lib/db/repos/refusal-device-blocks-repo.mjs b/src/lib/db/repos/refusal-device-blocks-repo.mjs index 731abe728..305395b97 100644 --- a/src/lib/db/repos/refusal-device-blocks-repo.mjs +++ b/src/lib/db/repos/refusal-device-blocks-repo.mjs @@ -20,12 +20,12 @@ function rowToRec(row) { export class RefusalDeviceBlocksRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM refusal_device_blocks WHERE device_id = ?') + this._get = db.prepare('SELECT * FROM refusal_device_blocks WHERE device_id = $1') this._insert = db.prepare(` INSERT INTO refusal_device_blocks ( device_id, first_seen_at, last_seen_at, hit_count, source_request_id, fingerprint, reason - ) VALUES (?, ?, ?, 0, ?, ?, ?) + ) VALUES ($1, $2, $3, 0, $4, $5, $6) ON CONFLICT(device_id) DO UPDATE SET last_seen_at = excluded.last_seen_at, source_request_id = COALESCE(refusal_device_blocks.source_request_id, excluded.source_request_id), @@ -33,52 +33,52 @@ export class RefusalDeviceBlocksRepo { reason = COALESCE(refusal_device_blocks.reason, excluded.reason) `) this._hit = db.prepare(` - UPDATE refusal_device_blocks SET hit_count = hit_count + 1, last_seen_at = ? - WHERE device_id = ? + UPDATE refusal_device_blocks SET hit_count = hit_count + 1, last_seen_at = $1 + WHERE device_id = $2 `) - this._list = db.prepare('SELECT * FROM refusal_device_blocks ORDER BY last_seen_at DESC, device_id LIMIT ?') + this._list = db.prepare('SELECT * FROM refusal_device_blocks ORDER BY last_seen_at DESC, device_id LIMIT $1') this._count = db.prepare('SELECT COUNT(*) AS n FROM refusal_device_blocks') - this._remove = db.prepare('DELETE FROM refusal_device_blocks WHERE device_id = ?') + this._remove = db.prepare('DELETE FROM refusal_device_blocks WHERE device_id = $1') this._clear = db.prepare('DELETE FROM refusal_device_blocks') } - get(deviceId) { + async get(deviceId) { const id = String(deviceId || '').trim() if (!id) return null - return rowToRec(this._get.get(id)) + return rowToRec(await this._get.get(id)) } - block({ deviceId, requestId = null, fingerprint = null, reason = 'refusal_guard' } = {}) { + async block({ deviceId, requestId = null, fingerprint = null, reason = 'refusal_guard' } = {}) { const id = String(deviceId || '').trim() if (!id) return null const now = new Date().toISOString() - this._insert.run(id, now, now, requestId, fingerprint, reason ? String(reason).slice(0, 80) : null) - return this.get(id) + await this._insert.run(id, now, now, requestId, fingerprint, reason ? String(reason).slice(0, 80) : null) + return await this.get(id) } - hit(deviceId) { + async hit(deviceId) { const id = String(deviceId || '').trim() if (!id) return null - this._hit.run(new Date().toISOString(), id) - return this.get(id) + await this._hit.run(new Date().toISOString(), id) + return await this.get(id) } - list(limit = 200) { + async list(limit = 200) { const n = Math.min(500, Math.max(1, Number(limit) || 200)) - return this._list.all(n).map(rowToRec) + return (await this._list.all(n)).map(rowToRec) } - count() { - return Number(this._count.get()?.n) || 0 + async count() { + return Number((await this._count.get())?.n) || 0 } - remove(deviceId) { + async remove(deviceId) { const id = String(deviceId || '').trim() if (!id) return false - return this._remove.run(id).changes > 0 + return (await this._remove.run(id)).changes > 0 } - clear() { - return this._clear.run().changes + async clear() { + return (await this._clear.run()).changes } } diff --git a/src/lib/db/repos/refusal-guards-repo.mjs b/src/lib/db/repos/refusal-guards-repo.mjs index 1b617ad78..4706af08b 100644 --- a/src/lib/db/repos/refusal-guards-repo.mjs +++ b/src/lib/db/repos/refusal-guards-repo.mjs @@ -23,12 +23,12 @@ function rowToRec(row) { export class RefusalGuardsRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM refusal_guards WHERE fingerprint = ?') + this._get = db.prepare('SELECT * FROM refusal_guards WHERE fingerprint = $1') this._insert = db.prepare(` INSERT INTO refusal_guards ( fingerprint, model, first_seen_at, last_seen_at, hit_count, source_request_id, error_message, preview, expires_at, signature - ) VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, 0, $5, $6, $7, $8, $9) ON CONFLICT(fingerprint) DO UPDATE SET last_seen_at = excluded.last_seen_at, error_message = COALESCE(excluded.error_message, refusal_guards.error_message), @@ -41,32 +41,34 @@ export class RefusalGuardsRepo { END `) this._hit = db.prepare(` - UPDATE refusal_guards SET hit_count = hit_count + 1, last_seen_at = ? - WHERE fingerprint = ? + UPDATE refusal_guards SET hit_count = hit_count + 1, last_seen_at = $1 + WHERE fingerprint = $2 `) this._list = db.prepare( `SELECT fingerprint, model, first_seen_at, last_seen_at, hit_count, source_request_id, error_message, preview, expires_at - FROM refusal_guards ORDER BY last_seen_at DESC, fingerprint LIMIT ?`, + FROM refusal_guards ORDER BY last_seen_at DESC, fingerprint LIMIT $1`, ) this._signatures = db.prepare( `SELECT fingerprint, signature FROM refusal_guards - WHERE signature IS NOT NULL AND (expires_at IS NULL OR expires_at > ?)`, + WHERE signature IS NOT NULL AND (expires_at IS NULL OR expires_at > $1)`, ) this._count = db.prepare('SELECT COUNT(*) AS n FROM refusal_guards') - this._remove = db.prepare('DELETE FROM refusal_guards WHERE fingerprint = ?') + this._remove = db.prepare('DELETE FROM refusal_guards WHERE fingerprint = $1') this._clear = db.prepare('DELETE FROM refusal_guards') } - get(fingerprint) { - const rec = rowToRec(this._get.get(String(fingerprint || ''))) + async get(fingerprint) { + const rec = rowToRec(await this._get.get(String(fingerprint || ''))) if (!rec?.expires_at) return rec if (Date.parse(rec.expires_at) > Date.now()) return rec - this.remove(rec.fingerprint) + await this.db + .prepare('DELETE FROM refusal_guards WHERE fingerprint = $1 AND expires_at = $2') + .run(rec.fingerprint, rec.expires_at) return null } - remember({ + async remember({ fingerprint, model = '', requestId = null, @@ -78,7 +80,7 @@ export class RefusalGuardsRepo { const fp = String(fingerprint || '').trim() if (!fp) return null const now = new Date().toISOString() - this._insert.run( + await this._insert.run( fp, String(model || ''), now, @@ -89,51 +91,48 @@ export class RefusalGuardsRepo { expiresAt || null, signature || null, ) - return this.get(fp) + return await this.get(fp) } - hit(fingerprint) { + async hit(fingerprint) { const fp = String(fingerprint || '').trim() if (!fp) return null - this._hit.run(new Date().toISOString(), fp) - return this.get(fp) + await this._hit.run(new Date().toISOString(), fp) + return await this.get(fp) } /** Highest stored user-text signature at or above the threshold. */ - nearest(signature, threshold = REFUSAL_SIMILARITY) { + async nearest(signature, threshold = REFUSAL_SIMILARITY) { const query = Array.isArray(signature) ? signature : decodeSignature(signature) if (!query) return null let best = null - for (const row of this._signatures.all(new Date().toISOString())) { + for (const row of await this._signatures.all(new Date().toISOString())) { const score = signatureSimilarity(query, decodeSignature(row.signature)) if (score < threshold) continue if (!best || score > best.score) best = { fingerprint: row.fingerprint, score } } if (!best) return null - const rec = this.get(best.fingerprint) + const rec = await this.get(best.fingerprint) if (!rec) return null return { ...rec, score: best.score } } - list(limit = 200) { + async list(limit = 200) { const n = Math.min(500, Math.max(1, Number(limit) || 200)) const now = Date.now() - return this._list - .all(n) - .map(rowToRec) - .filter((rec) => !rec.expires_at || Date.parse(rec.expires_at) > now) + return (await this._list.all(n)).map(rowToRec).filter((rec) => !rec.expires_at || Date.parse(rec.expires_at) > now) } - count() { - return Number(this._count.get()?.n) || 0 + async count() { + return Number((await this._count.get())?.n) || 0 } - remove(fingerprint) { + async remove(fingerprint) { const fp = String(fingerprint || '').trim() if (!fp) return false - return this._remove.run(fp).changes > 0 + return (await this._remove.run(fp)).changes > 0 } - clear() { - return this._clear.run().changes + async clear() { + return (await this._clear.run()).changes } } diff --git a/src/lib/db/repos/request-attempts-repo.mjs b/src/lib/db/repos/request-attempts-repo.mjs index 172b95e79..b704af0eb 100644 --- a/src/lib/db/repos/request-attempts-repo.mjs +++ b/src/lib/db/repos/request-attempts-repo.mjs @@ -43,26 +43,26 @@ export class RequestAttemptsRepo { started_at, completed_at, upstream_status, error_scope, action, cooldown_until, downstream_committed, terminal_state, usage_json, wait_ms, ttft_ms, latency_ms - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) `) this._complete = db.prepare(` UPDATE request_attempts SET - completed_at = ?, upstream_status = ?, error_scope = ?, action = ?, - cooldown_until = ?, downstream_committed = ?, terminal_state = ?, - usage_json = ?, ttft_ms = ?, latency_ms = ? - WHERE request_id = ? AND attempt_no = ? + completed_at = $1, upstream_status = $2, error_scope = $3, action = $4, + cooldown_until = $5, downstream_committed = $6, terminal_state = $7, + usage_json = $8, ttft_ms = $9, latency_ms = $10 + WHERE request_id = $11 AND attempt_no = $12 `) this._list = db.prepare(` - SELECT * FROM request_attempts WHERE request_id = ? ORDER BY attempt_no + SELECT * FROM request_attempts WHERE request_id = $1 ORDER BY attempt_no `) this._removeOld = db.prepare(` - DELETE FROM request_attempts WHERE started_at < ? + DELETE FROM request_attempts WHERE started_at < $1 `) } - begin({ requestId, attemptNo, vmId, accountId, model, selectionReason, waitMs = 0 }) { + async begin({ requestId, attemptNo, vmId, accountId, model, selectionReason, waitMs = 0 }) { const startedAt = new Date().toISOString() - this._insert.run( + await this._insert.run( requestId, attemptNo, vmId ?? null, @@ -82,11 +82,11 @@ export class RequestAttemptsRepo { null, null, ) - return this.list(requestId).find((item) => item.attempt_no === attemptNo) || null + return (await this.list(requestId)).find((item) => item.attempt_no === attemptNo) || null } - complete(requestId, attemptNo, result = {}) { - this._complete.run( + async complete(requestId, attemptNo, result = {}) { + await this._complete.run( new Date().toISOString(), result.upstreamStatus ?? null, result.errorScope ?? null, @@ -100,16 +100,16 @@ export class RequestAttemptsRepo { requestId, attemptNo, ) - return this.list(requestId).find((item) => item.attempt_no === attemptNo) || null + return (await this.list(requestId)).find((item) => item.attempt_no === attemptNo) || null } - list(requestId) { - return this._list.all(requestId).map(rowToAttempt) + async list(requestId) { + return (await this._list.all(requestId)).map(rowToAttempt) } - cleanup(retainDays = 7) { + async cleanup(retainDays = 7) { const days = Math.max(1, Number(retainDays) || 7) const cutoff = new Date(Date.now() - days * 86400_000).toISOString() - return this._removeOld.run(cutoff).changes + return (await this._removeOld.run(cutoff)).changes } } diff --git a/src/lib/db/repos/settings-repo.mjs b/src/lib/db/repos/settings-repo.mjs index 078de37b9..674a0776c 100644 --- a/src/lib/db/repos/settings-repo.mjs +++ b/src/lib/db/repos/settings-repo.mjs @@ -8,17 +8,17 @@ import { getDb } from '../database.mjs' export class SettingsRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT value FROM settings WHERE key = ?') + this._get = db.prepare('SELECT value FROM settings WHERE key = $1') this._set = db.prepare(` - INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?) + INSERT INTO settings (key, value, updated_at) VALUES ($1, $2, $3) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at `) - this._del = db.prepare('DELETE FROM settings WHERE key = ?') + this._del = db.prepare('DELETE FROM settings WHERE key = $1') this._all = db.prepare('SELECT key, value, updated_at FROM settings ORDER BY key') } - get(key, fallback = null) { - const row = this._get.get(key) + async get(key, fallback = null) { + const row = await this._get.get(key) if (!row) return fallback try { return JSON.parse(row.value) @@ -27,28 +27,33 @@ export class SettingsRepo { } } - set(key, value) { - this._set.run(key, JSON.stringify(value), new Date().toISOString()) + async set(key, value) { + await this._set.run(key, JSON.stringify(value), new Date().toISOString()) + this.db.afterCommit(() => this.db.settingsState?.set(key, value)) return value } - has(key) { - return !!this._get.get(key) + async has(key) { + return !!(await this._get.get(key)) } - remove(key) { - this._del.run(key) + async remove(key) { + await this._del.run(key) + this.db.afterCommit(() => this.db.settingsState?.delete(key)) } - all() { + async all() { const out = {} - for (const row of this._all.all()) { + for (const row of await this._all.all()) { try { out[row.key] = JSON.parse(row.value) } catch { out[row.key] = null } } + this.db.afterCommit(() => { + this.db.settingsState = new Map(Object.entries(out)) + }) return out } } diff --git a/src/lib/db/repos/sticky-repo.mjs b/src/lib/db/repos/sticky-repo.mjs index 17def6230..032543557 100644 --- a/src/lib/db/repos/sticky-repo.mjs +++ b/src/lib/db/repos/sticky-repo.mjs @@ -22,10 +22,10 @@ function rowToEntry(row) { export class StickyRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM sticky_sessions WHERE key = ?') + this._get = db.prepare('SELECT * FROM sticky_sessions WHERE key = $1') this._upsert = db.prepare(` INSERT INTO sticky_sessions (key, account_id, vm_id, session_id, device_id, bound_at, expires_at, hits, generation, slot_index) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT(key) DO UPDATE SET account_id = excluded.account_id, vm_id = excluded.vm_id, @@ -37,28 +37,28 @@ export class StickyRepo { generation = excluded.generation, slot_index = excluded.slot_index `) - this._delete = db.prepare('DELETE FROM sticky_sessions WHERE key = ?') - this._deleteByAccount = db.prepare('DELETE FROM sticky_sessions WHERE account_id = ? OR vm_id = ?') - this._purge = db.prepare('DELETE FROM sticky_sessions WHERE expires_at < ?') + this._delete = db.prepare('DELETE FROM sticky_sessions WHERE key = $1') + this._deleteByAccount = db.prepare('DELETE FROM sticky_sessions WHERE account_id = $1 OR vm_id = $2') + this._purge = db.prepare('DELETE FROM sticky_sessions WHERE expires_at < $1') this._dropAlias = db.prepare(` DELETE FROM sticky_sessions WHERE key = 'envelope' - OR key LIKE 'ch:%' - OR key LIKE 'dev:%' - OR key LIKE '%:envelope' - OR key LIKE '%:ch:%' - OR key LIKE '%:dev:%' + OR ascii_fold(key) LIKE 'ch:%' + OR ascii_fold(key) LIKE 'dev:%' + OR ascii_fold(key) LIKE '%:envelope' + OR ascii_fold(key) LIKE '%:ch:%' + OR ascii_fold(key) LIKE '%:dev:%' `) this._all = db.prepare('SELECT * FROM sticky_sessions') this._count = db.prepare('SELECT COUNT(*) c FROM sticky_sessions') } - get(key) { - return rowToEntry(this._get.get(key)) + async get(key) { + return rowToEntry(await this._get.get(key)) } - upsert(key, ent) { - this._upsert.run( + async upsert(key, ent) { + await this._upsert.run( key, ent.account_id ?? null, ent.vm_id ?? null, @@ -72,31 +72,31 @@ export class StickyRepo { ) } - remove(key) { - this._delete.run(key) + async remove(key) { + await this._delete.run(key) } - removeByAccount({ accountId = null, vmId = null } = {}) { + async removeByAccount({ accountId = null, vmId = null } = {}) { if (!accountId && !vmId) return 0 - return this._deleteByAccount.run(accountId || '', vmId || '').changes + return (await this._deleteByAccount.run(accountId || '', vmId || '')).changes } - purgeExpired(now = Date.now()) { - return this._purge.run(now).changes + async purgeExpired(now = Date.now()) { + return (await this._purge.run(now)).changes } /** Content fingerprints and device aliases are not conversation slots. */ - dropAliasKeys() { - return this._dropAlias.run().changes + async dropAliasKeys() { + return (await this._dropAlias.run()).changes } - all() { + async all() { const out = {} - for (const row of this._all.all()) out[row.key] = rowToEntry(row) + for (const row of await this._all.all()) out[row.key] = rowToEntry(row) return out } - count() { - return this._count.get().c + async count() { + return (await this._count.get()).c } } diff --git a/src/lib/db/repos/vm-pools-repo.mjs b/src/lib/db/repos/vm-pools-repo.mjs index 45f85dd22..1932c11b7 100644 --- a/src/lib/db/repos/vm-pools-repo.mjs +++ b/src/lib/db/repos/vm-pools-repo.mjs @@ -25,105 +25,115 @@ export class VmPoolsRepo { constructor(db = getDb()) { this.db = db this._list = db.prepare('SELECT * FROM vm_pools ORDER BY created_at, id') - this._get = db.prepare('SELECT * FROM vm_pools WHERE id = ?') - this._members = db.prepare('SELECT vm_id FROM vm_pool_members WHERE pool_id = ? ORDER BY vm_id') + this._get = db.prepare('SELECT * FROM vm_pools WHERE id = $1') + this._members = db.prepare('SELECT vm_id FROM vm_pool_members WHERE pool_id = $1 ORDER BY vm_id') this._allMembers = db.prepare('SELECT pool_id, vm_id FROM vm_pool_members ORDER BY vm_id') - this._insert = db.prepare('INSERT INTO vm_pools (id, name, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?)') - this._name = db.prepare('UPDATE vm_pools SET name = ?, updated_at = ? WHERE id = ?') - this._enabled = db.prepare('UPDATE vm_pools SET enabled = ?, updated_at = ? WHERE id = ?') - this._touch = db.prepare('UPDATE vm_pools SET updated_at = ? WHERE id = ?') - this._delete = db.prepare('DELETE FROM vm_pools WHERE id = ?') - this._clearMembers = db.prepare('DELETE FROM vm_pool_members WHERE pool_id = ?') - this._addMember = db.prepare('INSERT INTO vm_pool_members (pool_id, vm_id, created_at) VALUES (?, ?, ?)') - this._other = db.prepare('SELECT pool_id FROM vm_pool_members WHERE vm_id = ? AND pool_id != ?') - this._keyCount = db.prepare('SELECT COUNT(*) AS n FROM api_keys WHERE vm_pool_id = ? AND deleted_at IS NULL') + this._insert = db.prepare( + 'INSERT INTO vm_pools (id, name, enabled, created_at, updated_at) VALUES ($1, $2, $3, $4, $5)', + ) + this._name = db.prepare('UPDATE vm_pools SET name = $1, updated_at = $2 WHERE id = $3') + this._enabled = db.prepare('UPDATE vm_pools SET enabled = $1, updated_at = $2 WHERE id = $3') + this._touch = db.prepare('UPDATE vm_pools SET updated_at = $1 WHERE id = $2') + this._delete = db.prepare('DELETE FROM vm_pools WHERE id = $1') + this._clearMembers = db.prepare('DELETE FROM vm_pool_members WHERE pool_id = $1') + this._addMember = db.prepare('INSERT INTO vm_pool_members (pool_id, vm_id, created_at) VALUES ($1, $2, $3)') + this._other = db.prepare('SELECT pool_id FROM vm_pool_members WHERE vm_id = $1 AND pool_id != $2') + this._keyCount = db.prepare('SELECT COUNT(*) AS n FROM api_keys WHERE vm_pool_id = $1 AND deleted_at IS NULL') } - _ids(poolId) { - return this._members.all(poolId).map((row) => row.vm_id) + async _ids(poolId) { + return (await this._members.all(poolId)).map((row) => row.vm_id) } - list() { + async list() { const grouped = new Map() - for (const row of this._allMembers.all()) { + for (const row of await this._allMembers.all()) { const list = grouped.get(row.pool_id) || [] list.push(row.vm_id) grouped.set(row.pool_id, list) } - return this._list.all().map((row) => rowToPool(row, grouped.get(row.id) || [])) + return (await this._list.all()).map((row) => rowToPool(row, grouped.get(row.id) || [])) } - get(id) { - const row = this._get.get(id) + async get(id) { + const row = await this._get.get(id) if (!row) return null - return rowToPool(row, this._ids(id)) + return rowToPool(row, await this._ids(id)) } - countKeys(id) { - return Number(this._keyCount.get(id)?.n) || 0 + async countKeys(id) { + return Number((await this._keyCount.get(id))?.n) || 0 } - insert({ name, enabled = true }) { + async insert({ name, enabled = true }) { const now = nowIso() const id = `pool_${crypto.randomBytes(4).toString('hex')}` try { - this._insert.run(id, name, enabled ? 1 : 0, now, now) + await this._insert.run(id, name, enabled ? 1 : 0, now, now) } catch (error) { throw mapConstraint(error) } - return this.get(id) + return await this.get(id) } - rename(id, name) { + async rename(id, name) { try { - const info = this._name.run(name, nowIso(), id) + const info = await this._name.run(name, nowIso(), id) if (!info.changes) return null } catch (error) { throw mapConstraint(error) } - return this.get(id) + return await this.get(id) } - setEnabled(id, enabled) { - const info = this._enabled.run(enabled ? 1 : 0, nowIso(), id) + async setEnabled(id, enabled) { + const info = await this._enabled.run(enabled ? 1 : 0, nowIso(), id) if (!info.changes) return null - return this.get(id) + return await this.get(id) } /** * Replace membership. A vm already in another pool is rejected and nothing * is written. Passing the same vm ids is a no-op aside from updated_at. */ - setMembers(id, vmIds) { - if (!this._get.get(id)) return null - for (const vmId of vmIds) { - const other = this._other.get(vmId, id) - if (other) { - throw Object.assign(new Error(`槽位 ${vmId} 已在其他账号池`), { - code: 'vm_in_other_pool', - vm_id: vmId, - pool_id: other.pool_id, - }) + async setMembers(id, vmIds) { + return withTransaction(this.db, async () => { + await this.db.prepare('SELECT id FROM vm_pools WHERE id = $1 FOR UPDATE').get(id) + + if (!(await this._get.get(id))) return null + const lockedIds = [...new Set([...vmIds, ...(await this._ids(id))])].sort() + for (const vmId of lockedIds) { + await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(`vm_pool_member:${vmId}`) } - } - const now = nowIso() - withTransaction(this.db, () => { - this._clearMembers.run(id) - for (const vmId of vmIds) this._addMember.run(id, vmId, now) - this._touch.run(now, id) + for (const vmId of vmIds) { + const other = await this._other.get(vmId, id) + if (other) { + throw Object.assign(new Error(`槽位 ${vmId} 已在其他账号池`), { + code: 'vm_in_other_pool', + vm_id: vmId, + pool_id: other.pool_id, + }) + } + } + const now = nowIso() + await withTransaction(this.db, async () => { + await this._clearMembers.run(id) + for (const vmId of vmIds) await this._addMember.run(id, vmId, now) + await this._touch.run(now, id) + }) + return await this.get(id) }) - return this.get(id) } - remove(id) { - const info = this._delete.run(id) + async remove(id) { + const info = await this._delete.run(id) return info.changes > 0 } } function mapConstraint(error) { const message = String(error?.message || error) - if (/UNIQUE/i.test(message) && /vm_pools|name/i.test(message)) { + if (error?.code === '23505' && /vm_pools.*name|name.*vm_pools/i.test(error.constraint || message)) { return Object.assign(new Error('账号池名称已存在'), { code: 'vm_pool_name_taken' }) } return error diff --git a/src/lib/db/repos/vms-repo.mjs b/src/lib/db/repos/vms-repo.mjs index 675ba9f25..0539464bf 100644 --- a/src/lib/db/repos/vms-repo.mjs +++ b/src/lib/db/repos/vms-repo.mjs @@ -18,9 +18,9 @@ export class VmsRepo { constructor(db = getDb()) { this.db = db this.settings = new SettingsRepo(db) - this._get = db.prepare('SELECT * FROM vms WHERE id = ?') + this._get = db.prepare('SELECT * FROM vms WHERE id = $1') this._list = db.prepare('SELECT * FROM vms ORDER BY id') - this._remove = db.prepare('DELETE FROM vms WHERE id = ?') + this._remove = db.prepare('DELETE FROM vms WHERE id = $1') this._upsert = db.prepare(` INSERT INTO vms ( id, name, status, schedulable, email, account_uuid, org_uuid, @@ -28,7 +28,7 @@ export class VmsRepo { proxy_id, claude_code_version, timezone, locale, owner_user_id, origin, vm_json, encrypted, file_mtime_ms, synced_at, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) ON CONFLICT(id) DO UPDATE SET name = excluded.name, status = excluded.status, schedulable = excluded.schedulable, email = excluded.email, account_uuid = excluded.account_uuid, org_uuid = excluded.org_uuid, @@ -44,7 +44,7 @@ export class VmsRepo { } /** Mirror a parsed vm.json object into the table. */ - upsertFromVmJson(vm, { mtimeMs = null } = {}) { + async upsertFromVmJson(vm, { mtimeMs = null } = {}) { if (!vm?.id) return null const now = new Date().toISOString() const c = vm.claude || {} @@ -55,7 +55,7 @@ export class VmsRepo { }) const safeVm = { ...vm, claude: safeClaude } const enc = encryptionEnabled() - this._upsert.run( + await this._upsert.run( vm.id, vm.name ?? null, vm.status ?? null, @@ -81,43 +81,40 @@ export class VmsRepo { vm.created_at ?? now, now, ) - return this.get(vm.id) + return await this.get(vm.id) } /** Returns {row, vm} — row has decrypted credential columns, vm is the parsed vm.json. */ - get(id) { - return this._decode(this._get.get(id)) + async get(id) { + return this._decode(await this._get.get(id)) } - list() { - return this._list.all().map((r) => this._decode(r)) + async list() { + return (await this._list.all()).map((r) => this._decode(r)) } - ids() { - return this.db - .prepare('SELECT id FROM vms ORDER BY id') - .all() - .map((r) => r.id) + async ids() { + return (await this.db.prepare('SELECT id FROM vms ORDER BY id').all()).map((r) => r.id) } - remove(id) { - return this._remove.run(id).changes > 0 + async remove(id) { + return (await this._remove.run(id)).changes > 0 } - count() { - return this.db.prepare('SELECT COUNT(*) c FROM vms').get().c + async count() { + return (await this.db.prepare('SELECT COUNT(*) c FROM vms').get()).c } - getActiveVmId() { - return this.settings.get(ACTIVE_VM_KEY, null) + async getActiveVmId() { + return await this.settings.get(ACTIVE_VM_KEY, null) } - setActiveVmId(id) { - this.settings.set(ACTIVE_VM_KEY, id) + async setActiveVmId(id) { + await this.settings.set(ACTIVE_VM_KEY, id) } - clearActiveVmId() { - this.settings.remove(ACTIVE_VM_KEY) + async clearActiveVmId() { + await this.settings.remove(ACTIVE_VM_KEY) } _decode(row) { diff --git a/test/unit/api-endpoints.test.mjs b/test/unit/api-endpoints.test.mjs index 9ce59bfeb..1d96decb6 100644 --- a/test/unit/api-endpoints.test.mjs +++ b/test/unit/api-endpoints.test.mjs @@ -10,9 +10,9 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-api-ep-')) } -test('create endpoint with keys and models, list masks keys', () => { +test('create endpoint with keys and models, list masks keys', async () => { const store = new ApiEndpointStore({ dataDir: tmpDir() }) - const rec = store.create({ + const rec = await store.create({ name: 'openrouter', base_url: 'https://example.com', prefix: 'or', @@ -22,14 +22,14 @@ test('create endpoint with keys and models, list masks keys', () => { assert.equal(rec.base_url, 'https://example.com') assert.equal(rec.api_key_entries.length, 1) assert.equal(rec.models[0].alias, 'kimi') - const listed = store.list() + const listed = await store.list() assert.ok(listed[0].api_key_entries[0].api_key.includes('…')) assert.ok(!listed[0].api_key_entries[0].api_key.includes('secret-key')) }) -test('create strips trailing /v1 from base_url', () => { +test('create strips trailing /v1 from base_url', async () => { const store = new ApiEndpointStore({ dataDir: tmpDir() }) - const rec = store.create({ + const rec = await store.create({ name: 'a', base_url: 'https://example.com/v1/', api_key_entries: [{ api_key: 'sk-or-xxxx' }], @@ -38,23 +38,23 @@ test('create strips trailing /v1 from base_url', () => { assert.equal(rec.base_url, 'https://example.com') }) -test('official kinds lock base_url', () => { +test('official kinds lock base_url', async () => { const store = new ApiEndpointStore({ dataDir: tmpDir() }) - const claude = store.create({ kind: 'claude', api_key_entries: [{ api_key: 'sk-ant-xxxx' }] }) + const claude = await store.create({ kind: 'claude', api_key_entries: [{ api_key: 'sk-ant-xxxx' }] }) assert.equal(claude.kind, 'claude') assert.equal(claude.base_url, 'https://api.anthropic.com') assert.equal(claude.protocol, 'anthropic') - const openai = store.create({ kind: 'openai', name: 'oai', api_key_entries: [{ api_key: 'sk-oai-xxxx' }] }) + const openai = await store.create({ kind: 'openai', name: 'oai', api_key_entries: [{ api_key: 'sk-oai-xxxx' }] }) assert.equal(openai.kind, 'openai') assert.equal(openai.base_url, 'https://api.openai.com') assert.equal(openai.protocol, 'openai') }) -test('sk-kin category defaults oauth and can switch to api', () => { +test('sk-kin category defaults oauth and can switch to api', async () => { const keys = new ApiKeyStore({ dataDir: tmpDir() }) - const rec = keys.create({ name: 'c' }) + const rec = await keys.create({ name: 'c' }) assert.equal(rec.category, 'oauth') - const updated = keys.update(rec.id, { category: 'api' }) + const updated = await keys.update(rec.id, { category: 'api' }) assert.equal(updated.category, 'api') - assert.equal(keys.list()[0].category, 'api') + assert.equal((await keys.list())[0].category, 'api') }) diff --git a/test/unit/api-keys.test.mjs b/test/unit/api-keys.test.mjs index 35e7647a4..f5911b9aa 100644 --- a/test/unit/api-keys.test.mjs +++ b/test/unit/api-keys.test.mjs @@ -23,203 +23,242 @@ test('maskApiKey hides middle', () => { assert.ok(!m.includes('a'.repeat(20))) }) -test('create + authenticate + list mask', () => { +test('create + authenticate + list mask', async () => { const store = tmpStore() - const rec = store.create({ name: 'demo', max_concurrency: 3, quota_requests: 10, rpm: 60 }) + const rec = await store.create({ name: 'demo', max_concurrency: 3, quota_requests: 10, rpm: 60 }) assert.equal(rec.name, 'demo') assert.equal(rec.max_concurrency, 3) assert.equal(rec.quota_requests, 10) - const auth = store.authenticate(rec.key) + const auth = await store.authenticate(rec.key) assert.equal(auth.ok, true) assert.equal(auth.record.id, rec.id) - const stored = store.getById(rec.id) + const stored = await store.getById(rec.id) assert.notEqual(stored.key, rec.key) assert.match(stored.key, /^hmac:/) assert.ok(stored.key_hash) - const listed = store.list() + const listed = await store.list() assert.equal(listed.length, 1) assert.ok(listed[0].key.includes('…')) assert.notEqual(listed[0].key, rec.key) }) -test('reveal returns the plaintext for copy; rotate swaps it in place', () => { +test('reveal returns the plaintext for copy; rotate swaps it in place', async () => { const store = tmpStore() - const rec = store.create({ name: 'copyable', max_concurrency: 4 }) - const listed = store.list()[0] + const rec = await store.create({ name: 'copyable', max_concurrency: 4 }) + const listed = (await store.list())[0] assert.equal(listed.revealable, true) assert.ok(listed.key.includes('…'), 'list stays masked') - const shown = store.reveal(rec.id) + const shown = await store.reveal(rec.id) assert.equal(shown.ok, true) assert.equal(shown.key, rec.key) - const rotated = store.rotate(rec.id) + const rotated = await store.rotate(rec.id) assert.notEqual(rotated.key, rec.key) - assert.equal(store.authenticate(rec.key).ok, false, 'old value dies') - assert.equal(store.authenticate(rotated.key).ok, true) - assert.equal(store.reveal(rec.id).key, rotated.key) - const kept = store.getById(rec.id) + assert.equal((await store.authenticate(rec.key)).ok, false, 'old value dies') + assert.equal((await store.authenticate(rotated.key)).ok, true) + assert.equal((await store.reveal(rec.id)).key, rotated.key) + const kept = await store.getById(rec.id) assert.equal(kept.name, 'copyable') assert.equal(kept.max_concurrency, 4) }) -test('reveal reports hash-only rows as unrecoverable', () => { +test('reveal reports hash-only rows as unrecoverable', async () => { const store = tmpStore() - const rec = store.create({ name: 'legacy' }) - const row = store.getById(rec.id) + const rec = await store.create({ name: 'legacy' }) + const row = await store.getById(rec.id) row.key_secret = null - store.repo.update(row) + await store.repo.update(row) - const out = store.reveal(rec.id) + const out = await store.reveal(rec.id) assert.equal(out.ok, false) assert.equal(out.code, 'key_not_recoverable') - assert.equal(store.list()[0].revealable, false) - assert.equal(store.reveal('key_missing'), null) - assert.equal(store.authenticate(rec.key).ok, true, 'auth still works off the hash') + assert.equal((await store.list())[0].revealable, false) + assert.equal(await store.reveal('key_missing'), null) + assert.equal((await store.authenticate(rec.key)).ok, true, 'auth still works off the hash') }) -test('key_secret is encrypted at rest when KIN_DB_SECRET is set', () => { +test('key_secret is encrypted at rest when KIN_DB_SECRET is set', async () => { const prev = process.env.KIN_DB_SECRET process.env.KIN_DB_SECRET = 'unit-test-secret' try { const store = tmpStore() - const rec = store.create({ name: 'enc' }) - const row = store.getById(rec.id) + const rec = await store.create({ name: 'enc' }) + const row = await store.getById(rec.id) assert.match(row.key_secret, /^enc:v1:/) assert.ok(!row.key_secret.includes(rec.key)) - assert.equal(store.reveal(rec.id).key, rec.key) + assert.equal((await store.reveal(rec.id)).key, rec.key) } finally { if (prev === undefined) delete process.env.KIN_DB_SECRET else process.env.KIN_DB_SECRET = prev } }) -test('canAccept rejects disabled / expired / quota / concurrency', () => { +test('canAccept rejects disabled / expired / quota / concurrency', async () => { const store = tmpStore() - const rec = store.create({ name: 'lim', max_concurrency: 1, quota_requests: 2, rpm: 0 }) - assert.equal(store.canAccept(rec).ok, true) + const rec = await store.create({ name: 'lim', max_concurrency: 1, quota_requests: 2, rpm: 0 }) + assert.equal((await store.canAccept(rec)).ok, true) - store.update(rec.id, { status: 'disabled' }) - assert.equal(store.canAccept(store.getById(rec.id)).code, 'api_key_disabled') - store.update(rec.id, { status: 'active' }) + await store.update(rec.id, { status: 'disabled' }) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_disabled') + await store.update(rec.id, { status: 'active' }) - store.update(rec.id, { expires_at: new Date(Date.now() - 1000).toISOString() }) - assert.equal(store.canAccept(store.getById(rec.id)).code, 'api_key_expired') - store.update(rec.id, { expires_at: null }) + await store.update(rec.id, { expires_at: new Date(Date.now() - 1000).toISOString() }) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_expired') + await store.update(rec.id, { expires_at: null }) - const a = store.acquire(store.getById(rec.id)) + const a = await store.acquire(await store.getById(rec.id)) assert.equal(a.ok, true) - const blocked = store.canAccept(store.getById(rec.id)) + const blocked = await store.canAccept(await store.getById(rec.id)) assert.equal(blocked.code, 'api_key_concurrency_limit') store.release(rec.id) - store.recordUsage(rec.id) - store.recordUsage(rec.id) - assert.equal(store.canAccept(store.getById(rec.id)).code, 'api_key_quota_exhausted') + await store.recordUsage(rec.id) + await store.recordUsage(rec.id) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_quota_exhausted') }) -test('rpm window blocks then recovers', () => { +test('rpm window blocks then recovers', async () => { const store = tmpStore() - const rec = store.create({ name: 'rpm', max_concurrency: 0, quota_requests: 0, rpm: 2 }) - assert.equal(store.acquire(rec).ok, true) + const rec = await store.create({ name: 'rpm', max_concurrency: 0, quota_requests: 0, rpm: 2 }) + assert.equal((await store.acquire(rec)).ok, true) store.release(rec) - assert.equal(store.acquire(rec).ok, true) + assert.equal((await store.acquire(rec)).ok, true) store.release(rec) - const gate = store.canAccept(store.getById(rec.id)) + const gate = await store.canAccept(await store.getById(rec.id)) assert.equal(gate.code, 'api_key_rate_limit') }) -test('user concurrency caps regular users but never the admin operator', () => { +test('user concurrency caps regular users but never the admin operator', async () => { const store = tmpStore() const now = new Date().toISOString() for (const [id, role] of [ ['u-admin', 'admin'], ['u-user', 'user'], ]) { - store.users.insert({ id, username: id, password_hash: 'x', role, concurrency: 1, created_at: now, updated_at: now }) + await store.users.insert({ + id, + username: id, + password_hash: 'x', + role, + concurrency: 1, + created_at: now, + updated_at: now, + }) } - const admin = store.create({ name: 'admin-key', max_concurrency: 0, user_id: 'u-admin' }) - const user = store.create({ name: 'user-key', max_concurrency: 0, user_id: 'u-user' }) + const admin = await store.create({ name: 'admin-key', max_concurrency: 0, user_id: 'u-admin' }) + const user = await store.create({ name: 'user-key', max_concurrency: 0, user_id: 'u-user' }) - for (let i = 0; i < 3; i++) assert.equal(store.acquire(admin).ok, true) + for (let i = 0; i < 3; i++) assert.equal((await store.acquire(admin)).ok, true) - assert.equal(store.acquire(user).ok, true) - const blocked = store.canAccept(user) + assert.equal((await store.acquire(user)).ok, true) + const blocked = await store.canAccept(user) assert.equal(blocked.code, 'user_concurrency_limit') assert.deepEqual(blocked.detail, { inflight: 1, max: 1 }) }) -test('update reset_quota and remove', () => { +test('update reset_quota and remove', async () => { const store = tmpStore() - const rec = store.create({ name: 'x', quota_requests: 5 }) - store.recordUsage(rec.id) - store.recordUsage(rec.id) - assert.equal(store.getById(rec.id).quota_requests_used, 2) - store.update(rec.id, { reset_quota: true }) - assert.equal(store.getById(rec.id).quota_requests_used, 0) - assert.equal(store.remove(rec.id), true) - assert.equal(store.getById(rec.id), null) - assert.equal(store.authenticate(rec.key).ok, false) -}) - -test('recordUsage accumulates cache token columns', () => { + const rec = await store.create({ name: 'x', quota_requests: 5 }) + await store.recordUsage(rec.id) + await store.recordUsage(rec.id) + assert.equal((await store.getById(rec.id)).quota_requests_used, 2) + await store.update(rec.id, { reset_quota: true }) + assert.equal((await store.getById(rec.id)).quota_requests_used, 0) + assert.equal(await store.remove(rec.id), true) + assert.equal(await store.getById(rec.id), null) + assert.equal((await store.authenticate(rec.key)).ok, false) +}) + +test('recordUsage accumulates cache token columns', async () => { const store = tmpStore() - const rec = store.create({ name: 'cache' }) - store.recordUsage(rec.id, { + const rec = await store.create({ name: 'cache' }) + await store.recordUsage(rec.id, { input_tokens: 10, output_tokens: 5, cache_read_input_tokens: 3, cache_creation_input_tokens: 7, }) - store.recordUsage(rec.id, { input_tokens: 1, output_tokens: 1, cache_read_tokens: 2 }) - const saved = store.getById(rec.id) + await store.recordUsage(rec.id, { input_tokens: 1, output_tokens: 1, cache_read_tokens: 2 }) + const saved = await store.getById(rec.id) assert.equal(saved.tokens_in, 11) assert.equal(saved.tokens_out, 6) assert.equal(saved.cache_read_tokens, 5) assert.equal(saved.cache_creation_tokens, 7) }) -test('custom key rejected when duplicate', () => { +test('custom key rejected when duplicate', async () => { const store = tmpStore() - store.create({ name: 'a', key: 'sk-kin-customkey0001' }) - assert.throws(() => store.create({ name: 'b', key: 'sk-kin-customkey0001' }), /exists/) + await store.create({ name: 'a', key: 'sk-kin-customkey0001' }) + await assert.rejects(async () => await store.create({ name: 'b', key: 'sk-kin-customkey0001' }), /exists/) }) -test('keys persist in sqlite across store re-open (same dataDir)', () => { +test('keys persist in PostgreSQL across store recreation', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-keys-')) const store = new ApiKeyStore({ dataDir: dir }) - const rec = store.create({ name: 'persist', quota_requests: 9 }) - store.recordUsage(rec.id, { input_tokens: 11, output_tokens: 22 }) + const rec = await store.create({ name: 'persist', quota_requests: 9 }) + await store.recordUsage(rec.id, { input_tokens: 11, output_tokens: 22 }) const store2 = new ApiKeyStore({ dataDir: dir }) - const again = store2.getById(rec.id) + const again = await store2.getById(rec.id) assert.ok(again, 'record should survive re-open') assert.equal(again.name, 'persist') assert.equal(again.quota_requests_used, 1) assert.equal(again.tokens_in, 11) assert.equal(again.tokens_out, 22) - assert.equal(store2.authenticate(rec.key).ok, true) - assert.ok(fs.existsSync(path.join(dir, 'kin.db')), 'kin.db file exists') + assert.equal((await store2.authenticate(rec.key)).ok, true) }) -test('group_type all is the default and a platform group keeps its VM list', () => { +test('group_type all is the default and a platform group keeps its VM list', async () => { const store = tmpStore() - const open = store.create({ name: 'global' }) - assert.equal(store.list()[0].group_type, 'all') - assert.deepEqual(store.list()[0].allowed_vms, []) - const scoped = store.create({ name: 'claude', group_type: 'anthropic', allowed_vms: ['vm-01', 'vm-03'] }) - assert.equal(store.getById(scoped.id).group_type, 'anthropic') - assert.equal( - store - .list() - .find((k) => k.id === scoped.id) - .allowed_vms.join(','), - 'vm-01,vm-03', - ) - const widened = store.update(scoped.id, { group_type: 'all' }) + const open = await store.create({ name: 'global' }) + assert.equal((await store.list())[0].group_type, 'all') + assert.deepEqual((await store.list())[0].allowed_vms, []) + const scoped = await store.create({ name: 'claude', group_type: 'anthropic', allowed_vms: ['vm-01', 'vm-03'] }) + assert.equal((await store.getById(scoped.id)).group_type, 'anthropic') + assert.equal((await store.list()).find((k) => k.id === scoped.id).allowed_vms.join(','), 'vm-01,vm-03') + const widened = await store.update(scoped.id, { group_type: 'all' }) assert.equal(widened.group_type, 'all') assert.equal(widened.allowed_vms, '[]') - assert.throws(() => store.create({ name: 'bare', group_type: 'openai' }), /至少勾选一台 VM/) - store.update(open.id, { status: 'disabled' }) - assert.equal(store.getById(open.id).group_type, 'all') + await assert.rejects(async () => await store.create({ name: 'bare', group_type: 'openai' }), /至少勾选一台 VM/) + await store.update(open.id, { status: 'disabled' }) + assert.equal((await store.getById(open.id)).group_type, 'all') +}) + +test('simultaneous owned-key acquires reserve one concurrency slot', async () => { + const store = tmpStore() + await store.users.insert({ + id: 'parallel-owner', + username: 'parallel-owner', + password_hash: 'x', + role: 'user', + concurrency: 1, + }) + const key = await store.create({ name: 'parallel-key', user_id: 'parallel-owner', max_concurrency: 1 }) + const results = await Promise.all(Array.from({ length: 12 }, () => store.acquire(key))) + assert.equal(results.filter((r) => r.ok).length, 1) + assert.equal(store.inflight.get(key.id), 1) + store.release(key) + assert.equal((await store.acquire(key)).ok, true) + store.release(key) +}) + +test('simultaneous usage writes retain every counter and spending window', async () => { + const store = tmpStore() + const key = await store.create({ name: 'parallel-billing', quota: 2 }) + await Promise.all( + Array.from({ length: 12 }, () => + store.recordUsage(key.id, { input_tokens: 3, output_tokens: 2, total_cost: 0.25 }), + ), + ) + const row = await store.getById(key.id) + assert.equal(row.requests, 12) + assert.equal(row.quota_requests_used, 12) + assert.equal(row.tokens_in, 36) + assert.equal(row.tokens_out, 24) + assert.equal(row.quota_used, 3) + assert.equal(row.usage_5h, 3) + assert.equal(row.usage_1d, 3) + assert.equal(row.usage_7d, 3) + assert.equal(row.status, 'quota_exhausted') }) diff --git a/test/unit/cluster-node-edit-repo.test.mjs b/test/unit/cluster-node-edit-repo.test.mjs index ef4bc6c49..3fd740725 100644 --- a/test/unit/cluster-node-edit-repo.test.mjs +++ b/test/unit/cluster-node-edit-repo.test.mjs @@ -1,7 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' -import fs from 'node:fs' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import { ClusterNodesRepo } from '../../src/lib/db/repos/cluster-nodes-repo.mjs' const NODE = { @@ -19,7 +18,7 @@ const NODE = { jump_node_id: null, } -function setup(t, secret) { +async function setup(t, secret) { const env = { VM2API_DB_SECRET: process.env.VM2API_DB_SECRET, KIN_DB_SECRET: process.env.KIN_DB_SECRET } delete process.env.VM2API_DB_SECRET delete process.env.KIN_DB_SECRET @@ -30,9 +29,8 @@ function setup(t, secret) { else process.env[key] = value } }) - const db = new DatabaseSync(':memory:') - db.exec(fs.readFileSync(new URL('../../src/lib/db/migrations/026_cluster_nodes.sql', import.meta.url), 'utf8')) - t.after(() => db.close()) + const db = getDb() + t.after(async () => await db.close()) return { db, repo: new ClusterNodesRepo(db) } } @@ -42,12 +40,12 @@ function assertPublic(node) { } } -test('node rename preserves identity, pinned host key, jump node and credentials without exposing secrets', (t) => { - const { repo } = setup(t) - repo.insert({ ...NODE, id: 'node-jump', host: 'jump.example.test' }) +test('node rename preserves identity, pinned host key, jump node and credentials without exposing secrets', async (t) => { + const { repo } = await setup(t) + await repo.insert({ ...NODE, id: 'node-jump', host: 'jump.example.test' }) const spec = { ...NODE, jump_node_id: 'node-jump' } - const original = repo.insert(spec, '2020-01-01T00:00:00.000Z') - const saved = repo.update( + const original = await repo.insert(spec, '2020-01-01T00:00:00.000Z') + const saved = await repo.update( original.id, { ...spec, @@ -65,110 +63,112 @@ test('node rename preserves identity, pinned host key, jump node and credentials assert.equal(saved[key], original[key]) } assert.ok(Date.parse(saved.updated_at) > Date.parse(original.updated_at)) - assert.deepEqual(repo.getSecrets(original.id), { + assert.deepEqual(await repo.getSecrets(original.id), { password: NODE.password, privateKey: null, passphrase: null, }) assertPublic(saved) - assertPublic(repo.get(original.id)) - for (const node of repo.list()) assertPublic(node) + assertPublic(await repo.get(original.id)) + for (const node of await repo.list()) assertPublic(node) }) -test('label-only edits preserve encrypted credential bytes even when the current database key cannot decrypt them', (t) => { - const { db, repo } = setup(t, 'original-test-database-key') - const original = repo.insert({ ...NODE, private_key: 'saved-private-key', passphrase: 'saved-passphrase' }) - const credentials = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?') - const stored = credentials.get(NODE.id) +test('label-only edits preserve encrypted credential bytes even when the current database key cannot decrypt them', async (t) => { + const { db, repo } = await setup(t, 'original-test-database-key') + const original = await repo.insert({ ...NODE, private_key: 'saved-private-key', passphrase: 'saved-passphrase' }) + const credentials = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + const stored = await credentials.get(NODE.id) for (const value of Object.values(stored)) assert.match(value, /^enc:v1:/) process.env.VM2API_DB_SECRET = 'different-test-database-key' - assert.throws(() => repo.getSecrets(NODE.id)) - const saved = repo.updateLabel(NODE.id, 'Renamed without credentials', original.updated_at) + await assert.rejects(async () => await repo.getSecrets(NODE.id)) + const saved = await repo.updateLabel(NODE.id, 'Renamed without credentials', original.updated_at) assert.deepEqual(saved, { ...original, label: 'Renamed without credentials', updated_at: saved.updated_at }) assert.ok(Date.parse(saved.updated_at) > Date.parse(original.updated_at)) assertPublic(saved) - assert.deepEqual(credentials.get(NODE.id), stored) - assert.throws(() => repo.getSecrets(NODE.id)) + assert.deepEqual(await credentials.get(NODE.id), stored) + await assert.rejects(async () => await repo.getSecrets(NODE.id)) }) -test('label-only updates enforce CAS and advance versions when the clock moves backwards', (t) => { - const { repo } = setup(t) +test('label-only updates enforce CAS and advance versions when the clock moves backwards', async (t) => { + const { repo } = await setup(t) const timestamp = '2026-10-18T00:00:00.000Z' const initialMs = Date.parse(timestamp) - const original = repo.insert(NODE, timestamp) + const original = await repo.insert(NODE, timestamp) const clock = t.mock.method(Date, 'now', () => initialMs) - const first = repo.updateLabel(NODE.id, 'First rename', original.updated_at) + const first = await repo.updateLabel(NODE.id, 'First rename', original.updated_at) assert.equal(Date.parse(first.updated_at), initialMs + 1) - assert.equal(repo.updateLabel(NODE.id, 'Stale rename', original.updated_at), null) - assert.equal(repo.updateLabel('node-missing', 'Missing node', first.updated_at), null) - assert.equal(repo.updateLabel(NODE.id, 'Missing version', undefined), null) - assert.deepEqual(repo.get(NODE.id), first) + assert.equal(await repo.updateLabel(NODE.id, 'Stale rename', original.updated_at), null) + assert.equal(await repo.updateLabel('node-missing', 'Missing node', first.updated_at), null) + assert.equal(await repo.updateLabel(NODE.id, 'Missing version', undefined), null) + assert.deepEqual(await repo.get(NODE.id), first) clock.mock.mockImplementation(() => initialMs - 1000) - const second = repo.updateLabel(NODE.id, 'Second rename', first.updated_at) + const second = await repo.updateLabel(NODE.id, 'Second rename', first.updated_at) assert.equal(Date.parse(second.updated_at), initialMs + 2) - assert.equal(repo.getSecrets(NODE.id).password, NODE.password) + assert.equal((await repo.getSecrets(NODE.id)).password, NODE.password) }) -test('stale and missing node updates return null without overwriting a successful credential change', (t) => { - const { repo } = setup(t) - const original = repo.insert(NODE) - const saved = repo.update(original.id, { ...NODE, password: 'new-password' }, original.updated_at) +test('stale and missing node updates return null without overwriting a successful credential change', async (t) => { + const { repo } = await setup(t) + const original = await repo.insert(NODE) + const saved = await repo.update(original.id, { ...NODE, password: 'new-password' }, original.updated_at) assert.ok(saved) - assert.equal(repo.update(original.id, { ...NODE, label: 'Stale edit' }, original.updated_at), null) - assert.equal(repo.update('node-missing', NODE, original.updated_at), null) - assert.equal(repo.update(original.id, NODE, undefined), null) - assert.deepEqual(repo.get(original.id), saved) - assert.equal(repo.getSecrets(original.id).password, 'new-password') + assert.equal(await repo.update(original.id, { ...NODE, label: 'Stale edit' }, original.updated_at), null) + assert.equal(await repo.update('node-missing', NODE, original.updated_at), null) + assert.equal(await repo.update(original.id, NODE, undefined), null) + assert.deepEqual(await repo.get(original.id), saved) + assert.equal((await repo.getSecrets(original.id)).password, 'new-password') }) -test('each successful update advances its version even when the clock has not advanced or moves backwards', (t) => { - const { repo } = setup(t) +test('each successful update advances its version even when the clock has not advanced or moves backwards', async (t) => { + const { repo } = await setup(t) const timestamp = '2026-10-18T00:00:00.000Z' const initialMs = Date.parse(timestamp) - const original = repo.insert(NODE, timestamp) + const original = await repo.insert(NODE, timestamp) const clock = t.mock.method(Date, 'now', () => initialMs) - const first = repo.update(original.id, { ...NODE, label: 'First' }, original.updated_at) + const first = await repo.update(original.id, { ...NODE, label: 'First' }, original.updated_at) assert.equal(Date.parse(first.updated_at), initialMs + 1) clock.mock.mockImplementation(() => initialMs - 1000) - const second = repo.update(original.id, { ...NODE, label: 'Second' }, first.updated_at) + const second = await repo.update(original.id, { ...NODE, label: 'Second' }, first.updated_at) assert.equal(Date.parse(second.updated_at), initialMs + 2) clock.mock.mockImplementation(() => initialMs + 10_000) - const third = repo.update(original.id, { ...NODE, label: 'Third' }, second.updated_at) + const third = await repo.update(original.id, { ...NODE, label: 'Third' }, second.updated_at) assert.equal(Date.parse(third.updated_at), initialMs + 10_000) }) -test('switching to key authentication encrypts new secrets and removes the old password', (t) => { - const { db, repo } = setup(t, 'test-only-database-secret') - const original = repo.insert(NODE) - const saved = repo.update( +test('switching to key authentication encrypts new secrets and removes the old password', async (t) => { + const { db, repo } = await setup(t, 'test-only-database-secret') + const original = await repo.insert(NODE) + const saved = await repo.update( original.id, { ...NODE, auth_type: 'key', password: null, private_key: 'new-private-key', passphrase: 'new-passphrase' }, original.updated_at, ) assert.equal(saved.auth_type, 'key') assertPublic(saved) - const stored = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?').get(NODE.id) + const stored = await db + .prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + .get(NODE.id) assert.equal(stored.password, null) assert.match(stored.private_key, /^enc:v1:/) assert.match(stored.passphrase, /^enc:v1:/) - assert.deepEqual(repo.getSecrets(NODE.id), { + assert.deepEqual(await repo.getSecrets(NODE.id), { password: null, privateKey: 'new-private-key', passphrase: 'new-passphrase', }) }) -test('switching to password authentication clears old key secrets without requiring database encryption', (t) => { - const { db, repo } = setup(t) - const original = repo.insert({ +test('switching to password authentication clears old key secrets without requiring database encryption', async (t) => { + const { db, repo } = await setup(t) + const original = await repo.insert({ ...NODE, auth_type: 'key', password: null, private_key: 'old-private-key', passphrase: 'old-passphrase', }) - const saved = repo.update( + const saved = await repo.update( original.id, { ...NODE, host: 'new.example.test', port: 2222, username: 'new-operator', password: 'new-password' }, original.updated_at, @@ -178,23 +178,25 @@ test('switching to password authentication clears old key secrets without requir ['new.example.test', 2222, 'new-operator', 'password'], ) assertPublic(saved) - const stored = db.prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = ?').get(NODE.id) + const stored = await db + .prepare('SELECT password, private_key, passphrase FROM cluster_nodes WHERE id = $1') + .get(NODE.id) assert.deepEqual({ ...stored }, { password: 'new-password', private_key: null, passphrase: null }) }) -test('a duplicate endpoint rejects the whole update and preserves the previous version and credentials', (t) => { - const { repo } = setup(t) - const original = repo.insert(NODE) - repo.insert({ ...NODE, id: 'node-b', host: 'node-b.example.test' }) - assert.throws( - () => - repo.update( +test('a duplicate endpoint rejects the whole update and preserves the previous version and credentials', async (t) => { + const { repo } = await setup(t) + const original = await repo.insert(NODE) + await repo.insert({ ...NODE, id: 'node-b', host: 'node-b.example.test' }) + await assert.rejects( + async () => + await repo.update( NODE.id, { ...NODE, host: 'node-b.example.test', label: 'Conflict', password: 'uncommitted-password' }, original.updated_at, ), - /UNIQUE constraint failed/, + { code: '23505' }, ) - assert.deepEqual(repo.get(NODE.id), original) - assert.equal(repo.getSecrets(NODE.id).password, NODE.password) + assert.deepEqual(await repo.get(NODE.id), original) + assert.equal((await repo.getSecrets(NODE.id)).password, NODE.password) }) diff --git a/test/unit/panel-users.test.mjs b/test/unit/panel-users.test.mjs index e6de6afaa..7619a2127 100644 --- a/test/unit/panel-users.test.mjs +++ b/test/unit/panel-users.test.mjs @@ -3,12 +3,14 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { RedeemCodesRepo } from '../../src/lib/db/repos/redeem-codes-repo.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' import { PanelUserStore, hashPassword, verifyPassword, publicUserView } from '../../src/lib/admin/panel-users.mjs' -function tmpStore() { +async function tmpStore() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-panel-users-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() return { dir, db, store: new PanelUserStore({ db, projectRoot: dir }) } } @@ -20,82 +22,88 @@ test('scrypt hash verifies and is not reversible', () => { assert.doesNotMatch(hashed, /correct-horse/) }) -test('bootstrap seeds env admin once', () => { - const { store } = tmpStore() - const first = store.bootstrapFromEnv({ username: 'admin', password: 'testpass1' }) - const second = store.bootstrapFromEnv({ username: 'admin', password: 'otherpass' }) +test('bootstrap seeds env admin once', async () => { + const { store } = await tmpStore() + const first = await store.bootstrapFromEnv({ username: 'admin', password: 'testpass1' }) + const second = await store.bootstrapFromEnv({ username: 'admin', password: 'otherpass' }) assert.equal(first.username, 'admin') assert.equal(first.role, 'admin') assert.equal(second.id, first.id) - assert.ok(store.authenticate('admin', 'testpass1')) - assert.equal(store.authenticate('admin', 'otherpass'), null) + assert.ok(await store.authenticate('admin', 'testpass1')) + assert.equal(await store.authenticate('admin', 'otherpass'), null) }) -test('bootstrap imports legacy env passwords without relaxing user password validation', () => { - const { store } = tmpStore() +test('bootstrap imports legacy env passwords without relaxing user password validation', async () => { + const { store } = await tmpStore() for (const [username, password] of [ ['admin', '123456'], ['legacy', 'short'], ]) { - const admin = store.bootstrapFromEnv({ username, password }) + const admin = await store.bootstrapFromEnv({ username, password }) assert.equal(admin.role, 'admin') assert.equal(admin.enabled, true) assert.equal(admin.password_hash, undefined) - assert.ok(store.list().some((user) => user.id === admin.id)) - assert.ok(store.authenticate(username, password)) - assert.notEqual(store.getById(admin.id).password_hash, password) - assert.throws(() => store.update(admin.id, { password: 'short' }), /密码至少 8 位/) + assert.ok((await store.list()).some((user) => user.id === admin.id)) + assert.ok(await store.authenticate(username, password)) + assert.notEqual((await store.getById(admin.id)).password_hash, password) + await assert.rejects(async () => await store.update(admin.id, { password: 'short' }), /密码至少 8 位/) - store.update(admin.id, { password: 'changed-admin-pass' }) - const again = store.bootstrapFromEnv({ username, password }) + await store.update(admin.id, { password: 'changed-admin-pass' }) + const again = await store.bootstrapFromEnv({ username, password }) assert.equal(again.id, admin.id) - assert.equal(store.authenticate(username, password), null) - assert.ok(store.authenticate(username, 'changed-admin-pass')) + assert.equal(await store.authenticate(username, password), null) + assert.ok(await store.authenticate(username, 'changed-admin-pass')) } - assert.throws(() => store.create({ username: 'reader', password: '123456' }), /密码至少 8 位/) + await assert.rejects(async () => await store.create({ username: 'reader', password: '123456' }), /密码至少 8 位/) }) -test('bootstrap preserves disabled admins and validates usernames', () => { - const { store } = tmpStore() - store.create({ username: 'owner', password: 'owner-pass', role: 'admin' }) - const admin = store.create({ username: 'admin', password: 'admin-pass', role: 'admin', enabled: false }) - const imported = store.bootstrapFromEnv({ username: 'admin', password: '123456' }) +test('bootstrap preserves disabled admins and validates usernames', async () => { + const { store } = await tmpStore() + await store.create({ username: 'owner', password: 'owner-pass', role: 'admin' }) + const admin = await store.create({ username: 'admin', password: 'admin-pass', role: 'admin', enabled: false }) + const imported = await store.bootstrapFromEnv({ username: 'admin', password: '123456' }) assert.equal(imported.id, admin.id) assert.equal(imported.enabled, false) - assert.equal(store.authenticate('admin', '123456'), null) - assert.equal(store.bootstrapFromEnv({ username: 'empty', password: '' }), null) - assert.throws(() => store.bootstrapFromEnv({ username: '1bad', password: '123456' }), /用户名/) + assert.equal(await store.authenticate('admin', '123456'), null) + assert.equal(await store.bootstrapFromEnv({ username: 'empty', password: '' }), null) + await assert.rejects(async () => await store.bootstrapFromEnv({ username: '1bad', password: '123456' }), /用户名/) }) -test('create / authenticate / change password / roles', () => { - const { store } = tmpStore() - store.bootstrapFromEnv({ username: 'admin', password: 'adminpass' }) - const viewer = store.create({ username: 'reader', password: 'reader-pass', role: 'user' }) - const superu = store.create({ username: 'ops', password: 'super-pass', role: 'super' }) +test('create / authenticate / change password / roles', async () => { + const { store } = await tmpStore() + await store.bootstrapFromEnv({ username: 'admin', password: 'adminpass' }) + const viewer = await store.create({ username: 'reader', password: 'reader-pass', role: 'user' }) + const superu = await store.create({ username: 'ops', password: 'super-pass', role: 'super' }) assert.equal(viewer.role, 'user') assert.equal(superu.role, 'super') - assert.equal(store.authenticate('reader', 'reader-pass').role, 'user') - assert.equal(store.authenticate('ops', 'wrong-pass'), null) + assert.equal((await store.authenticate('reader', 'reader-pass')).role, 'user') + assert.equal(await store.authenticate('ops', 'wrong-pass'), null) - const updated = store.update(viewer.id, { password: 'reader-next' }) - assert.equal(store.authenticate('reader', 'reader-pass'), null) - assert.ok(store.authenticate('reader', 'reader-next')) + const updated = await store.update(viewer.id, { password: 'reader-next' }) + assert.equal(await store.authenticate('reader', 'reader-pass'), null) + assert.ok(await store.authenticate('reader', 'reader-next')) assert.equal(publicUserView(updated).password_hash, undefined) }) -test('cannot remove or demote the last admin', () => { - const { store } = tmpStore() - const admin = store.create({ username: 'admin', password: 'adminpass', role: 'admin' }) - assert.throws(() => store.update(admin.id, { role: 'user' }), /最后一个管理员/) - assert.throws(() => store.update(admin.id, { enabled: false }), /最后一个管理员/) - assert.throws(() => store.remove(admin.id), /最后一个管理员/) +test('cannot remove or demote the last admin', async () => { + const { store } = await tmpStore() + const admin = await store.create({ username: 'admin', password: 'adminpass', role: 'admin' }) + await assert.rejects(async () => await store.update(admin.id, { role: 'user' }), /最后一个管理员/) + await assert.rejects(async () => await store.update(admin.id, { enabled: false }), /最后一个管理员/) + await assert.rejects(async () => await store.remove(admin.id), /最后一个管理员/) }) -test('rejects bad username / short password / unknown role', () => { - const { store } = tmpStore() - assert.throws(() => store.create({ username: '1bad', password: 'longenough', role: 'user' }), /用户名/) - assert.throws(() => store.create({ username: 'okuser', password: 'short', role: 'user' }), /密码/) - assert.throws(() => store.create({ username: 'okuser', password: 'longenough', role: 'root' }), /角色/) +test('rejects bad username / short password / unknown role', async () => { + const { store } = await tmpStore() + await assert.rejects( + async () => await store.create({ username: '1bad', password: 'longenough', role: 'user' }), + /用户名/, + ) + await assert.rejects(async () => await store.create({ username: 'okuser', password: 'short', role: 'user' }), /密码/) + await assert.rejects( + async () => await store.create({ username: 'okuser', password: 'longenough', role: 'root' }), + /角色/, + ) }) function seedOwnedVm(project, id, owner, extra = {}) { @@ -115,21 +123,21 @@ function seedOwnedVm(project, id, owner, extra = {}) { return vm } -test('deleting a user reclaims assigned VMs while preserving private and unrelated slots', (t) => { - const { dir, db, store } = tmpStore() - t.after(() => { - db.close() +test('deleting a user reclaims assigned VMs while preserving private and unrelated slots', async (t) => { + const { dir, db, store } = await tmpStore() + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) - const user = store.create({ username: 'tenant', password: 'tenant-pass' }) + const user = await store.create({ username: 'tenant', password: 'tenant-pass' }) const assigned = seedOwnedVm(dir, 'assigned', user.id) seedOwnedVm(dir, 'legacy', user.id, { origin: undefined }) const privateVm = seedOwnedVm(dir, 'private', user.id, { origin: 'user_created' }) const otherVm = seedOwnedVm(dir, 'other', 'another-tenant') const publicVm = seedOwnedVm(dir, 'public', null, { origin: 'platform' }) - assert.equal(store.remove(user.id), true) - assert.equal(store.getById(user.id), null) + assert.equal(await store.remove(user.id), true) + assert.equal(await store.getById(user.id), null) const read = (id) => JSON.parse(fs.readFileSync(path.join(dir, 'vms', `${id}.json`), 'utf8')) const reclaimed = read('assigned') assert.equal(reclaimed.owner_user_id, null) @@ -144,37 +152,83 @@ test('deleting a user reclaims assigned VMs while preserving private and unrelat assert.deepEqual(read('public'), publicVm) }) -test('rejected user deletions leave assigned VMs with their owners', (t) => { - const { dir, db, store } = tmpStore() - t.after(() => { - db.close() +test('rejected user deletions leave assigned VMs with their owners', async (t) => { + const { dir, db, store } = await tmpStore() + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) - const admin = store.create({ username: 'admin', password: 'admin-pass', role: 'admin' }) - const user = store.create({ username: 'tenant', password: 'tenant-pass' }) + const admin = await store.create({ username: 'admin', password: 'admin-pass', role: 'admin' }) + const user = await store.create({ username: 'tenant', password: 'tenant-pass' }) const assigned = seedOwnedVm(dir, 'assigned', user.id) const adminVm = seedOwnedVm(dir, 'admin-vm', admin.id) - assert.throws(() => store.remove(user.id, { actorId: user.id }), /不能删除当前登录账号/) - assert.throws(() => store.remove(admin.id), /最后一个管理员/) - assert.equal(store.remove('missing-user'), false) + await assert.rejects(async () => await store.remove(user.id, { actorId: user.id }), /不能删除当前登录账号/) + await assert.rejects(async () => await store.remove(admin.id), /最后一个管理员/) + assert.equal(await store.remove('missing-user'), false) assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'vms', 'assigned.json'), 'utf8')), assigned) assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'vms', 'admin-vm.json'), 'utf8')), adminVm) }) -test('a VM write failure does not soft-delete the user', (t) => { - const { dir, db, store } = tmpStore() - t.after(() => { - db.close() +test('a VM write failure does not soft-delete the user', async (t) => { + const { dir, db, store } = await tmpStore() + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) - const user = store.create({ username: 'tenant', password: 'tenant-pass' }) + const user = await store.create({ username: 'tenant', password: 'tenant-pass' }) const assigned = seedOwnedVm(dir, 'assigned', user.id) t.mock.method(fs, 'renameSync', () => { throw new Error('simulated VM write failure') }) - assert.throws(() => store.remove(user.id), /simulated VM write failure/) - assert.ok(store.getById(user.id)) + await assert.rejects(async () => await store.remove(user.id), /simulated VM write failure/) + assert.ok(await store.getById(user.id)) assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'vms', 'assigned.json'), 'utf8')), assigned) }) + +test('concurrent redemption credits one balance and rolls back on a failed credit', async () => { + const { store, db } = await tmpStore() + const user = await store.create({ username: 'redeemer', password: 'long-password' }) + const codes = new RedeemCodesRepo(db) + const code = await codes.create({ value: 5 }) + const results = await Promise.all(Array.from({ length: 8 }, () => codes.redeem(code.code, user.id))) + assert.equal(results.filter((r) => r.ok).length, 1) + assert.equal((await store.getById(user.id)).balance, 5) + const failed = await codes.create({ value: 7 }) + codes.users.addBalance = async () => { + throw new Error('credit failed') + } + await assert.rejects(codes.redeem(failed.code, user.id), /credit failed/) + assert.equal((await codes.getByCode(failed.code)).status, 'unused') + assert.equal((await store.getById(user.id)).balance, 5) +}) + +test('concurrent admin demotions keep one enabled administrator', async () => { + const { store } = await tmpStore() + const a = await store.create({ username: 'admin_a', password: 'long-password', role: 'admin' }) + const b = await store.create({ username: 'admin_b', password: 'long-password', role: 'admin' }) + const results = await Promise.allSettled([store.update(a.id, { role: 'user' }), store.update(b.id, { role: 'user' })]) + assert.equal(results.filter((r) => r.status === 'fulfilled').length, 1) + assert.equal(await store.countEnabledAdmins(), 1) +}) + +test('settings cache changes only after a successful transaction commit', async () => { + const db = getDb() + const settings = new SettingsRepo(db) + await settings.set('transaction_setting', 'before') + await assert.rejects( + withTransaction(db, async () => { + await settings.set('transaction_setting', 'after') + assert.equal(db.settingsState.get('transaction_setting'), 'before') + throw new Error('rollback') + }), + /rollback/, + ) + assert.equal(await settings.get('transaction_setting'), 'before') + assert.equal(db.settingsState.get('transaction_setting'), 'before') + await withTransaction(db, async () => { + await settings.set('transaction_setting', 'committed') + }) + assert.equal(db.settingsState.get('transaction_setting'), 'committed') +}) diff --git a/test/unit/pool-runtime-repos.test.mjs b/test/unit/pool-runtime-repos.test.mjs index 942e00e58..bcbf0caa0 100644 --- a/test/unit/pool-runtime-repos.test.mjs +++ b/test/unit/pool-runtime-repos.test.mjs @@ -3,16 +3,16 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { AccountRuntimeRepo } from '../../src/lib/db/repos/account-runtime-repo.mjs' import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' -test('account runtime state persists account and model cooldowns', () => { +test('account runtime state persists account and model cooldowns', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) - repo.upsert({ + await repo.upsert({ account_id: 'account-1', vm_id: 'vm-01', status: 'ready', @@ -22,131 +22,131 @@ test('account runtime state persists account and model cooldowns', () => { worker_status: { ok: true }, }) const until = Date.now() + 60_000 - repo.markCooldown('account-1', { + await repo.markCooldown('account-1', { vmId: 'vm-01', model: 'claude-test', until, reason: 'model_rate_limit', }) - const state = repo.get('account-1') + const state = await repo.get('account-1') assert.equal(state.priority, 3) assert.equal(state.weight, 5) assert.equal(state.model_states['claude-test'].reason, 'model_rate_limit') assert.equal(state.model_states['claude-test'].cooldown_until, until) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('account cooldown mirrors structured rate-limit columns (sub2api style)', () => { +test('account cooldown mirrors structured rate-limit columns (sub2api style)', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) const until = Date.now() + 300_000 - repo.markCooldown('account-rl', { + await repo.markCooldown('account-rl', { vmId: 'vm-01', until, reason: 'account_quota_exhausted', }) - let state = repo.get('account-rl') + let state = await repo.get('account-rl') assert.equal(state.rate_limit_reset_at, until) assert.ok(state.rate_limited_at <= Date.now()) assert.equal(state.overload_until, null) - repo.markCooldown('account-rl', { vmId: 'vm-01', until: until + 1000, reason: 'provider_overloaded' }) - state = repo.get('account-rl') + await repo.markCooldown('account-rl', { vmId: 'vm-01', until: until + 1000, reason: 'provider_overloaded' }) + state = await repo.get('account-rl') assert.equal(state.overload_until, null) assert.equal(state.cooldown_until, until + 1000) assert.equal(state.cooldown_reason, 'provider_overloaded') assert.equal(state.rate_limit_reset_at, until) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('updateWindow persists session window without touching cooldowns', () => { +test('updateWindow persists session window without touching cooldowns', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) const end = Date.now() + 3600_000 - repo.updateWindow('account-w', { + await repo.updateWindow('account-w', { vmId: 'vm-02', sessionWindowStart: end - 5 * 3600_000, sessionWindowEnd: end, sessionWindowStatus: 'allowed_warning', }) - const state = repo.get('account-w') + const state = await repo.get('account-w') assert.equal(state.session_window_end, end) assert.equal(state.session_window_start, end - 5 * 3600_000) assert.equal(state.session_window_status, 'allowed_warning') assert.equal(state.cooldown_until, null) // partial update keeps prior fields - repo.updateWindow('account-w', { sessionWindowStatus: 'rejected' }) - assert.equal(repo.get('account-w').session_window_end, end) - assert.equal(repo.get('account-w').session_window_status, 'rejected') + await repo.updateWindow('account-w', { sessionWindowStatus: 'rejected' }) + assert.equal((await repo.get('account-w')).session_window_end, end) + assert.equal((await repo.get('account-w')).session_window_status, 'rejected') } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('clearAuthCooldown drops 401 leftover and keeps quota cools', () => { +test('clearAuthCooldown drops 401 leftover and keeps quota cools', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-auth-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new AccountRuntimeRepo(db) const until = Date.now() + 600_000 - repo.markCooldown('auth-1', { + await repo.markCooldown('auth-1', { vmId: 'vm-11', until, reason: 'authentication_failed_after_refresh', }) - repo.markCooldown('quota-1', { + await repo.markCooldown('quota-1', { vmId: 'vm-12', until, reason: 'account_quota_exhausted', }) - assert.equal(repo.clearAuthCooldown('auth-1', { vmId: 'vm-11' }), true) - assert.equal(repo.get('auth-1').cooldown_reason, null) - assert.equal(repo.get('auth-1').cooldown_until, null) - assert.equal(repo.clearAuthCooldown('quota-1', { vmId: 'vm-12' }), false) - assert.equal(repo.get('quota-1').cooldown_reason, 'account_quota_exhausted') - repo.markCooldown('revoked-1', { + assert.equal(await repo.clearAuthCooldown('auth-1', { vmId: 'vm-11' }), true) + assert.equal((await repo.get('auth-1')).cooldown_reason, null) + assert.equal((await repo.get('auth-1')).cooldown_until, null) + assert.equal(await repo.clearAuthCooldown('quota-1', { vmId: 'vm-12' }), false) + assert.equal((await repo.get('quota-1')).cooldown_reason, 'account_quota_exhausted') + await repo.markCooldown('revoked-1', { vmId: 'vm-100', until: Number.MAX_SAFE_INTEGER, reason: 'oauth_revoked', status: 'disabled', }) - assert.equal(repo.clearAuthCooldown('revoked-1', { vmId: 'vm-100' }), false) - assert.equal(repo.get('revoked-1').cooldown_reason, 'oauth_revoked') - assert.equal(repo.clearGrantRevokeCooldown('revoked-1', { vmId: 'vm-100' }), true) - assert.equal(repo.get('revoked-1').cooldown_reason, null) - assert.equal(repo.get('revoked-1').status, 'ready') - repo.markCooldown('setup-1', { + assert.equal(await repo.clearAuthCooldown('revoked-1', { vmId: 'vm-100' }), false) + assert.equal((await repo.get('revoked-1')).cooldown_reason, 'oauth_revoked') + assert.equal(await repo.clearGrantRevokeCooldown('revoked-1', { vmId: 'vm-100' }), true) + assert.equal((await repo.get('revoked-1')).cooldown_reason, null) + assert.equal((await repo.get('revoked-1')).status, 'ready') + await repo.markCooldown('setup-1', { vmId: 'vm-05', until: Number.MAX_SAFE_INTEGER, reason: 'oauth_no_refresh', status: 'disabled', }) - assert.equal(repo.clearGrantRevokeCooldown('setup-1', { vmId: 'vm-05' }), true) - assert.equal(repo.get('setup-1').cooldown_reason, null) - assert.equal(repo.get('setup-1').status, 'ready') + assert.equal(await repo.clearGrantRevokeCooldown('setup-1', { vmId: 'vm-05' }), true) + assert.equal((await repo.get('setup-1')).cooldown_reason, null) + assert.equal((await repo.get('setup-1')).status, 'ready') } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('request attempt ledger records final state and commit boundary', () => { +test('request attempt ledger records final state and commit boundary', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-attempt-repo-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RequestAttemptsRepo(db) - repo.begin({ + await repo.begin({ requestId: 'req-1', attemptNo: 1, vmId: 'vm-01', @@ -155,7 +155,7 @@ test('request attempt ledger records final state and commit boundary', () => { selectionReason: 'balanced', waitMs: 4, }) - repo.complete('req-1', 1, { + await repo.complete('req-1', 1, { upstreamStatus: 429, errorScope: 'account', action: 'continue-and-cooldown', @@ -163,7 +163,7 @@ test('request attempt ledger records final state and commit boundary', () => { terminalState: 'rejected', latencyMs: 12, }) - repo.begin({ + await repo.begin({ requestId: 'req-1', attemptNo: 2, vmId: 'vm-02', @@ -171,7 +171,7 @@ test('request attempt ledger records final state and commit boundary', () => { model: 'claude-test', selectionReason: 'failover', }) - repo.complete('req-1', 2, { + await repo.complete('req-1', 2, { upstreamStatus: 200, downstreamCommitted: true, terminalState: 'verified', @@ -179,14 +179,14 @@ test('request attempt ledger records final state and commit boundary', () => { ttftMs: 20, latencyMs: 40, }) - const attempts = repo.list('req-1') + const attempts = await repo.list('req-1') assert.equal(attempts.length, 2) assert.equal(attempts[0].action, 'continue-and-cooldown') assert.equal(attempts[1].terminal_state, 'verified') assert.equal(attempts[1].downstream_committed, true) assert.deepEqual(attempts[1].usage, { input_tokens: 2, output_tokens: 3 }) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) diff --git a/test/unit/refusal-guard.test.mjs b/test/unit/refusal-guard.test.mjs index c7e85aefe..402825467 100644 --- a/test/unit/refusal-guard.test.mjs +++ b/test/unit/refusal-guard.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { RefusalGuardsRepo } from '../../src/lib/db/repos/refusal-guards-repo.mjs' import { REFUSAL_GUARD_MESSAGE, @@ -88,41 +88,41 @@ test('zero inject still allows distill skip while refusal fingerprint is indepen assert.ok(refusalFingerprint(inbound)) }) -test('remember then lookup hits the same prompt', () => { +test('remember then lookup hits the same prompt', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-refusal-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RefusalGuardsRepo(db) const req = body('Return exactly CACHE_30_5m_OK.') const fp = refusalFingerprint(req) - assert.equal(repo.get(fp), null) - repo.remember({ + assert.equal(await repo.get(fp), null) + await repo.remember({ fingerprint: fp, model: req.model, requestId: 'req-1', errorMessage: 'violate our Usage Policy', preview: 'Return exactly CACHE_30_5m_OK.', }) - const hit = repo.get(fp) + const hit = await repo.get(fp) assert.ok(hit) assert.equal(hit.hit_count, 0) - const bumped = repo.hit(fp) + const bumped = await repo.hit(fp) assert.equal(bumped.hit_count, 1) const other = refusalFingerprint(body('Return exactly CACHE_30_5m_OK.', { model: 'claude-opus-4-8' })) - assert.equal(repo.get(other), null) + assert.equal(await repo.get(other), null) } finally { - db.close() + await db.close() } }) -test('timed refusal expires and no longer blocks', () => { +test('timed refusal expires and no longer blocks', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-refusal-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RefusalGuardsRepo(db) const req = body('pause this 500') const fp = refusalFingerprint(req) - repo.remember({ + await repo.remember({ fingerprint: fp, model: req.model, requestId: 'req-500', @@ -130,10 +130,10 @@ test('timed refusal expires and no longer blocks', () => { preview: 'pause this 500', expiresAt: new Date(Date.now() - 1000).toISOString(), }) - assert.equal(repo.get(fp), null) - assert.equal(repo.list().length, 0) + assert.equal(await repo.get(fp), null) + assert.equal((await repo.list()).length, 0) } finally { - db.close() + await db.close() } }) test('guard error is a non-retryable HTTP 400 request rejection, not a login error', () => { @@ -170,27 +170,27 @@ test('guard error is a non-retryable HTTP 400 request rejection, not a login err }) }) -test('list remove and clear', () => { +test('list remove and clear', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-refusal-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const repo = new RefusalGuardsRepo(db) const a = refusalFingerprint(body('A')) const b = refusalFingerprint(body('B')) - repo.remember({ fingerprint: a, model: 'claude-opus-5', preview: 'A' }) - repo.remember({ fingerprint: b, model: 'claude-opus-5', preview: 'B' }) - assert.equal(repo.count(), 2) - assert.equal(repo.list().length, 2) - assert.equal(repo.remove(a), true) - assert.equal(repo.count(), 1) - assert.equal(repo.clear(), 1) - assert.equal(repo.count(), 0) + await repo.remember({ fingerprint: a, model: 'claude-opus-5', preview: 'A' }) + await repo.remember({ fingerprint: b, model: 'claude-opus-5', preview: 'B' }) + assert.equal(await repo.count(), 2) + assert.equal((await repo.list()).length, 2) + assert.equal(await repo.remove(a), true) + assert.equal(await repo.count(), 1) + assert.equal(await repo.clear(), 1) + assert.equal(await repo.count(), 0) } finally { - db.close() + await db.close() } }) -test('panel setting and env can disable the guard', () => { +test('panel setting and env can disable the guard', async () => { const settings = new Map([[REFUSAL_GUARD_SETTING, true]]) const read = (key, fb) => (settings.has(key) ? settings.get(key) : fb) assert.equal(isRefusalGuardEnabled(read), true) diff --git a/test/unit/refusal-similarity.test.mjs b/test/unit/refusal-similarity.test.mjs index 8295bd6c4..dee7e5c23 100644 --- a/test/unit/refusal-similarity.test.mjs +++ b/test/unit/refusal-similarity.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { RefusalGuardsRepo } from '../../src/lib/db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../../src/lib/db/repos/refusal-device-blocks-repo.mjs' import { @@ -99,9 +99,9 @@ test('a Responses prompt with a small tail edit stays above the similarity line' assert.ok(nearScore >= REFUSAL_SIMILARITY, `near ${nearScore}`) }) -test('stored near-duplicate blocks and then the device blocks any later prompt', () => { +test('stored near-duplicate blocks and then the device blocks any later prompt', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-near-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const guards = new RefusalGuardsRepo(db) const devices = new RefusalDeviceBlocksRepo(db) @@ -109,7 +109,7 @@ test('stored near-duplicate blocks and then the device blocks any later prompt', metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-1' } }, }) const fp = refusalFingerprint(original) - guards.remember({ + await guards.remember({ fingerprint: fp, model: original.model, signature: refusalPromptSignature(original), @@ -119,41 +119,41 @@ test('stored near-duplicate blocks and then the device blocks any later prompt', metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-1' } }, }) assert.notEqual(refusalFingerprint(edited), fp) - const similar = matchStoredRefusal({ inbound: edited, body: edited, repo: guards, devices }) + const similar = await matchStoredRefusal({ inbound: edited, body: edited, repo: guards, devices }) assert.equal(similar.kind, 'similar') assert.ok(similar.score >= REFUSAL_SIMILARITY) - devices.block({ deviceId: similar.deviceId, fingerprint: similar.fingerprint, reason: 'refusal_similar' }) + await devices.block({ deviceId: similar.deviceId, fingerprint: similar.fingerprint, reason: 'refusal_similar' }) const other = body('completely different short ask', { metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-2' } }, }) - const banned = matchStoredRefusal({ inbound: other, body: other, repo: guards, devices }) + const banned = await matchStoredRefusal({ inbound: other, body: other, repo: guards, devices }) assert.equal(banned.kind, 'device') assert.equal(inboundRefusalDeviceId({ inbound: { device_id: 'short' } }), '') - assert.equal(devices.block({ deviceId: '' }), null) + assert.equal(await devices.block({ deviceId: '' }), null) } finally { - db.close() + await db.close() } }) -test('different user text under the same system is not a near match', () => { +test('different user text under the same system is not a near match', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-near-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const guards = new RefusalGuardsRepo(db) const stored = body(longUser('payload-a')) - guards.remember({ + await guards.remember({ fingerprint: refusalFingerprint(stored), model: stored.model, signature: refusalPromptSignature(stored), }) const other = body(longUser('unrelated-question-about-refactoring-the-scheduler')) - assert.equal(matchStoredRefusal({ inbound: other, body: other, repo: guards, devices: null }), null) + assert.equal(await matchStoredRefusal({ inbound: other, body: other, repo: guards, devices: null }), null) } finally { - db.close() + await db.close() } }) -test('panel policy defaults on at 90 and rejects an unknown threshold', () => { +test('panel policy defaults on at 90 and rejects an unknown threshold', async () => { assert.deepEqual(refusalGuardPolicy(), { enabled: true, similarity_enabled: true, @@ -165,8 +165,8 @@ test('panel policy defaults on at 90 and rejects an unknown threshold', () => { get: (key, fallback) => (settings.has(key) ? settings.get(key) : fallback), set: (key, value) => settings.set(key, value), } - assert.equal(applyRefusalGuardPatch(store, { similarity: 70 }).ok, false) - assert.equal(applyRefusalGuardPatch(store, { similarity: 85, similarity_enabled: false }).ok, true) + assert.equal((await applyRefusalGuardPatch(store, { similarity: 70 })).ok, false) + assert.equal((await applyRefusalGuardPatch(store, { similarity: 85, similarity_enabled: false })).ok, true) assert.equal(normalizeRefusalSimilarity(85), 85) assert.deepEqual( refusalGuardPolicy((key, fallback) => store.get(key, fallback)), @@ -179,26 +179,26 @@ test('panel policy defaults on at 90 and rejects an unknown threshold', () => { ) }) -test('turning similarity or device block off skips that check', () => { +test('turning similarity or device block off skips that check', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-near-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() try { const guards = new RefusalGuardsRepo(db) const devices = new RefusalDeviceBlocksRepo(db) const stored = body(longUser('payload'), { metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-1' } }, }) - guards.remember({ + await guards.remember({ fingerprint: refusalFingerprint(stored), model: stored.model, signature: refusalPromptSignature(stored), }) - devices.block({ deviceId: 'device-aaaabbbbcccc', reason: 'refusal_guard' }) + await devices.block({ deviceId: 'device-aaaabbbbcccc', reason: 'refusal_guard' }) const edited = body(longUser('payload').slice(0, -20) + 'tail-edit-xx', { metadata: { user_id: { device_id: 'device-aaaabbbbcccc', session_id: 'sess-9' } }, }) assert.equal( - matchStoredRefusal({ + await matchStoredRefusal({ inbound: edited, body: edited, repo: guards, @@ -209,6 +209,6 @@ test('turning similarity or device block off skips that check', () => { null, ) } finally { - db.close() + await db.close() } }) diff --git a/test/unit/tenant-billing.test.mjs b/test/unit/tenant-billing.test.mjs index 96b15ea29..3f2f0b32c 100644 --- a/test/unit/tenant-billing.test.mjs +++ b/test/unit/tenant-billing.test.mjs @@ -3,12 +3,12 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' -function tmpRepo() { +async function tmpRepo() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-billing-')) - const db = createDatabase({ dataDir: dir }) + const db = await getDb() return { dir, db, logs: new UsageLogsRepo(db) } } @@ -28,40 +28,44 @@ function row(partial) { } } -test('ownerBilling and query hide other tenants', () => { - const { logs, db } = tmpRepo() - db.prepare( - "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-a','a','{}','user-a','user_created')", - ).run() - db.prepare( - "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-b','b','{}','user-b','user_created')", - ).run() - logs.insertSummary( +test('ownerBilling and query hide other tenants', async () => { + const { logs, db } = await tmpRepo() + await db + .prepare( + "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-a','a','{}','user-a','user_created')", + ) + .run() + await db + .prepare( + "INSERT INTO vms (id, name, vm_json, owner_user_id, origin) VALUES ('vm-b','b','{}','user-b','user_created')", + ) + .run() + await logs.insertSummary( row({ id: 'l1', request_id: 'r1', vm_id: 'vm-a', user_id: 'user-a', total_cost: 1, actual_cost: 1 }), ) - logs.insertSummary( + await logs.insertSummary( row({ id: 'l2', request_id: 'r2', vm_id: 'vm-b', user_id: 'user-b', total_cost: 9, actual_cost: 9, status: 500 }), ) - const billed = logs.ownerBilling({ ownerUserId: 'user-a', groupBy: 'vm' }) + const billed = await logs.ownerBilling({ ownerUserId: 'user-a', groupBy: 'vm' }) assert.equal(billed.totals.requests, 1) assert.equal(billed.items.length, 1) assert.equal(billed.items[0].vm_id, 'vm-a') assert.equal(billed.items[0].cost_usd, 1) assert.equal(billed.items[0].official_cost_usd, 1) - const listed = logs.query({ owner_user_id: 'user-a' }) + const listed = await logs.query({ owner_user_id: 'user-a' }) assert.equal(listed.total, 1) assert.equal(listed.items[0].request_id, 'r1') - assert.equal(logs.belongsToOwner('r1', 'user-a'), true) - assert.equal(logs.belongsToOwner('r2', 'user-a'), false) + assert.equal(await logs.belongsToOwner('r1', 'user-a'), true) + assert.equal(await logs.belongsToOwner('r2', 'user-a'), false) }) -test('ownerBilling exposes official cost next to rate-multiplied cost', () => { - const { logs } = tmpRepo() - logs.insertSummary(row({ id: 'm1', request_id: 'rm1', vm_id: 'vm-x', total_cost: 1, actual_cost: 2 })) - const billed = logs.ownerBilling({ groupBy: 'vm' }) +test('ownerBilling exposes official cost next to rate-multiplied cost', async () => { + const { logs } = await tmpRepo() + await logs.insertSummary(row({ id: 'm1', request_id: 'rm1', vm_id: 'vm-x', total_cost: 1, actual_cost: 2 })) + const billed = await logs.ownerBilling({ groupBy: 'vm' }) assert.equal(billed.totals.cost_usd, 2) assert.equal(billed.totals.official_cost_usd, 1) assert.equal(billed.items[0].cost_usd, 2) diff --git a/test/unit/vm-pools.test.mjs b/test/unit/vm-pools.test.mjs index 48641dcd2..622b415d9 100644 --- a/test/unit/vm-pools.test.mjs +++ b/test/unit/vm-pools.test.mjs @@ -16,49 +16,55 @@ function open() { return { dir, store, repo: new VmPoolsRepo(store.db) } } -test('pool membership is shared by keys and applied on the next read', () => { +test('pool membership is shared by keys and applied on the next read', async () => { const { dir, store, repo } = open() try { - const pool = createVmPool(repo, { name: 'Pool A', vm_ids: ['vm-01', 'vm-02'] }, vms) - const key = store.create({ name: 'a', vm_pool_id: pool.id }) - const other = store.create({ name: 'b', vm_pool_id: pool.id }) + const pool = await createVmPool(repo, { name: 'Pool A', vm_ids: ['vm-01', 'vm-02'] }, vms) + const key = await store.create({ name: 'a', vm_pool_id: pool.id }) + const other = await store.create({ name: 'b', vm_pool_id: pool.id }) assert.equal(key.vm_pool_id, pool.id) assert.equal(other.vm_pool_id, pool.id) - const before = applyVmPool(keyScopeFromRecord(store.getById(key.id)), (id) => repo.get(id)) + const before = await applyVmPool(keyScopeFromRecord(await store.getById(key.id)), async (id) => await repo.get(id)) assert.deepEqual(before.allowed_vms, ['vm-01', 'vm-02']) - updateVmPool(repo, pool.id, { vm_ids: ['vm-03'] }, vms) - const after = applyVmPool(keyScopeFromRecord(store.getById(other.id)), (id) => repo.get(id)) + await updateVmPool(repo, pool.id, { vm_ids: ['vm-03'] }, vms) + const after = await applyVmPool(keyScopeFromRecord(await store.getById(other.id)), async (id) => await repo.get(id)) assert.deepEqual(after.allowed_vms, ['vm-03']) assert.equal(after.vm_pool_error, null) - assert.throws(() => createVmPool(repo, { name: 'Pool B', vm_ids: ['vm-03'] }, vms), /其他账号池/) + await assert.rejects(async () => await createVmPool(repo, { name: 'Pool B', vm_ids: ['vm-03'] }, vms), /其他账号池/) assert.equal( - repo.list().some((item) => item.name === 'Pool B'), + (await repo.list()).some((item) => item.name === 'Pool B'), false, ) - updateVmPool(repo, pool.id, { enabled: false }, vms) - const disabled = applyVmPool(keyScopeFromRecord(store.getById(key.id)), (id) => repo.get(id)) + await updateVmPool(repo, pool.id, { enabled: false }, vms) + const disabled = await applyVmPool( + keyScopeFromRecord(await store.getById(key.id)), + async (id) => await repo.get(id), + ) assert.equal(disabled.vm_pool_error, 'disabled') assert.deepEqual(disabled.allowed_vms, []) - assert.equal(deleteVmPool(repo, pool.id).error, 'vm_pool_in_use') - store.update(key.id, { vm_pool_id: null }) - store.update(other.id, { vm_pool_id: null }) - assert.equal(deleteVmPool(repo, pool.id).ok, true) - const gone = applyVmPool(keyScopeFromRecord({ vm_pool_id: pool.id, group_type: 'all' }), (id) => repo.get(id)) + assert.equal((await deleteVmPool(repo, pool.id)).error, 'vm_pool_in_use') + await store.update(key.id, { vm_pool_id: null }) + await store.update(other.id, { vm_pool_id: null }) + assert.equal((await deleteVmPool(repo, pool.id)).ok, true) + const gone = await applyVmPool( + keyScopeFromRecord({ vm_pool_id: pool.id, group_type: 'all' }), + async (id) => await repo.get(id), + ) assert.equal(gone.vm_pool_error, 'missing') } finally { fs.rmSync(dir, { recursive: true, force: true }) } }) -test('an empty pool stays empty instead of meaning every VM', () => { +test('an empty pool stays empty instead of meaning every VM', async () => { const { dir, store, repo } = open() try { - const pool = createVmPool(repo, { name: 'Empty' }, vms) - const key = store.create({ name: 'e', vm_pool_id: pool.id }) - const scope = applyVmPool(keyScopeFromRecord(store.getById(key.id)), (id) => repo.get(id)) + const pool = await createVmPool(repo, { name: 'Empty' }, vms) + const key = await store.create({ name: 'e', vm_pool_id: pool.id }) + const scope = await applyVmPool(keyScopeFromRecord(await store.getById(key.id)), async (id) => await repo.get(id)) assert.equal(scope.vm_pool_error, 'empty') assert.deepEqual(scope.allowed_vms, []) - assert.equal(store.list()[0].vm_pool_name, 'Empty') + assert.equal((await store.list())[0].vm_pool_name, 'Empty') } finally { fs.rmSync(dir, { recursive: true, force: true }) } From 893cb3cea41e7277f00610c124b302ceb15cc3ac Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:31:14 +0800 Subject: [PATCH 005/473] feat(db): migrate usage logging statistics and billing queries --- src/lib/admin/error-class.mjs | 8 +- src/lib/admin/request-log.mjs | 96 ++--- src/lib/db/repos/statistics-repo.mjs | 37 +- src/lib/db/repos/usage-log-preds.mjs | 8 +- src/lib/db/repos/usage-logs-repo.mjs | 535 ++++++++++++------------ src/lib/db/repos/usage-logs-view.mjs | 220 +++++----- test/unit/error-class.test.mjs | 19 +- test/unit/panel-billing-lookup.test.mjs | 12 +- test/unit/request-log.test.mjs | 484 +++++++++++---------- test/unit/request-logs-export.test.mjs | 48 ++- test/unit/statistics-repo.test.mjs | 133 +++--- test/unit/usage-logs-repo.test.mjs | 230 +++++----- test/unit/usage-logs-view.test.mjs | 249 +++++------ 13 files changed, 1067 insertions(+), 1012 deletions(-) diff --git a/src/lib/admin/error-class.mjs b/src/lib/admin/error-class.mjs index dfef4267f..f74f56157 100644 --- a/src/lib/admin/error-class.mjs +++ b/src/lib/admin/error-class.mjs @@ -161,11 +161,11 @@ export function ingressAuthSql() { const codes = [...INGRESS_AUTH_ERROR_CODES].map((code) => `'${code}'`).join(', ') const cred = "'upstream_auth_error','oauth_need_reimport','oauth_refresh_failed'" return `( - IFNULL(error_code, '') IN (${codes}) + COALESCE(error_code, '') IN (${codes}) OR ( status IN (401, 403) - AND IFNULL(vm_id, '') = '' - AND IFNULL(error_code, '') NOT IN (${cred}) + AND COALESCE(vm_id, '') = '' + AND COALESCE(error_code, '') NOT IN (${cred}) ) )` } @@ -180,7 +180,7 @@ function mutedAuthSql() { .filter((code) => CODE_MAP[code] !== 'auth') .map((code) => `'${code}'`) .join(',') - return `(${ingressAuthSql()} AND IFNULL(error_code, '') NOT IN (${classified}))` + return `(${ingressAuthSql()} AND COALESCE(error_code, '') NOT IN (${classified}))` } export function excludeErrorClassSql(classes) { diff --git a/src/lib/admin/request-log.mjs b/src/lib/admin/request-log.mjs index 512ee62c5..4216c999b 100644 --- a/src/lib/admin/request-log.mjs +++ b/src/lib/admin/request-log.mjs @@ -1,5 +1,5 @@ /** - * Request logging — sub2api UsageLog + access-log inspired (SQLite-backed). + * Request logging — sub2api UsageLog + access-log inspired (PostgreSQL-backed). * * Modes: * off — no writes @@ -390,7 +390,7 @@ export class RequestLogStore { this._cleanupTimer = null } - setConfig({ mode, retainDays, debugRetainDays, maxMb, mutedErrorClasses } = {}) { + async setConfig({ mode, retainDays, debugRetainDays, maxMb, mutedErrorClasses } = {}) { if (mode != null) { const m = String(mode).trim().toLowerCase() if (MODES.has(m)) this.mode = m @@ -411,7 +411,7 @@ export class RequestLogStore { if (mutedErrorClasses !== undefined) { this.mutedErrorClasses = resolveMutedErrorClasses(mutedErrorClasses) } - return this.snapshot() + return await this.snapshot() } /** Kept for API compat + post-restore hook (state lives in DB). */ @@ -499,7 +499,7 @@ export class RequestLogStore { * Write normal summary always (unless off). * Write debug full record when mode===debug. */ - finish(ctx, extra = {}) { + async finish(ctx, extra = {}) { if (!ctx || ctx.mode === 'off') return null const duration_ms = Math.max(0, Date.now() - (ctx.t0 || Date.now())) const usage = normalizeUsage(disjointOpenaiUsage(extra.usage || {}) || {}) @@ -570,7 +570,7 @@ export class RequestLogStore { intercept: extra.intercept ? String(extra.intercept).slice(0, 400) : null, // sub2api ownership + billing columns user_id: extra.user_id ?? null, - vm_owner_user_id: this.repo.vmOwnerOf(extra.vm_id), + vm_owner_user_id: await this.repo.vmOwnerOf(extra.vm_id), group_id: extra.group_id ?? 1, rate_multiplier: Number.isFinite(Number(extra.rate_multiplier)) ? Number(extra.rate_multiplier) : 1, ...costColumnsForLog(billedUsage, extra.upstream_model || extra.model || extra.requested_model, extra), @@ -582,9 +582,7 @@ export class RequestLogStore { const classified = classifyRequestError(summary) if (classified?.error_class) summary.error_class = classified.error_class - try { - this.repo.insertSummaryIfAbsent(summary) - } catch {} + await this.repo.insertSummaryIfAbsent(summary) if (this.jsonlMirror) this._appendJsonl(summary) this._mem.push(summary) if (this._mem.length > this._memMax) this._mem = this._mem.slice(-this._memMax) @@ -615,9 +613,7 @@ export class RequestLogStore { : null, response: ctx.response ? debugResponse(ctx.response) : null, } - try { - this.repo.insertDebug(summary.request_id || summary.id, summary.ts, debugRec) - } catch {} + await this.repo.insertDebug(summary.request_id || summary.id, summary.ts, debugRec) } return summary } @@ -651,66 +647,66 @@ export class RequestLogStore { return (items || []).filter((row) => !row.error_class || !blocked.has(row.error_class)) } - listNormal({ limit = 50, ...filters } = {}) { - return this.queryNormal({ limit, ...filters }).items + async listNormal({ limit = 50, ...filters } = {}) { + return (await this.queryNormal({ limit, ...filters })).items } /** Filtered + paginated query with total (panel). Default hides muted classes. */ - queryNormal(opts = {}) { + async queryNormal(opts = {}) { const exclude = this._mutedExclude(opts) - const result = this.repo.query({ ...opts, exclude_error_class: exclude }) + const result = await this.repo.query({ ...opts, exclude_error_class: exclude }) result.items = this._dropMuted(result.items, exclude) return result } - exportRows(opts = {}) { + async exportRows(opts = {}) { const limit = Math.max(1, Math.min(5000, Number(opts.limit) || 2000)) - return this.queryNormal({ ...opts, limit, offset: 0, maxLimit: 5000 }) + return await this.queryNormal({ ...opts, limit, offset: 0, maxLimit: 5000 }) } /** Aggregated stats for charts. */ - aggregate(opts = {}) { - return this.repo.aggregate(opts) + async aggregate(opts = {}) { + return await this.repo.aggregate(opts) } - totals() { - return this.repo.totals() + async totals() { + return await this.repo.totals() } - hasSuccessfulRequest(opts = {}) { - return this.repo.hasSuccessfulRequest(opts) + async hasSuccessfulRequest(opts = {}) { + return await this.repo.hasSuccessfulRequest(opts) } - billingStats(opts = {}) { - return this.repo.billingStats(opts) + async billingStats(opts = {}) { + return await this.repo.billingStats(opts) } - ownerBilling(opts = {}) { - return this.repo.ownerBilling(opts) + async ownerBilling(opts = {}) { + return await this.repo.ownerBilling(opts) } - costByModel(opts = {}) { - return this.repo.costByModel(opts) + async costByModel(opts = {}) { + return await this.repo.costByModel(opts) } - vmUsageStats(opts = {}) { - return this.repo.vmUsageStats(opts) + async vmUsageStats(opts = {}) { + return await this.repo.vmUsageStats(opts) } - keyUsageStats(opts = {}) { - return this.repo.keyUsageStats(opts) + async keyUsageStats(opts = {}) { + return await this.repo.keyUsageStats(opts) } /** Windowed SLA / QPS / TTFT snapshot for overview + log analysis. */ - windowStats(opts = {}) { - return this.repo.windowStats(opts) + async windowStats(opts = {}) { + return await this.repo.windowStats(opts) } - protocolEntryStats(opts = {}) { - return this.repo.protocolEntryStats(opts) + async protocolEntryStats(opts = {}) { + return await this.repo.protocolEntryStats(opts) } - listDebug({ + async listDebug({ limit = 20, exclude_error_class = null, include_muted = false, @@ -719,20 +715,20 @@ export class RequestLogStore { } = {}) { const exclude = this._mutedExclude({ exclude_error_class, include_muted, error_class }) const n = Math.max(1, Math.min(100, Number(limit) || 20)) - const pool = this.repo.listDebug({ limit: exclude ? Math.min(200, n * 15) : n, owner_user_id }) + const pool = await this.repo.listDebug({ limit: exclude ? Math.min(200, n * 15) : n, owner_user_id }) const items = error_class ? pool.filter((row) => row.error_class === error_class) : this._dropMuted(pool, exclude) return items.slice(0, n) } - getDebug(requestId, { owner_user_id = null } = {}) { + async getDebug(requestId, { owner_user_id = null } = {}) { if (!requestId) return null - const rec = this.repo.getDebug(requestId) || this.repo.getByRequestId(requestId) + const rec = (await this.repo.getDebug(requestId)) || (await this.repo.getByRequestId(requestId)) if (!rec) return null - if (owner_user_id && !this.repo.belongsToOwner(requestId, owner_user_id)) return null + if (owner_user_id && !(await this.repo.belongsToOwner(requestId, owner_user_id))) return null return rec } - snapshot() { + async snapshot() { return { mode: this.mode, retain_days: this.retainDays, @@ -740,11 +736,11 @@ export class RequestLogStore { max_mb: this.maxMb, muted_error_classes: this.mutedErrorClasses.slice(), recent_normal: this._mem.length, - backend: 'sqlite', + backend: 'postgresql', jsonl_mirror: this.jsonlMirror, - total_rows: (() => { + total_rows: await (async () => { try { - return this.repo.count() + return await this.repo.count() } catch { return null } @@ -753,9 +749,9 @@ export class RequestLogStore { } /** Retention: delete rows older than retain windows, then enforce max_mb. */ - cleanup() { + async cleanup() { try { - this.repo.cleanup({ + await this.repo.cleanup({ retainDays: this.retainDays, debugRetainDays: this.debugRetainDays, maxBytes: this.maxMb > 0 ? this.maxMb * 1024 * 1024 : 0, @@ -768,9 +764,9 @@ export class RequestLogStore { startCleanupScheduler({ intervalMs = CLEANUP_INTERVAL_MS } = {}) { this.stopCleanupScheduler() const ms = Math.max(60_000, Number(intervalMs) || CLEANUP_INTERVAL_MS) - const run = () => { + const run = async () => { try { - this.cleanup() + await this.cleanup() } catch {} } this._cleanupTimer = setInterval(run, ms) diff --git a/src/lib/db/repos/statistics-repo.mjs b/src/lib/db/repos/statistics-repo.mjs index 707e9127d..1795f0f01 100644 --- a/src/lib/db/repos/statistics-repo.mjs +++ b/src/lib/db/repos/statistics-repo.mjs @@ -34,7 +34,8 @@ const DIMENSION_COLUMN = { user: 'user_id', key: 'api_key_id', model: 'model', v const TOKENS_SQL = TOTAL_TOKENS_SQL /** created_at (`YYYY-MM-DDTHH:MM:SS.sssZ`) floored to its 15-minute UTC slot. */ -const SLOT_SQL = "substr(created_at, 1, 14) || printf('%02d', (CAST(substr(created_at, 15, 2) AS INTEGER) / 15) * 15)" +const SLOT_SQL = + "to_char(date_bin(interval '15 minutes', created_at::timestamptz, timestamptz '2000-01-01 00:00:00+00') AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI')" function num(v) { const n = Number(v) @@ -104,9 +105,9 @@ export class StatisticsRepo { } _scope(sinceMs, untilMs, ownerUserId) { - const where = ['created_at >= ?', 'created_at < ?'] + const where = ['created_at >= $1', 'created_at < $2'] const params = [new Date(sinceMs).toISOString(), new Date(untilMs).toISOString()] - const own = ownerPred(ownerUserId) + const own = ownerPred(ownerUserId, params.length) if (own.sql) { where.push(own.sql) params.push(...own.params) @@ -114,9 +115,9 @@ export class StatisticsRepo { return { cond: `WHERE ${where.join(' AND ')}`, params } } - _totals(sinceMs, untilMs, ownerUserId) { + async _totals(sinceMs, untilMs, ownerUserId) { const { cond, params } = this._scope(sinceMs, untilMs, ownerUserId) - const row = this.db + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END), 0) AS errors, @@ -153,10 +154,10 @@ export class StatisticsRepo { } } - _names(dimension, ids) { + async _names(dimension, ids) { const real = ids.filter((id) => id !== NONE_ID) if (dimension === 'model') return new Map(real.map((id) => [id, id])) - const names = lookupNames(this.db, { + const names = await lookupNames(this.db, { userIds: dimension === 'user' ? real : [], keyIds: dimension === 'key' ? real : [], vmIds: dimension === 'vm' ? real : [], @@ -169,14 +170,14 @@ export class StatisticsRepo { } /** Stacked-area series (top 8 by cost + others), window totals, and deltas vs the previous equal window. */ - statistics({ range = 'today', dimension = 'model', tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { + async statistics({ range = 'today', dimension = 'model', tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { const r = normalizeRange(range) || 'today' const dim = normalizeDimension(dimension) || 'model' const zone = reportTimezone(tz) const { since, until, resolution, buckets } = resolveWindow(r, zone, now) const col = DIMENSION_COLUMN[dim] const { cond, params } = this._scope(since, until, owner_user_id) - const rows = this.db + const rows = await this.db .prepare(` SELECT ${col} AS sid, ${SLOT_SQL} AS slot, COUNT(*) AS requests, @@ -201,7 +202,7 @@ export class StatisticsRepo { (a, b) => b.cost - a.cost || b.requests - a.requests || a.id.localeCompare(b.id), ) const kept = ranked.slice(0, MAX_SERIES) - const names = this._names( + const names = await this._names( dim, kept.map((s) => s.id), ) @@ -232,8 +233,8 @@ export class StatisticsRepo { } } - const totals = this._totals(since, until, owner_user_id) - const prev = this._totals(since - (until - since), since, owner_user_id) + const totals = await this._totals(since, until, owner_user_id) + const prev = await this._totals(since - (until - since), since, owner_user_id) return { range: r, resolution, @@ -254,7 +255,7 @@ export class StatisticsRepo { } /** Per-entity totals over the same window, by official cost. */ - leaderboard({ + async leaderboard({ range = 'today', scope = 'model', limit = LEADERBOARD_DEFAULT, @@ -269,9 +270,9 @@ export class StatisticsRepo { const { since, until } = resolveWindow(r, zone, now) const col = DIMENSION_COLUMN[s] const { cond, params } = this._scope(since, until, owner_user_id) - const rows = this.db + const rows = await this.db .prepare(` - SELECT ${col} AS sid, + SELECT COALESCE(${col}, '') AS sid, COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${SUCCESS_PRED} THEN 1 ELSE 0 END), 0) AS success, COALESCE(SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END), 0) AS errors, @@ -285,13 +286,13 @@ export class StatisticsRepo { AVG(duration_ms) AS avg_duration_ms, AVG(first_token_ms) AS avg_ttft_ms FROM usage_logs ${cond} - GROUP BY IFNULL(${col}, '') + GROUP BY COALESCE(${col}, '') ORDER BY cost DESC, requests DESC - LIMIT ? + LIMIT $${params.length + 1} `) .all(...params, n) const ids = rows.map((row) => (row.sid == null || row.sid === '' ? NONE_ID : String(row.sid))) - const names = this._names(s, ids) + const names = await this._names(s, ids) return { scope: s, range: r, diff --git a/src/lib/db/repos/usage-log-preds.mjs b/src/lib/db/repos/usage-log-preds.mjs index 7b20ef315..893824e22 100644 --- a/src/lib/db/repos/usage-log-preds.mjs +++ b/src/lib/db/repos/usage-log-preds.mjs @@ -18,9 +18,9 @@ export const SLA_ERROR_PRED = `(${ERROR_PRED} AND NOT (${SLA_OK_PRED}))` // After migration 037 every row stores disjoint input. Prompt (cache hit // denominator) is the input-side total; total tokens also include output. export const PROMPT_TOKENS_SQL = - 'IFNULL(input_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' + 'COALESCE(input_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)' export const TOTAL_TOKENS_SQL = - 'IFNULL(input_tokens, 0) + IFNULL(output_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)' + 'COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)' export function totalTokens(row = {}) { return ( @@ -32,11 +32,11 @@ export function totalTokens(row = {}) { } /** Rows a panel `user` owns: own user_id, one of their live keys, or served by a slot they owned at the time. */ -export function ownerPred(ownerUserId) { +export function ownerPred(ownerUserId, offset = 0) { const id = String(ownerUserId || '').trim() if (!id) return { sql: '', params: [] } return { - sql: `(user_id = ? OR IFNULL(api_key_id, '') IN (SELECT id FROM api_keys WHERE user_id = ? AND deleted_at IS NULL) OR vm_owner_user_id = ?)`, + sql: `(user_id = $${offset + 1} OR COALESCE(api_key_id, '') IN (SELECT id FROM api_keys WHERE user_id = $${offset + 2} AND deleted_at IS NULL) OR vm_owner_user_id = $${offset + 3})`, params: [id, id, id], } } diff --git a/src/lib/db/repos/usage-logs-repo.mjs b/src/lib/db/repos/usage-logs-repo.mjs index d5b4c5a74..8fd8ea17b 100644 --- a/src/lib/db/repos/usage-logs-repo.mjs +++ b/src/lib/db/repos/usage-logs-repo.mjs @@ -8,7 +8,7 @@ * may pass either name. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' import { collectErrors, enrichLogRow, excludeErrorClassSql, ingressAuthSql } from '../../admin/error-class.mjs' import { costColumnsFromUsage, @@ -160,11 +160,11 @@ function timeCond(since, until) { const where = [] const params = [] if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } if (until) { - where.push('created_at <= ?') + where.push(`created_at <= $${params.length + 1}`) params.push(new Date(until).toISOString()) } return { cond: where.length ? `WHERE ${where.join(' AND ')}` : '', params } @@ -238,15 +238,15 @@ function filterCond({ since = null, until = null, vmId = null, accountId = null, const { cond, params } = timeCond(since, until) const parts = cond ? [cond.replace(/^WHERE /, '')] : [] if (vmId) { - parts.push('vm_id = ?') + parts.push(`vm_id = $${params.length + 1}`) params.push(vmId) } if (apiKeyId) { - parts.push('api_key_id = ?') + parts.push(`api_key_id = $${params.length + 1}`) params.push(apiKeyId) } if (accountId) { - parts.push('(account_id = ? OR final_account_id = ?)') + parts.push(`(account_id = $${params.length + 1} OR final_account_id = $${params.length + 2})`) params.push(accountId, accountId) } return { cond: parts.length ? `WHERE ${parts.join(' AND ')}` : '', params } @@ -288,86 +288,91 @@ export class UsageLogsRepo { this._billingBackfillAt = 0 this._insert = db.prepare(` INSERT INTO usage_logs (${SUMMARY_COLUMNS.join(', ')}) - VALUES (${SUMMARY_COLUMNS.map(() => '?').join(', ')}) + VALUES (${SUMMARY_COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) this._insertIfAbsent = db.prepare(` - INSERT OR IGNORE INTO usage_logs (${SUMMARY_COLUMNS.join(', ')}) - VALUES (${SUMMARY_COLUMNS.map(() => '?').join(', ')}) + INSERT INTO usage_logs (${SUMMARY_COLUMNS.join(', ')}) + VALUES (${SUMMARY_COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) + ON CONFLICT(request_id) DO NOTHING `) this._insertDebug = db.prepare(` - INSERT INTO request_log_debug (request_id, ts, record_json) VALUES (?, ?, ?) + INSERT INTO request_log_debug (request_id, ts, record_json) VALUES ($1, $2, $3) ON CONFLICT(request_id) DO UPDATE SET ts = excluded.ts, record_json = excluded.record_json `) this._insertDebugIfAbsent = db.prepare(` - INSERT OR IGNORE INTO request_log_debug (request_id, ts, record_json) VALUES (?, ?, ?) + INSERT INTO request_log_debug (request_id, ts, record_json) VALUES ($1, $2, $3) + ON CONFLICT(request_id) DO NOTHING `) - this._getDebug = db.prepare('SELECT record_json FROM request_log_debug WHERE request_id = ?') - this._vmOwner = db.prepare('SELECT owner_user_id FROM vms WHERE id = ?') - this._listDebug = db.prepare('SELECT record_json FROM request_log_debug ORDER BY ts DESC LIMIT ?') - this._getByRequestId = db.prepare('SELECT * FROM usage_logs WHERE request_id = ?') - // OCTET_LENGTH counts encoded bytes and lets SQLite read TEXT size from metadata without loading each body. + this._getDebug = db.prepare('SELECT record_json FROM request_log_debug WHERE request_id = $1') + this._vmOwner = db.prepare('SELECT owner_user_id FROM vms WHERE id = $1') + this._listDebug = db.prepare('SELECT record_json FROM request_log_debug ORDER BY ts DESC LIMIT $1') + this._getByRequestId = db.prepare('SELECT * FROM usage_logs WHERE request_id = $1') + // OCTET_LENGTH counts encoded bytes rather than characters. this._debugBytes = db.prepare('SELECT COALESCE(SUM(OCTET_LENGTH(record_json)), 0) AS n FROM request_log_debug') this._debugCount = db.prepare('SELECT COUNT(*) AS n FROM request_log_debug') - this._deleteOldUsage = db.prepare('DELETE FROM usage_logs WHERE created_at < ?') - this._deleteOldDebug = db.prepare('DELETE FROM request_log_debug WHERE ts < ?') + this._deleteOldUsage = db.prepare('DELETE FROM usage_logs WHERE created_at < $1') + this._deleteOldDebug = db.prepare('DELETE FROM request_log_debug WHERE ts < $1') this._deleteOldestDebug = db.prepare(` DELETE FROM request_log_debug WHERE request_id IN ( - SELECT request_id FROM request_log_debug ORDER BY ts ASC LIMIT ? + SELECT request_id FROM request_log_debug ORDER BY ts ASC LIMIT $1 ) `) } /** Cheap EXISTS: ≥1 2xx row visible to the caller (same ownerPred as request-log stats). */ - hasSuccessfulRequest({ owner_user_id = null } = {}) { + async hasSuccessfulRequest({ owner_user_id = null } = {}) { const own = ownerPred(owner_user_id) const where = own.sql ? `AND ${own.sql}` : '' - const row = this.db + const row = await this.db .prepare(`SELECT 1 AS ok FROM usage_logs WHERE status >= 200 AND status < 300 ${where} LIMIT 1`) .get(...own.params) return !!row } - belongsToOwner(requestId, ownerUserId) { + async belongsToOwner(requestId, ownerUserId) { if (!requestId) return false if (!ownerUserId) return true - const own = ownerPred(ownerUserId) - const row = this.db - .prepare(`SELECT 1 AS ok FROM usage_logs WHERE request_id = ? AND ${own.sql} LIMIT 1`) + const own = ownerPred(ownerUserId, 1) + const row = await this.db + .prepare(`SELECT 1 AS ok FROM usage_logs WHERE request_id = $1 AND ${own.sql} LIMIT 1`) .get(requestId, ...own.params) return !!row } - getByRequestId(requestId) { + async getByRequestId(requestId) { if (!requestId) return null - const row = this._getByRequestId.get(requestId) + const row = await this._getByRequestId.get(requestId) return row ? enrichLogRow(fromRow(row)) : null } - insertSummary(rec) { - this._insert.run(...toRow(rec)) + async insertSummary(rec) { + await this._insert.run(...toRow(rec)) return rec } - vmOwnerOf(vmId) { + async vmOwnerOf(vmId) { if (!vmId) return null - return this._vmOwner.get(String(vmId))?.owner_user_id || null + return (await this._vmOwner.get(String(vmId)))?.owner_user_id || null } - /** INSERT OR IGNORE (legacy import path). @returns true when inserted */ - insertSummaryIfAbsent(rec) { - return this._insertIfAbsent.run(...toRow(rec)).changes > 0 + /** Idempotent insert for legacy imports. @returns true when inserted */ + async insertSummaryIfAbsent(rec) { + return (await this._insertIfAbsent.run(...toRow(rec))).changes > 0 } - insertDebug(requestId, ts, record) { - this._insertDebug.run(requestId, ts || new Date().toISOString(), JSON.stringify(record)) + async insertDebug(requestId, ts, record) { + await this._insertDebug.run(requestId, ts || new Date().toISOString(), JSON.stringify(record)) } - insertDebugIfAbsent(requestId, ts, record) { - return this._insertDebugIfAbsent.run(requestId, ts || new Date().toISOString(), JSON.stringify(record)).changes > 0 + async insertDebugIfAbsent(requestId, ts, record) { + return ( + (await this._insertDebugIfAbsent.run(requestId, ts || new Date().toISOString(), JSON.stringify(record))).changes > + 0 + ) } - getDebug(requestId) { - const row = this._getDebug.get(requestId) + async getDebug(requestId) { + const row = await this._getDebug.get(requestId) if (!row) return null try { return enrichLogRow(JSON.parse(row.record_json)) @@ -376,12 +381,11 @@ export class UsageLogsRepo { } } - listDebug({ limit = 20, owner_user_id = null } = {}) { + async listDebug({ limit = 20, owner_user_id = null } = {}) { const n = Math.max(1, Math.min(100, Number(limit) || 20)) const own = ownerPred(owner_user_id) if (!own.sql) { - return this._listDebug - .all(n) + return (await this._listDebug.all(n)) .map((r) => { try { return enrichLogRow(JSON.parse(r.record_json)) @@ -391,16 +395,18 @@ export class UsageLogsRepo { }) .filter(Boolean) } - return this.db - .prepare(` + return ( + await this.db + .prepare(` SELECT d.record_json FROM request_log_debug d JOIN usage_logs u ON u.request_id = d.request_id WHERE ${own.sql} ORDER BY d.ts DESC - LIMIT ? + LIMIT $${own.params.length + 1} `) - .all(...own.params, n) + .all(...own.params, n) + ) .map((r) => { try { return enrichLogRow(JSON.parse(r.record_json)) @@ -415,7 +421,7 @@ export class UsageLogsRepo { * Filtered, paginated query over summaries (newest first). * @returns {{ items: object[], total: number }} */ - query({ + async query({ limit = 50, offset = 0, api_key_id = null, @@ -440,23 +446,23 @@ export class UsageLogsRepo { params.push(...own.params) } if (api_key_id) { - where.push('api_key_id = ?') + where.push(`api_key_id = $${params.length + 1}`) params.push(api_key_id) } if (vm_id) { - where.push('vm_id = ?') + where.push(`vm_id = $${params.length + 1}`) params.push(vm_id) } if (account_id) { - where.push('account_id = ?') + where.push(`account_id = $${params.length + 1}`) params.push(account_id) } if (model) { - where.push('model = ?') + where.push(`model = $${params.length + 1}`) params.push(model) } if (protocol) { - where.push('protocol = ?') + where.push(`protocol = $${params.length + 1}`) params.push(protocol) } if (error_class && (status == null || status === '')) status = 'error' @@ -464,7 +470,7 @@ export class UsageLogsRepo { if (String(status) === 'error') where.push(ERROR_PRED) else if (String(status) === 'ok') where.push('(status < 400 AND status IS NOT NULL)') else { - where.push('status = ?') + where.push(`status = $${params.length + 1}`) params.push(Number(status)) } } @@ -473,15 +479,17 @@ export class UsageLogsRepo { if (excludeSql) where.push(excludeSql) } if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } if (until) { - where.push('created_at <= ?') + where.push(`created_at <= $${params.length + 1}`) params.push(new Date(until).toISOString()) } if (q) { - where.push('(path LIKE ? OR error_code LIKE ? OR error_message LIKE ? OR request_id LIKE ?)') + where.push( + `(ascii_fold(path) LIKE ascii_fold($${params.length + 1}) ESCAPE '' OR ascii_fold(error_code) LIKE ascii_fold($${params.length + 2}) ESCAPE '' OR ascii_fold(error_message) LIKE ascii_fold($${params.length + 3}) ESCAPE '' OR ascii_fold(request_id) LIKE ascii_fold($${params.length + 4}) ESCAPE '')`, + ) const like = `%${String(q).slice(0, 100)}%` params.push(like, like, like, like) } @@ -491,18 +499,24 @@ export class UsageLogsRepo { const off = Math.max(0, Number(offset) || 0) if (error_class) { const scan = Math.min(5000, Math.max(n + off, 2000)) - const rows = this.db - .prepare(`SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ?`) - .all(...params, scan) + const rows = ( + await this.db + .prepare(`SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1}`) + .all(...params, scan) + ) .map(fromRow) .map(enrichLogRow) .filter((row) => row.error_class === error_class) return { items: rows.slice(off, off + n), total: rows.length } } - const total = this.db.prepare(`SELECT COUNT(*) c FROM usage_logs ${cond}`).get(...params).c - const items = this.db - .prepare(`SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`) - .all(...params, n, off) + const total = (await this.db.prepare(`SELECT COUNT(*) c FROM usage_logs ${cond}`).get(...params)).c + const items = ( + await this.db + .prepare( + `SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1} OFFSET $${params.length + 2}`, + ) + .all(...params, n, off) + ) .map(fromRow) .map(enrichLogRow) return { items, total } @@ -512,19 +526,19 @@ export class UsageLogsRepo { * Aggregate stats bucketed by day or hour (dashboard/usage charts). * @returns {Array<{bucket, requests, errors, input_tokens, output_tokens, avg_duration_ms}>} */ - aggregate({ since = null, until = null, bucket = 'day', owner_user_id = null } = {}) { - const fmt = bucket === 'hour' ? '%Y-%m-%dT%H:00' : '%Y-%m-%d' + async aggregate({ since = null, until = null, bucket = 'day', owner_user_id = null } = {}) { + const fmt = bucket === 'hour' ? 'YYYY-MM-DD\"T\"HH24:00' : 'YYYY-MM-DD' const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) let where = cond const allParams = [...params] if (own.sql) { where = where ? `${where} AND ${own.sql}` : `WHERE ${own.sql}` allParams.push(...own.params) } - return this.db + return await this.db .prepare(` - SELECT strftime('${fmt}', created_at) AS bucket, + SELECT to_char(created_at::timestamptz AT TIME ZONE 'UTC', '${fmt}') AS bucket, COUNT(*) AS requests, SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END) AS errors, COALESCE(SUM(input_tokens), 0) AS input_tokens, @@ -533,8 +547,8 @@ export class UsageLogsRepo { COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens, COALESCE(SUM(${PROMPT_TOKENS_SQL}), 0) AS prompt_tokens, COALESCE(SUM(total_cost), 0) AS total_cost, - CAST(AVG(duration_ms) AS INTEGER) AS avg_duration_ms, - CAST(AVG(first_token_ms) AS INTEGER) AS avg_first_token_ms + TRUNC(AVG(duration_ms))::bigint AS avg_duration_ms, + TRUNC(AVG(first_token_ms))::bigint AS avg_first_token_ms FROM usage_logs ${where} GROUP BY bucket ORDER BY bucket `) @@ -542,8 +556,8 @@ export class UsageLogsRepo { } /** Grand totals (dashboard db_totals). */ - totals() { - const row = this.db + async totals() { + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END) AS errors, @@ -577,29 +591,28 @@ export class UsageLogsRepo { } } - _repriceRows(rows, { writeActualCost = false } = {}) { + async _repriceRows(rows, { writeActualCost = false } = {}) { if (!rows.length) return { rows: 0, changed: 0, still_unpriced: 0, delta_usd: 0 } const upd = writeActualCost ? this.db.prepare(` UPDATE usage_logs - SET input_cost = ?, output_cost = ?, cache_read_cost = ?, - cache_creation_cost = ?, web_search_cost = ?, total_cost = ?, - actual_cost = ?, pricing_model = ?, price_source = ?, price_version = ?, - long_context = ? - WHERE id = ? + SET input_cost = $1, output_cost = $2, cache_read_cost = $3, + cache_creation_cost = $4, web_search_cost = $5, total_cost = $6, + actual_cost = $7, pricing_model = $8, price_source = $9, price_version = $10, + long_context = $11 + WHERE id = $12 `) : this.db.prepare(` UPDATE usage_logs - SET input_cost = ?, output_cost = ?, cache_read_cost = ?, - cache_creation_cost = ?, web_search_cost = ?, total_cost = ?, - pricing_model = ?, price_source = ?, price_version = ?, long_context = ? - WHERE id = ? + SET input_cost = $1, output_cost = $2, cache_read_cost = $3, + cache_creation_cost = $4, web_search_cost = $5, total_cost = $6, + pricing_model = $7, price_source = $8, price_version = $9, long_context = $10 + WHERE id = $11 `) let changed = 0 let still_unpriced = 0 let delta_usd = 0 - this.db.exec('BEGIN') - try { + await withTransaction(this.db, async () => { for (const r of rows) { const model = r.upstream_model || r.model || r.requested_model const priced = costColumnsFromUsage(this._usageForPrice(r), model) @@ -624,7 +637,7 @@ export class UsageLogsRepo { const parsedRate = r.rate_multiplier == null ? 1 : Number(r.rate_multiplier) const rate = Number.isFinite(parsedRate) ? parsedRate : 1 const actual = priced.total_cost == null ? null : priced.total_cost * rate - upd.run( + await upd.run( priced.input_cost ?? null, priced.output_cost ?? null, priced.cache_read_cost ?? null, @@ -639,7 +652,7 @@ export class UsageLogsRepo { r.id, ) } else { - upd.run( + await upd.run( priced.input_cost ?? null, priced.output_cost ?? null, priced.cache_read_cost ?? null, @@ -654,13 +667,7 @@ export class UsageLogsRepo { ) } } - this.db.exec('COMMIT') - } catch (e) { - try { - this.db.exec('ROLLBACK') - } catch {} - throw e - } + }) return { rows: rows.length, changed, still_unpriced, delta_usd } } @@ -670,8 +677,8 @@ export class UsageLogsRepo { * (cost stays NULL) so they are never guessed at standard rates. Idempotent. * Only `pricing_model IS NULL` rows; those set actual_cost = total_cost × rate_multiplier. */ - backfillMissingCosts({ limit = 4000 } = {}) { - const rows = this.db + async backfillMissingCosts({ limit = 4000 } = {}) { + const rows = await this.db .prepare(` SELECT id, model, upstream_model, requested_model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, @@ -681,53 +688,54 @@ export class UsageLogsRepo { total_cost, pricing_model, price_source, price_version, long_context FROM usage_logs WHERE pricing_model IS NULL - LIMIT ? + LIMIT $1 `) .all(Math.max(1, Math.min(20000, Number(limit) || 4000))) - return this._repriceRows(rows, { writeActualCost: true }).rows + return (await this._repriceRows(rows, { writeActualCost: true })).rows } /** Rewrite cost columns in [since, until]; never actual_cost / rate_multiplier. */ - repriceRange({ since, until } = {}) { + async repriceRange({ since, until } = {}) { const { cond, params } = timeCond(since, until) - const where = cond ? `${cond} AND rowid > ?` : 'WHERE rowid > ?' + const where = `${cond || 'WHERE 1 = 1'} AND id > $${params.length + 1}` // Keyset pages keep a month-long range from loading every row at once. const page = this.db.prepare(` - SELECT rowid AS _rowid, id, model, upstream_model, requested_model, + SELECT id, model, upstream_model, requested_model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, cache_creation_5m_tokens, cache_creation_1h_tokens, web_search_requests, rate_multiplier, service_tier, speed, input_cost, output_cost, cache_read_cost, cache_creation_cost, web_search_cost, total_cost, pricing_model, price_source, price_version, long_context FROM usage_logs ${where} - ORDER BY rowid + ORDER BY id LIMIT 2000 `) const out = { rows: 0, changed: 0, still_unpriced: 0, delta_usd: 0 } - let after = 0 + let after = '' for (;;) { - const rows = page.all(...params, after) + const rows = await page.all(...params, after) if (!rows.length) break - const part = this._repriceRows(rows, { writeActualCost: false }) + const part = await this._repriceRows(rows, { writeActualCost: false }) out.rows += part.rows out.changed += part.changed out.still_unpriced += part.still_unpriced out.delta_usd += part.delta_usd - after = rows.at(-1)._rowid + after = rows.at(-1).id } out.delta_usd = Math.round(out.delta_usd * 1e8) / 1e8 return out } - unpricedModels({ since } = {}) { + async unpricedModels({ since } = {}) { const where = [`pricing_model = '${UNPRICED_MODEL}'`] const params = [] if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } - return this.db - .prepare(` + return ( + await this.db + .prepare(` SELECT COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '') AS model, COUNT(*) AS requests, MAX(created_at) AS last_seen @@ -736,16 +744,16 @@ export class UsageLogsRepo { GROUP BY 1 ORDER BY requests DESC, last_seen DESC `) - .all(...params) - .map((r) => ({ - model: r.model, - requests: Number(r.requests || 0), - last_seen: r.last_seen || null, - })) + .all(...params) + ).map((r) => ({ + model: r.model, + requests: Number(r.requests || 0), + last_seen: r.last_seen || null, + })) } - _costSelect(cond, params) { - const row = this.db + async _costSelect(cond, params) { + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, @@ -780,9 +788,9 @@ export class UsageLogsRepo { return { ...out, ...cacheHitStats(out) } } - costByAccount({ since = null, until = null } = {}) { + async costByAccount({ since = null, until = null } = {}) { const { cond, params } = timeCond(since, until) - const rows = this.db + const rows = await this.db .prepare(` SELECT COALESCE(NULLIF(final_account_id, ''), NULLIF(account_id, ''), vm_id, '—') AS account_id, @@ -831,9 +839,9 @@ export class UsageLogsRepo { * service_tier is folded to fast / flex / other-raw (standard aliases → NULL); * speed is 'fast' or NULL. unpriced_requests counts rows left without an estimate. */ - costByModel({ vmId = null, accountId = null, since = null, until = null } = {}) { + async costByModel({ vmId = null, accountId = null, since = null, until = null } = {}) { const { cond, params } = filterCond({ since, until, vmId, accountId }) - const rows = this.db + const rows = await this.db .prepare(` SELECT COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '—') AS model, CASE @@ -887,8 +895,8 @@ export class UsageLogsRepo { * `models` / `endpoints` are ranked by request count and capped; `endpoints` * is the inbound request path (usage_logs keeps no separate upstream endpoint). */ - vmUsageStats({ vmId, days = 30 } = {}) { - return this._usageWindow({ + async vmUsageStats({ vmId, days = 30 } = {}) { + return await this._usageWindow({ vmId, days, rankExpr: "COALESCE(NULLIF(path, ''), '—')", @@ -898,8 +906,8 @@ export class UsageLogsRepo { } /** Same window as vmUsageStats, isolated to one managed key. Third rank is the VM. */ - keyUsageStats({ apiKeyId, days = 30 } = {}) { - return this._usageWindow({ + async keyUsageStats({ apiKeyId, days = 30 } = {}) { + return await this._usageWindow({ apiKeyId, days, rankExpr: "COALESCE(NULLIF(vm_id, ''), '—')", @@ -908,15 +916,16 @@ export class UsageLogsRepo { }) } - _usageWindow({ vmId = null, apiKeyId = null, days = 30, rankExpr, rankKey, rankLimit = 8 } = {}) { + async _usageWindow({ vmId = null, apiKeyId = null, days = 30, rankExpr, rankKey, rankLimit = 8 } = {}) { const empty = { days: 0, since: null, history: [], models: [], [rankKey]: [] } if (!vmId && !apiKeyId) return empty const span = Math.max(1, Math.min(90, Math.floor(Number(days)) || 30)) const since = new Date(Date.parse(shanghaiDayStartIso()) - (span - 1) * 86400_000).toISOString() const { cond, params } = filterCond({ since, vmId, apiKeyId }) - const history = this.db - .prepare(` - SELECT strftime('%Y-%m-%d', created_at, '+8 hours') AS day, + const history = ( + await this.db + .prepare(` + SELECT to_char(created_at::timestamptz AT TIME ZONE 'Asia/Shanghai', 'YYYY-MM-DD') AS day, COUNT(*) AS requests, SUM(CASE WHEN ${SLA_ERROR_PRED} THEN 1 ELSE 0 END) AS errors, COALESCE(SUM(input_tokens), 0) AS input_tokens, @@ -930,23 +939,24 @@ export class UsageLogsRepo { FROM usage_logs ${cond} GROUP BY day ORDER BY day `) - .all(...params) - .map((r) => ({ - day: r.day, - requests: Number(r.requests || 0), - errors: Number(r.errors || 0), - input_tokens: Number(r.input_tokens || 0), - output_tokens: Number(r.output_tokens || 0), - cache_read_tokens: Number(r.cache_read_tokens || 0), - cache_creation_tokens: Number(r.cache_creation_tokens || 0), - prompt_tokens: Number(r.prompt_tokens || 0), - total_cost: Number(r.total_cost || 0), - duration_ms_sum: Number(r.duration_ms_sum || 0), - duration_n: Number(r.duration_n || 0), - })) - const group = (expr, limit) => - this.db - .prepare(` + .all(...params) + ).map((r) => ({ + day: r.day, + requests: Number(r.requests || 0), + errors: Number(r.errors || 0), + input_tokens: Number(r.input_tokens || 0), + output_tokens: Number(r.output_tokens || 0), + cache_read_tokens: Number(r.cache_read_tokens || 0), + cache_creation_tokens: Number(r.cache_creation_tokens || 0), + prompt_tokens: Number(r.prompt_tokens || 0), + total_cost: Number(r.total_cost || 0), + duration_ms_sum: Number(r.duration_ms_sum || 0), + duration_n: Number(r.duration_n || 0), + })) + const group = async (expr, limit) => + ( + await this.db + .prepare(` SELECT ${expr} AS name, COUNT(*) AS requests, COALESCE(SUM(${TOTAL_TOKENS_SQL}), 0) AS tokens, @@ -954,19 +964,22 @@ export class UsageLogsRepo { FROM usage_logs ${cond} GROUP BY name ORDER BY requests DESC, total_cost DESC LIMIT ${limit} `) - .all(...params) - .map((r) => ({ - name: r.name, - requests: Number(r.requests || 0), - tokens: Number(r.tokens || 0), - total_cost: Number(r.total_cost || 0), - })) + .all(...params) + ).map((r) => ({ + name: r.name, + requests: Number(r.requests || 0), + tokens: Number(r.tokens || 0), + total_cost: Number(r.total_cost || 0), + })) return { days: span, since, history, - models: group("COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '—')", 12), - [rankKey]: group(rankExpr, rankLimit), + models: await group( + "COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), NULLIF(requested_model, ''), '—')", + 12, + ), + [rankKey]: await group(rankExpr, rankLimit), } } @@ -975,7 +988,7 @@ export class UsageLogsRepo { * `total` is every inbound inference request (warmup answers included) plus * gate blocks on other paths, so each stage's share uses one denominator. */ - protocolEntryStats({ since = shanghaiDayStartIso() } = {}) { + async protocolEntryStats({ since = shanghaiDayStartIso() } = {}) { // COALESCE: a bare `col = x` on NULL yields NULL, and `NOT NULL` drops the row. const block = `( COALESCE(error_code, '') = 'policy_blocked' @@ -983,7 +996,7 @@ export class UsageLogsRepo { OR COALESCE(final_state, '') IN ('policy_blocked', 'distill_blocked', 'refusal_guard', 'refusal_similar', 'refusal_device') )` const inference = `path IN ('/v1/messages', '/v1/responses', '/v1/chat/completions')` - const row = this.db + const row = await this.db .prepare( `SELECT SUM(CASE WHEN ${inference} OR ${block} THEN 1 ELSE 0 END) AS total, @@ -993,14 +1006,14 @@ export class UsageLogsRepo { AND COALESCE(via, '') != 'warmup-intercept' THEN 1 ELSE 0 END) AS passed FROM usage_logs - WHERE created_at >= ?`, + WHERE created_at >= $1`, ) .get(since) - const keywords = this.db + const keywords = await this.db .prepare( `SELECT error_message AS message, COUNT(*) AS count FROM usage_logs - WHERE created_at >= ? + WHERE created_at >= $1 AND ${block} AND error_message IS NOT NULL AND error_message != '' @@ -1009,13 +1022,13 @@ export class UsageLogsRepo { LIMIT 50`, ) .all(since) - const grouped = this.db + const grouped = await this.db .prepare( `SELECT intercept, via, error_message AS message, CASE WHEN ${block} THEN 1 ELSE 0 END AS blocked, COUNT(*) AS count FROM usage_logs - WHERE created_at >= ? + WHERE created_at >= $1 AND ( ${block} OR (${inference} AND COALESCE(via, '') != 'warmup-intercept') @@ -1036,7 +1049,7 @@ export class UsageLogsRepo { } /** Official-standard totals: all-time + Shanghai calendar today + per account. */ - billingStats(opts = {}) { + async billingStats(opts = {}) { const { accountWindows = null, now = Date.now() } = opts const cacheable = !Object.hasOwn(opts, 'now') const cacheNow = Date.now() @@ -1050,11 +1063,11 @@ export class UsageLogsRepo { if (cacheNow - this._billingBackfillAt >= BILLING_BACKFILL_INTERVAL_MS) { this._billingBackfillAt = cacheNow try { - this.backfillMissingCosts() + await this.backfillMissingCosts() } catch {} } - const value = this._computeBillingStats({ accountWindows, now }) + const value = await this._computeBillingStats({ accountWindows, now }) if (this._billingStatsCache.size > 32) { for (const [cacheKey, entry] of this._billingStatsCache) { if (entry.expires_at <= cacheNow) this._billingStatsCache.delete(cacheKey) @@ -1068,12 +1081,12 @@ export class UsageLogsRepo { return value } - _computeBillingStats({ accountWindows = null, now = Date.now() } = {}) { + async _computeBillingStats({ accountWindows = null, now = Date.now() } = {}) { const todayStart = shanghaiDayStartIso() const fiveHStart = new Date(now - WINDOW_5H_MS).toISOString() const sevenDStart = new Date(now - WINDOW_7D_MS).toISOString() - const accountsTotal = this.costByAccount() - const accountsToday = this.costByAccount({ since: todayStart }) + const accountsTotal = await this.costByAccount() + const accountsToday = await this.costByAccount({ since: todayStart }) // The per-account queries already cover every row. Re-sum them for the // global buckets instead of scanning the entire usage table two more times. // This matters on multi-GB databases where each synchronous full scan also @@ -1100,9 +1113,13 @@ export class UsageLogsRepo { } eventsSince = new Date(earliest).toISOString() } - const events = aligned ? this._costEventsSince(eventsSince) : null - const fiveById = aligned ? null : new Map(this.costByAccount({ since: fiveHStart }).map((a) => [keyOf(a), a])) - const sevenById = aligned ? null : new Map(this.costByAccount({ since: sevenDStart }).map((a) => [keyOf(a), a])) + const events = aligned ? await this._costEventsSince(eventsSince) : null + const fiveById = aligned + ? null + : new Map((await this.costByAccount({ since: fiveHStart })).map((a) => [keyOf(a), a])) + const sevenById = aligned + ? null + : new Map((await this.costByAccount({ since: sevenDStart })).map((a) => [keyOf(a), a])) const eventsByKey = aligned ? groupCostEvents(events) : null const accounts = accountsTotal.map((a) => { const t = todayById.get(keyOf(a)) || emptyCostBucket() @@ -1168,14 +1185,14 @@ export class UsageLogsRepo { ...sumCostBuckets(accounts.map((a) => a.window_5h)), ...cacheHitStats(sumCostBuckets(accounts.map((a) => a.window_5h))), } - : this._costSelect('WHERE created_at >= ?', [fiveHStart]) + : await this._costSelect('WHERE created_at >= $1', [fiveHStart]) const window7d = aligned ? { ...sumCostBuckets(accounts.map((a) => a.window_7d)), ...cacheHitStats(sumCostBuckets(accounts.map((a) => a.window_7d))), } - : this._costSelect('WHERE created_at >= ?', [sevenDStart]) - const coverage = this.logCoverageStart() + : await this._costSelect('WHERE created_at >= $1', [sevenDStart]) + const coverage = await this.logCoverageStart() return { source: 'anthropic-official', currency: 'USD', @@ -1201,17 +1218,18 @@ export class UsageLogsRepo { * surviving row (a quiet hour looks like a gap) nor the current retain_days (raising * it does not bring deleted rows back) can stand in for that boundary. */ - logCoverageStart() { + async logCoverageStart() { const created = Date.parse( - this.db.prepare("SELECT MIN(applied_at) AS t FROM schema_migrations WHERE version = '001'").get()?.t || '', + (await this.db.prepare("SELECT MIN(applied_at) AS t FROM schema_migrations WHERE version = '001'").get())?.t || + '', ) - const pruned = Date.parse(this._prunedThrough() || '') + const pruned = Date.parse((await this._prunedThrough()) || '') const start = Math.max(Number.isFinite(created) ? created : -Infinity, Number.isFinite(pruned) ? pruned : -Infinity) return Number.isFinite(start) ? new Date(start).toISOString() : null } - _prunedThrough() { - const row = this.db.prepare(`SELECT value FROM settings WHERE key = '${PRUNED_THROUGH_KEY}'`).get() + async _prunedThrough() { + const row = await this.db.prepare(`SELECT value FROM settings WHERE key = '${PRUNED_THROUGH_KEY}'`).get() try { return row ? JSON.parse(row.value) : null } catch { @@ -1219,8 +1237,8 @@ export class UsageLogsRepo { } } - _costEventsSince(since) { - return this.db + async _costEventsSince(since) { + return await this.db .prepare( ` SELECT @@ -1243,13 +1261,13 @@ export class UsageLogsRepo { COALESCE(cache_creation_cost, 0) AS cache_creation_cost, COALESCE(total_cost, 0) AS total_cost FROM usage_logs - WHERE created_at >= ? + WHERE created_at >= $1 `, ) .all(since) } - ownerBilling({ ownerUserId = null, since = null, until = null, groupBy = 'vm' } = {}) { + async ownerBilling({ ownerUserId = null, since = null, until = null, groupBy = 'vm' } = {}) { const where = [] const params = [] const own = ownerPred(ownerUserId) @@ -1258,17 +1276,17 @@ export class UsageLogsRepo { params.push(...own.params) } if (since) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(new Date(since).toISOString()) } if (until) { - where.push('created_at <= ?') + where.push(`created_at <= $${params.length + 1}`) params.push(new Date(until).toISOString()) } const cond = where.length ? `WHERE ${where.join(' AND ')}` : '' const grouped = groupBy === 'key' ? 'key' : 'vm' const groupCol = grouped === 'key' ? 'api_key_id' : 'vm_id' - const totals = this.db + const totals = await this.db .prepare(` SELECT COUNT(*) AS requests, SUM(CASE WHEN ${SUCCESS_PRED} THEN 1 ELSE 0 END) AS ok, @@ -1280,7 +1298,7 @@ export class UsageLogsRepo { FROM usage_logs ${cond} `) .get(...params) - const items = this.db + const items = await this.db .prepare(` SELECT ${groupCol} AS id, COUNT(*) AS requests, @@ -1327,16 +1345,16 @@ export class UsageLogsRepo { * Windowed ops snapshot aligned with sub2api overview cards: * SLA / error / 429 / 503, QPS·TPS, duration + first_token percentiles, per-model. */ - windowStats({ since = null, until = null, owner_user_id = null } = {}) { + async windowStats({ since = null, until = null, owner_user_id = null } = {}) { const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) let where = cond const allParams = [...params] if (own.sql) { where = where ? `${where} AND ${own.sql}` : `WHERE ${own.sql}` allParams.push(...own.params) } - const counts = this.db + const counts = await this.db .prepare(` SELECT COUNT(*) AS requests, @@ -1376,9 +1394,9 @@ export class UsageLogsRepo { const windowSeconds = Number.isFinite(startMs) && Number.isFinite(endMs) ? Math.max(1, (endMs - startMs) / 1000) : Math.max(1, requests) - const minuteRows = this.db + const minuteRows = await this.db .prepare(` - SELECT strftime('%Y-%m-%dT%H:%M', created_at) AS minute, + SELECT to_char(created_at::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI') AS minute, COUNT(*) AS requests, COALESCE(SUM(${TOTAL_TOKENS_SQL}), 0) AS tokens FROM usage_logs ${where} @@ -1391,23 +1409,21 @@ export class UsageLogsRepo { const ttftWhere = where ? `${where} AND first_token_ms IS NOT NULL` : 'WHERE first_token_ms IS NOT NULL' const durWhere = where ? `${where} AND duration_ms IS NOT NULL` : 'WHERE duration_ms IS NOT NULL' - const ttfts = this.db - .prepare(`SELECT first_token_ms AS v FROM usage_logs ${ttftWhere}`) - .all(...allParams) - .map((r) => r.v) - const durs = this.db - .prepare(`SELECT duration_ms AS v FROM usage_logs ${durWhere}`) - .all(...allParams) - .map((r) => r.v) + const ttfts = ( + await this.db.prepare(`SELECT first_token_ms AS v FROM usage_logs ${ttftWhere}`).all(...allParams) + ).map((r) => r.v) + const durs = (await this.db.prepare(`SELECT duration_ms AS v FROM usage_logs ${durWhere}`).all(...allParams)).map( + (r) => r.v, + ) - const byModel = this.db + const byModel = await this.db .prepare(` SELECT COALESCE(NULLIF(upstream_model, ''), NULLIF(model, ''), '—') AS model, COUNT(*) AS requests, SUM(CASE WHEN ${SLA_ERROR_PRED} THEN 1 ELSE 0 END) AS errors, SUM(CASE WHEN first_token_ms IS NOT NULL THEN 1 ELSE 0 END) AS ttft_samples, - CAST(AVG(first_token_ms) AS INTEGER) AS avg_first_token_ms, - CAST(AVG(duration_ms) AS INTEGER) AS avg_duration_ms + TRUNC(AVG(first_token_ms))::bigint AS avg_first_token_ms, + TRUNC(AVG(duration_ms))::bigint AS avg_duration_ms FROM usage_logs ${where} GROUP BY 1 ORDER BY requests DESC @@ -1434,7 +1450,7 @@ export class UsageLogsRepo { cache_creation_tokens: cacheWrite, prompt_tokens: cache.prompt_tokens, cache_hit_rate: cache.cache_hit_rate, - sticky: this._stickyWindow({ since, until, owner_user_id }), + sticky: await this._stickyWindow({ since, until, owner_user_id }), total_cost: Number(counts?.total_cost || 0), qps: { current: lastMinute ? Number(lastMinute.requests || 0) / 60 : 0, @@ -1456,30 +1472,30 @@ export class UsageLogsRepo { avg_first_token_ms: r.avg_first_token_ms ?? null, avg_duration_ms: r.avg_duration_ms ?? null, })), - error_collection: this.errorCollection({ since, until, owner_user_id }), + error_collection: await this.errorCollection({ since, until, owner_user_id }), } } - _stickyWindow({ since = null, until = null, owner_user_id = null } = {}) { + async _stickyWindow({ since = null, until = null, owner_user_id = null } = {}) { const empty = { selections: 0, hits: 0, rate: null } try { const parts = [] const params = [] if (since) { - parts.push('started_at >= ?') + parts.push(`started_at >= $${params.length + 1}`) params.push(since) } if (until) { - parts.push('started_at <= ?') + parts.push(`started_at <= $${params.length + 1}`) params.push(until) } - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) if (own.sql) { parts.push(`request_id IN (SELECT request_id FROM usage_logs WHERE ${own.sql})`) params.push(...own.params) } const where = parts.length ? `WHERE ${parts.join(' AND ')}` : '' - const rows = this.db + const rows = await this.db .prepare(` SELECT selection_reason, COUNT(*) AS n FROM request_attempts ${where} @@ -1503,11 +1519,11 @@ export class UsageLogsRepo { } } - errorCollection({ since = null, until = null, owner_user_id = null } = {}) { + async errorCollection({ since = null, until = null, owner_user_id = null } = {}) { const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) const errCond = `${cond ? `${cond} AND ` : 'WHERE '}${ERROR_PRED}${own.sql ? ` AND ${own.sql}` : ''}` - const rows = this.db + const rows = await this.db .prepare(` SELECT created_at AS ts, request_id, status, error_code, error_message, model, upstream_model, vm_id, ip_address AS ip, api_key_presented FROM usage_logs ${errCond} @@ -1517,19 +1533,20 @@ export class UsageLogsRepo { .all(...params, ...own.params) return { ...collectErrors(rows), - ingress_hits: this.ingressHits({ since, until, owner_user_id }), + ingress_hits: await this.ingressHits({ since, until, owner_user_id }), } } /** Invalid-key 401s grouped by presented token + IP (still counted when muted). */ - ingressHits({ since = null, until = null, owner_user_id = null } = {}) { + async ingressHits({ since = null, until = null, owner_user_id = null } = {}) { const { cond, params } = timeCond(since, until) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) const authPred = own.sql ? `${ingressAuthSql()} AND ${own.sql}` : ingressAuthSql() const where = cond ? `${cond} AND ${authPred}` : `WHERE ${authPred}` try { - return this.db - .prepare(` + return ( + await this.db + .prepare(` SELECT COALESCE(api_key_presented, '') AS api_key_presented, COALESCE(ip_address, '') AS ip, @@ -1542,14 +1559,14 @@ export class UsageLogsRepo { ORDER BY count DESC LIMIT 40 `) - .all(...params, ...own.params) - .map((row) => ({ - api_key_presented: row.api_key_presented || '', - ip: row.ip || '', - count: Number(row.count || 0), - last_ts: row.last_ts || null, - error_code: row.error_code || null, - })) + .all(...params, ...own.params) + ).map((row) => ({ + api_key_presented: row.api_key_presented || '', + ip: row.ip || '', + count: Number(row.count || 0), + last_ts: row.last_ts || null, + error_code: row.error_code || null, + })) } catch { return [] } @@ -1559,19 +1576,11 @@ export class UsageLogsRepo { * Age out summaries/debug independently, then trim debug blobs to maxBytes. * Number form `cleanup(7)` keeps the old retainDays-only call. */ - cleanup(opts = {}) { + async cleanup(opts = {}) { const cfg = this._cleanupOpts(opts) - const usage_logs = this._pruneUsage(cfg.usageCutoff) - const debug_by_age = this._deleteOldDebug.run(cfg.debugCutoff).changes - const debug_by_size = this.trimDebugToMaxBytes(cfg.maxBytes) - if (usage_logs + debug_by_age + debug_by_size > 0) { - try { - this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)') - } catch {} - } - try { - this.db.exec('PRAGMA optimize') - } catch {} + const usage_logs = await this._pruneUsage(cfg.usageCutoff) + const debug_by_age = (await this._deleteOldDebug.run(cfg.debugCutoff)).changes + const debug_by_size = await this.trimDebugToMaxBytes(cfg.maxBytes) return { usage_logs, request_log_debug: debug_by_age + debug_by_size, @@ -1581,29 +1590,17 @@ export class UsageLogsRepo { } /** Delete summaries before `cutoff` and advance the prune boundary in one transaction. */ - _pruneUsage(cutoff) { - this.db.exec('BEGIN') - try { - const deleted = this._deleteOldUsage.run(cutoff).changes + async _pruneUsage(cutoff) { + return withTransaction(this.db, async () => { + const deleted = (await this._deleteOldUsage.run(cutoff)).changes if (deleted > 0) { - const prev = this._prunedThrough() - if (!prev || prev < cutoff) { - this.db - .prepare(` - INSERT INTO settings (key, value, updated_at) VALUES ('${PRUNED_THROUGH_KEY}', ?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at - `) - .run(JSON.stringify(cutoff), new Date().toISOString()) - } + await this.db + .prepare(`INSERT INTO settings (key, value, updated_at) VALUES ('${PRUNED_THROUGH_KEY}', $1, $2) + ON CONFLICT(key) DO UPDATE SET value = GREATEST(settings.value, excluded.value), updated_at = excluded.updated_at`) + .run(JSON.stringify(cutoff), new Date().toISOString()) } - this.db.exec('COMMIT') return deleted - } catch (e) { - try { - this.db.exec('ROLLBACK') - } catch {} - throw e - } + }) } _cleanupOpts(opts) { @@ -1618,35 +1615,35 @@ export class UsageLogsRepo { } } - debugBytes() { - return Number(this._debugBytes.get().n || 0) + async debugBytes() { + return Number((await this._debugBytes.get()).n || 0) } - countDebug() { - return Number(this._debugCount.get().n || 0) + async countDebug() { + return Number((await this._debugCount.get()).n || 0) } /** Delete oldest debug rows until the stored JSON bytes fit maxBytes. */ - trimDebugToMaxBytes(maxBytes) { + async trimDebugToMaxBytes(maxBytes) { const cap = Math.max(0, Number(maxBytes) || 0) if (cap <= 0) return 0 let deleted = 0 for (let i = 0; i < 8; i++) { - const used = this.debugBytes() + const used = await this.debugBytes() if (used <= cap) return deleted - const count = this.countDebug() + const count = await this.countDebug() if (!count) return deleted const avg = Math.max(1, Math.floor(used / count)) const need = Math.min(count, Math.max(1, Math.ceil((used - cap) / avg) + 1)) - const n = this._deleteOldestDebug.run(need).changes + const n = (await this._deleteOldestDebug.run(need)).changes if (!n) return deleted deleted += n } return deleted } - count() { - return this.db.prepare('SELECT COUNT(*) c FROM usage_logs').get().c + async count() { + return (await this.db.prepare('SELECT COUNT(*) c FROM usage_logs').get()).c } } diff --git a/src/lib/db/repos/usage-logs-view.mjs b/src/lib/db/repos/usage-logs-view.mjs index 7e7b2b90a..52e9fcb42 100644 --- a/src/lib/db/repos/usage-logs-view.mjs +++ b/src/lib/db/repos/usage-logs-view.mjs @@ -73,30 +73,30 @@ function uniq(values) { } function placeholders(list) { - return list.map(() => '?').join(', ') + return list.map((_, i) => '$' + (i + 1)).join(', ') } /** * Batched display names for one page / one aggregate. One IN query per entity * kind so a page of 100 rows costs at most four lookups. */ -export function lookupNames(db, { userIds = [], keyIds = [], vmIds = [], accountIds = [] } = {}) { - const pick = (sql, ids, nameOf) => { +export async function lookupNames(db, { userIds = [], keyIds = [], vmIds = [], accountIds = [] } = {}) { + const pick = async (sql, ids, nameOf) => { const list = uniq(ids) const map = new Map() if (!list.length) return map - for (const r of db.prepare(sql.replace('$IN', placeholders(list))).all(...list)) map.set(r.id, nameOf(r)) + for (const r of await db.prepare(sql.replace('$IN', placeholders(list))).all(...list)) map.set(r.id, nameOf(r)) return map } return { - users: pick( + users: await pick( 'SELECT id, username, email FROM users WHERE id IN ($IN)', userIds, (r) => r.username || r.email || r.id, ), - keys: pick('SELECT id, name FROM api_keys WHERE id IN ($IN)', keyIds, (r) => r.name || r.id), - vms: pick('SELECT id, name FROM vms WHERE id IN ($IN)', vmIds, (r) => r.name || r.id), - accounts: pick( + keys: await pick('SELECT id, name FROM api_keys WHERE id IN ($IN)', keyIds, (r) => r.name || r.id), + vms: await pick('SELECT id, name FROM vms WHERE id IN ($IN)', vmIds, (r) => r.name || r.id), + accounts: await pick( 'SELECT id, email, name FROM accounts WHERE id IN ($IN)', accountIds, (r) => r.email || r.name || r.id, @@ -297,7 +297,7 @@ function buildFilter(filters, defaultMuted) { } const eq = (col, val) => { if (val == null || val === '') return - where.push(`${col} = ?`) + where.push(`${col} = $${params.length + 1}`) params.push(String(val)) } eq('user_id', filters.user_id) @@ -305,22 +305,22 @@ function buildFilter(filters, defaultMuted) { eq('vm_id', filters.vm_id) if (filters.session_id) { // Either side of the mapping finds the conversation. - where.push('(outbound_session_id = ? OR session_id = ?)') + where.push(`(outbound_session_id = $${params.length + 1} OR session_id = $${params.length + 2})`) params.push(String(filters.session_id), String(filters.session_id)) } eq('path', filters.endpoint) eq('protocol', filters.protocol) if (filters.account_id) { - where.push('(account_id = ? OR final_account_id = ?)') + where.push(`(account_id = $${params.length + 1} OR final_account_id = $${params.length + 2})`) params.push(filters.account_id, filters.account_id) } if (filters.model) { - where.push('(model = ? OR requested_model = ?)') + where.push(`(model = $${params.length + 1} OR requested_model = $${params.length + 2})`) params.push(filters.model, filters.model) } const statusCode = clampInt(filters.status_code, 0, 999, null) if (statusCode != null) { - where.push('status = ?') + where.push(`status = $${params.length + 1}`) params.push(statusCode) } else if (filters.exclude_status_200) { where.push('(status IS NULL OR status < 200 OR status >= 300)') @@ -328,7 +328,7 @@ function buildFilter(filters, defaultMuted) { if (filters.model_mismatch) where.push('model_mismatch = 1') const minAttempts = clampInt(filters.min_attempt_count, 0, 1000, null) if (minAttempts != null) { - where.push('IFNULL(attempt_count, 0) >= ?') + where.push(`COALESCE(attempt_count, 0) >= $${params.length + 1}`) params.push(minAttempts) } if (filters.debug_only) where.push("log_mode = 'debug'") @@ -336,16 +336,18 @@ function buildFilter(filters, defaultMuted) { if (filters.errors_only) where.push(ERROR_PRED) const start = isoOrNull(filters.start_time) if (start) { - where.push('created_at >= ?') + where.push(`created_at >= $${params.length + 1}`) params.push(start) } const end = isoOrNull(filters.end_time) if (end) { - where.push('created_at < ?') + where.push(`created_at < $${params.length + 1}`) params.push(end) } if (filters.q) { - where.push('(path LIKE ? OR error_code LIKE ? OR error_message LIKE ? OR request_id LIKE ?)') + where.push( + `(ascii_fold(path) LIKE ascii_fold($${params.length + 1}) ESCAPE '' OR ascii_fold(error_code) LIKE ascii_fold($${params.length + 2}) ESCAPE '' OR ascii_fold(error_message) LIKE ascii_fold($${params.length + 3}) ESCAPE '' OR ascii_fold(request_id) LIKE ascii_fold($${params.length + 4}) ESCAPE '')`, + ) const like = `%${String(filters.q).slice(0, 100)}%` params.push(like, like, like, like) } @@ -378,7 +380,7 @@ function whereSql(where) { export class UsageLogsView { /** - * @param {import('node:sqlite').DatabaseSync} db + * @param {object} db * @param {{ mutedErrorClasses?: string[] }} [opts] settings default mute list */ constructor(db = getDb(), { mutedErrorClasses = DEFAULT_MUTED_ERROR_CLASSES } = {}) { @@ -392,7 +394,7 @@ export class UsageLogsView { * that cap is hit before `limit` matches, the cursor points at the last * scanned row and `hasMore` stays true so the client keeps paging. */ - listBatch(filters = {}) { + async listBatch(filters = {}) { const limit = clampInt(filters.limit, 1, LIST_LIMIT_MAX, LIST_LIMIT_DEFAULT) const { where, params, keep } = buildFilter(filters, this.mutedErrorClasses) let cursor = @@ -408,11 +410,11 @@ export class UsageLogsView { const w = where.slice() const p = params.slice() if (cursor) { - w.push('(created_at < ? OR (created_at = ? AND id < ?))') + w.push(`(created_at < $${p.length + 1} OR (created_at = $${p.length + 2} AND id < $${p.length + 3}))`) p.push(cursor.createdAt, cursor.createdAt, cursor.id) } - const rows = this.db - .prepare(`SELECT * FROM usage_logs ${whereSql(w)} ORDER BY created_at DESC, id DESC LIMIT ?`) + const rows = await this.db + .prepare(`SELECT * FROM usage_logs ${whereSql(w)} ORDER BY created_at DESC, id DESC LIMIT $${p.length + 1}`) .all(...p, want) for (const raw of rows) { scanned++ @@ -448,24 +450,24 @@ export class UsageLogsView { hasMore = true } return { - logs: this._hydrate(page), + logs: await this._hydrate(page), nextCursor: next ? { createdAt: next.created_at, id: next.id } : null, hasMore, } } /** Names + provider chains for a whole page: one attempts query, one lookup per entity kind. */ - _hydrate(rows) { + async _hydrate(rows) { if (!rows.length) return [] const requestIds = uniq(rows.map((r) => r.request_id)) const attempts = requestIds.length - ? this.db + ? await this.db .prepare( `SELECT * FROM request_attempts WHERE request_id IN (${placeholders(requestIds)}) ORDER BY request_id, attempt_no`, ) .all(...requestIds) : [] - const names = lookupNames(this.db, { + const names = await lookupNames(this.db, { userIds: rows.map((r) => r.user_id), keyIds: rows.map((r) => r.api_key_id), vmIds: [...rows.map((r) => r.vm_id), ...attempts.map((a) => a.vm_id)], @@ -489,11 +491,11 @@ export class UsageLogsView { * SQL-expressible `auth` class; otherwise aggregates the newest * SUMMARY_SCAN_CAP rows after JS classification. */ - summary(filters = {}) { + async summary(filters = {}) { const { where, params, keep, sqlOnlySummary } = buildFilter(filters, this.mutedErrorClasses) const cond = whereSql(where) if (sqlOnlySummary) { - const row = this.db + const row = await this.db .prepare(` SELECT COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${SUCCESS_PRED} THEN 1 ELSE 0 END), 0) AS success, @@ -514,11 +516,13 @@ export class UsageLogsView { .get(...params) return toSummary(row) } - const rows = this.db - .prepare( - `SELECT *, (${PROMPT_TOKENS_SQL}) AS prompt_tokens FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ?`, - ) - .all(...params, SUMMARY_SCAN_CAP) + const rows = ( + await this.db + .prepare( + `SELECT *, (${PROMPT_TOKENS_SQL}) AS prompt_tokens FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1}`, + ) + .all(...params, SUMMARY_SCAN_CAP) + ) .map(enrichLogRow) .filter(keep) const acc = { @@ -576,23 +580,20 @@ export class UsageLogsView { * most EXPORT_SCAN_CAP newest rows are read and `capped` says the scan ended * before the table did. */ - exportRows(filters = {}) { + async exportRows(filters = {}) { const limit = clampInt(filters.limit, 1, EXPORT_LIMIT_MAX, EXPORT_LIMIT_DEFAULT) const { where, params, keep, sqlOnlySummary } = buildFilter(filters, this.mutedErrorClasses) const cond = whereSql(where) - const select = `SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT ?` + const select = `SELECT * FROM usage_logs ${cond} ORDER BY created_at DESC, id DESC LIMIT $${params.length + 1}` if (!keep || sqlOnlySummary) { - const total = this.db.prepare(`SELECT COUNT(*) AS c FROM usage_logs ${cond}`).get(...params).c + const total = (await this.db.prepare(`SELECT COUNT(*) AS c FROM usage_logs ${cond}`).get(...params)).c return { - items: this.db - .prepare(select) - .all(...params, limit) - .map(exportRow), + items: (await this.db.prepare(select).all(...params, limit)).map(exportRow), total, capped: false, } } - const scanned = this.db.prepare(select).all(...params, EXPORT_SCAN_CAP) + const scanned = await this.db.prepare(select).all(...params, EXPORT_SCAN_CAP) const matched = scanned.map(exportRow).filter(keep) return { items: matched.slice(0, limit), @@ -602,17 +603,18 @@ export class UsageLogsView { } /** Lazy dropdown values. Entity lists come from their tables; free-form columns from recent rows. */ - filterOptions({ owner_user_id = null } = {}) { + async filterOptions({ owner_user_id = null } = {}) { const own = ownerPred(owner_user_id) const recentWhere = own.sql ? `WHERE ${own.sql}` : '' const recent = `(SELECT * FROM usage_logs ${recentWhere} ORDER BY created_at DESC LIMIT ${RECENT_SCAN})` - const distinct = (col) => - this.db - .prepare( - `SELECT ${col} AS v FROM ${recent} WHERE ${col} IS NOT NULL AND ${col} != '' GROUP BY ${col} ORDER BY MAX(created_at) DESC LIMIT ${OPTION_LIMIT}`, - ) - .all(...own.params) - .map((r) => r.v) + const distinct = async (col) => + ( + await this.db + .prepare( + `SELECT ${col} AS v FROM ${recent} AS recent_rows WHERE ${col} IS NOT NULL ${col === 'status' ? '' : `AND ${col} != ''`} GROUP BY ${col} ORDER BY MAX(created_at) DESC LIMIT ${OPTION_LIMIT}`, + ) + .all(...own.params) + ).map((r) => r.v) const named = (ids, map) => ids.map((id) => ({ id: String(id), name: map.get(id) ?? String(id) })) let users @@ -620,105 +622,105 @@ export class UsageLogsView { let vmIds let accountIds if (owner_user_id) { - users = this.db.prepare('SELECT id, username, email FROM users WHERE id = ?').all(String(owner_user_id)) - keys = this.db - .prepare('SELECT id, name, user_id FROM api_keys WHERE user_id = ? AND deleted_at IS NULL ORDER BY name') + users = await this.db.prepare('SELECT id, username, email FROM users WHERE id = $1').all(String(owner_user_id)) + keys = await this.db + .prepare('SELECT id, name, user_id FROM api_keys WHERE user_id = $1 AND deleted_at IS NULL ORDER BY name') .all(String(owner_user_id)) // Pool slots that served this tenant appear in its rows even when not owned. vmIds = uniq([ - ...this.db - .prepare('SELECT id FROM vms WHERE owner_user_id = ?') - .all(String(owner_user_id)) - .map((r) => r.id), - ...distinct('vm_id'), + ...(await this.db.prepare('SELECT id FROM vms WHERE owner_user_id = $1').all(String(owner_user_id))).map( + (r) => r.id, + ), + ...(await distinct('vm_id')), ]) - accountIds = uniq([...distinct('final_account_id'), ...distinct('account_id')]) + accountIds = uniq([...(await distinct('final_account_id')), ...(await distinct('account_id'))]) } else { - users = this.db.prepare('SELECT id, username, email FROM users WHERE deleted_at IS NULL ORDER BY username').all() - keys = this.db.prepare('SELECT id, name, user_id FROM api_keys WHERE deleted_at IS NULL ORDER BY name').all() - vmIds = this.db - .prepare('SELECT id FROM vms ORDER BY id') + users = await this.db + .prepare('SELECT id, username, email FROM users WHERE deleted_at IS NULL ORDER BY username') .all() - .map((r) => r.id) - accountIds = this.db - .prepare('SELECT id FROM accounts WHERE deleted_at IS NULL ORDER BY id') + keys = await this.db + .prepare('SELECT id, name, user_id FROM api_keys WHERE deleted_at IS NULL ORDER BY name') .all() - .map((r) => r.id) + vmIds = (await this.db.prepare('SELECT id FROM vms ORDER BY id').all()).map((r) => r.id) + accountIds = (await this.db.prepare('SELECT id FROM accounts WHERE deleted_at IS NULL ORDER BY id').all()).map( + (r) => r.id, + ) } - const names = lookupNames(this.db, { vmIds, accountIds }) + const names = await lookupNames(this.db, { vmIds, accountIds }) return { - users: users.map((u) => ({ id: String(u.id), name: u.username || u.email || String(u.id) })), + users: await users.map((u) => ({ id: String(u.id), name: u.username || u.email || String(u.id) })), keys: keys.map((k) => ({ id: String(k.id), name: k.name || String(k.id), userId: k.user_id ?? null })), vms: named(vmIds, names.vms), accounts: named(accountIds, names.accounts), - models: uniq([...distinct('model'), ...distinct('requested_model')]).sort(), - protocols: distinct('protocol').sort(), - endpoints: distinct('path').sort(), - statusCodes: distinct('status') - .map(Number) - .sort((a, b) => a - b), + models: uniq([...(await distinct('model')), ...(await distinct('requested_model'))]).sort(), + protocols: (await distinct('protocol')).sort(), + endpoints: (await distinct('path')).sort(), + statusCodes: (await distinct('status')).map(Number).sort((a, b) => a - b), } } /** Outbound session ids starting with `q`, most recently used first (index range scan). */ - sessionSuggestions({ q = '', limit = 20, owner_user_id = null } = {}) { + async sessionSuggestions({ q = '', limit = 20, owner_user_id = null } = {}) { const n = clampInt(limit, 1, 50, 20) - const own = ownerPred(owner_user_id) + let own = ownerPred(owner_user_id) const prefix = String(q || '').trim() if (prefix) { - const where = ['outbound_session_id >= ?', 'outbound_session_id < ?'] + const where = ['outbound_session_id COLLATE "C" >= $1', 'outbound_session_id COLLATE "C" < $2'] const params = [prefix, `${prefix}\uffff`] + own = ownerPred(owner_user_id, params.length) if (own.sql) { where.push(own.sql) params.push(...own.params) } - return this.db - .prepare( - `SELECT outbound_session_id AS sid FROM usage_logs WHERE ${where.join(' AND ')} GROUP BY outbound_session_id ORDER BY MAX(created_at) DESC LIMIT ?`, - ) - .all(...params, n) - .map((r) => r.sid) + return ( + await this.db + .prepare( + `SELECT outbound_session_id AS sid FROM usage_logs WHERE ${where.join(' AND ')} GROUP BY outbound_session_id ORDER BY MAX(created_at) DESC LIMIT $${params.length + 1}`, + ) + .all(...params, n) + ).map((r) => r.sid) } const where = ['outbound_session_id IS NOT NULL'] if (own.sql) where.push(own.sql) - return this.db - .prepare( - `SELECT sid FROM (SELECT outbound_session_id AS sid, created_at FROM usage_logs WHERE ${where.join(' AND ')} ORDER BY created_at DESC LIMIT ${RECENT_SCAN}) GROUP BY sid ORDER BY MAX(created_at) DESC LIMIT ?`, - ) - .all(...own.params, n) - .map((r) => r.sid) + return ( + await this.db + .prepare( + `SELECT sid FROM (SELECT outbound_session_id AS sid, created_at FROM usage_logs WHERE ${where.join(' AND ')} ORDER BY created_at DESC LIMIT ${RECENT_SCAN}) AS recent_sessions GROUP BY sid ORDER BY MAX(created_at) DESC LIMIT $${own.params.length + 1}`, + ) + .all(...own.params, n) + ).map((r) => r.sid) } /** Outbound sessions with a completed request in the last `minutes`. */ - activeSessions({ minutes = 5, limit = 50, owner_user_id = null, now = Date.now() } = {}) { + async activeSessions({ minutes = 5, limit = 50, owner_user_id = null, now = Date.now() } = {}) { const mins = clampInt(minutes, 1, 1440, 5) const n = clampInt(limit, 1, 200, 50) - const where = ['created_at >= ?', 'outbound_session_id IS NOT NULL'] + const where = ['created_at >= $1', 'outbound_session_id IS NOT NULL'] const params = [new Date(now - mins * 60_000).toISOString()] - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, params.length) if (own.sql) { where.push(own.sql) params.push(...own.params) } const cond = `WHERE ${where.join(' AND ')}` - const total = this.db - .prepare(`SELECT COUNT(DISTINCT outbound_session_id) AS n FROM usage_logs ${cond}`) - .get(...params).n + const total = ( + await this.db.prepare(`SELECT COUNT(DISTINCT outbound_session_id) AS n FROM usage_logs ${cond}`).get(...params) + ).n // Aggregate first, then pull the newest row per session by id; SQLite's // bare-column-with-MAX shortcut is undefined once MIN() is also present. - const rows = this.db + const rows = await this.db .prepare(` WITH s AS ( SELECT outbound_session_id AS sid, MAX(created_at) AS last_at, MIN(created_at) AS first_at, COUNT(*) AS requests, - SUM(IFNULL(input_tokens, 0) + IFNULL(output_tokens, 0) + IFNULL(cache_read_tokens, 0) + IFNULL(cache_creation_tokens, 0)) AS tokens, + SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)) AS tokens, COALESCE(SUM(total_cost), 0) AS cost FROM usage_logs ${cond} GROUP BY outbound_session_id ORDER BY last_at DESC - LIMIT ? + LIMIT $${params.length + 1} ) SELECT s.*, l.session_id AS client_session_id, l.user_id, l.api_key_id, l.vm_id, l.model, l.status, l.duration_ms, COALESCE(l.final_account_id, l.account_id) AS account @@ -730,8 +732,8 @@ export class UsageLogsView { ) ORDER BY s.last_at DESC `) - .all(...params, n, ...own.params) - const names = lookupNames(this.db, { + .all(...params, n) + const names = await lookupNames(this.db, { userIds: rows.map((r) => r.user_id), keyIds: rows.map((r) => r.api_key_id), vmIds: rows.map((r) => r.vm_id), @@ -764,20 +766,20 @@ export class UsageLogsView { * Header cards. "Today" = local midnight in `tz` → now; "yesterday" = the * same elapsed span starting at yesterday's local midnight. */ - overview({ tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { + async overview({ tz = 'UTC', owner_user_id = null, now = Date.now() } = {}) { const zone = reportTimezone(tz) - const own = ownerPred(owner_user_id) + const own = ownerPred(owner_user_id, 2) const todayStart = zonedDayStartMs(now, zone) const yesterdayStart = zonedDayStartMs(now, zone, -1) const yesterdayEnd = Math.min(todayStart, yesterdayStart + (now - todayStart)) - const agg = (sinceMs, untilMs) => { - const where = ['created_at >= ?', 'created_at < ?'] + const agg = async (sinceMs, untilMs) => { + const where = ['created_at >= $1', 'created_at < $2'] const params = [new Date(sinceMs).toISOString(), new Date(untilMs).toISOString()] if (own.sql) { where.push(own.sql) params.push(...own.params) } - return this.db + return await this.db .prepare(` SELECT COUNT(*) AS requests, COALESCE(SUM(CASE WHEN ${ERROR_PRED} THEN 1 ELSE 0 END), 0) AS errors, @@ -790,10 +792,10 @@ export class UsageLogsView { .get(...params) } // +1ms so a row stamped exactly `now` still counts. - const today = agg(todayStart, now + 1) - const yesterday = agg(yesterdayStart, yesterdayEnd) - const active = agg(now - 5 * 60_000, now + 1) - const lastMinute = agg(now - 60_000, now + 1) + const today = await agg(todayStart, now + 1) + const yesterday = await agg(yesterdayStart, yesterdayEnd) + const active = await agg(now - 5 * 60_000, now + 1) + const lastMinute = await agg(now - 60_000, now + 1) const avg = (v) => (v == null ? null : Math.round(Number(v))) return { activeSessions: num(active.sessions), diff --git a/test/unit/error-class.test.mjs b/test/unit/error-class.test.mjs index 7a8260e93..2e822ac92 100644 --- a/test/unit/error-class.test.mjs +++ b/test/unit/error-class.test.mjs @@ -1,5 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' import { classifyRequestError, collectErrors, @@ -119,15 +120,14 @@ test('excludeErrorClassSql hides ingress auth without dropping oauth credential const sql = excludeErrorClassSql('auth') assert.match(sql, /invalid_api_key/) assert.match(sql, /NOT \(/) - assert.match(sql, /IFNULL\(error_code/) + assert.match(sql, /COALESCE\(error_code/) assert.match(sql, /upstream_auth_error/) }) test('default auth mute keeps policy 403s that already name a non-auth class', async () => { - const { DatabaseSync } = await import('node:sqlite') - const db = new DatabaseSync(':memory:') - db.exec('CREATE TABLE logs (status INTEGER, error_code TEXT, vm_id TEXT)') - const insert = db.prepare('INSERT INTO logs VALUES (?, ?, ?)') + const db = getDb() + await db.exec('CREATE TABLE logs (status INTEGER, error_code TEXT, vm_id TEXT)') + const insert = db.prepare('INSERT INTO logs VALUES ($1, $2, $3)') for (const row of [ [401, 'invalid_api_key', null], [403, null, null], @@ -136,11 +136,10 @@ test('default auth mute keeps policy 403s that already name a non-auth class', a [403, 'distill_blocked', null], [401, 'upstream_auth_error', null], ]) { - insert.run(...row) + await insert.run(...row) } - const kept = db - .prepare(`SELECT error_code FROM logs WHERE ${excludeErrorClassSql('auth')}`) - .all() - .map((r) => r.error_code) + const kept = (await db.prepare(`SELECT error_code FROM logs WHERE ${excludeErrorClassSql('auth')}`).all()).map( + (r) => r.error_code, + ) assert.deepEqual(kept.sort(), ['claude_code_only', 'distill_blocked', 'key_group_mismatch', 'upstream_auth_error']) }) diff --git a/test/unit/panel-billing-lookup.test.mjs b/test/unit/panel-billing-lookup.test.mjs index ac849a5bc..80faaa516 100644 --- a/test/unit/panel-billing-lookup.test.mjs +++ b/test/unit/panel-billing-lookup.test.mjs @@ -30,7 +30,7 @@ test('billing index keeps UUID row when leftover vm-id account is empty', () => assert.equal(lookupBilling(index, { id: 'vm-50' }), real) }) -test('billing rows of an older account on a reused slot keep their own email', (t) => { +test('billing rows of an older account on a reused slot keep their own email', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-billing-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) const billing = { @@ -48,7 +48,7 @@ test('billing rows of an older account on a reused slot keep their own email', ( ], }), } - const usage = buildUsage({ + const usage = await buildUsage({ accountQuota, cfg: { paths: { project: root } }, requestLog: { billingStats: () => billing }, @@ -95,12 +95,12 @@ test('VM list carries lifetime tokens from the same billing row as total_cost', assert.equal(byId['vm-08'].total_tokens, 0) }) -test('usage rows of a deleted VM are not listed', (t) => { +test('usage rows of a deleted VM are not listed', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-usage-deleted-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) fs.mkdirSync(path.join(root, 'vms'), { recursive: true }) fs.writeFileSync(path.join(root, 'vms', 'vm-02.json'), JSON.stringify({ id: 'vm-02' })) - const usage = buildUsage({ + const usage = await buildUsage({ accountQuota: { snapshot: () => ({ accounts: [ @@ -122,7 +122,7 @@ test('usage rows of a deleted VM are not listed', (t) => { assert.equal(usage.data.billing.accounts[0].email, 'three@example.com') }) -test('GPT usage row without pool email takes slot email', (t) => { +test('GPT usage row without pool email takes slot email', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-gpt-usage-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) fs.mkdirSync(path.join(root, 'vms'), { recursive: true }) @@ -137,7 +137,7 @@ test('GPT usage row without pool email takes slot email', (t) => { codex: { email: 'lemeryvorhees@gmail.com' }, }), ) - const usage = buildUsage({ + const usage = await buildUsage({ accountQuota: { snapshot: () => ({ accounts: [{ account_id: 'vm-codex-01', vm_id: 'vm-codex-01', email: null }], diff --git a/test/unit/request-log.test.mjs b/test/unit/request-log.test.mjs index 333e4ddc7..1c0542724 100644 --- a/test/unit/request-log.test.mjs +++ b/test/unit/request-log.test.mjs @@ -48,14 +48,14 @@ test('summarizeBody counts messages and tools', () => { assert.equal(s.system_len, 10) }) -test('normal mode writes jsonl without body', () => { +test('normal mode writes jsonl without body', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: { 'user-agent': 't' }, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) assert.equal(ctx.mode, 'normal') - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, model: 'claude-haiku-4-5-20251001', stream: false, @@ -66,20 +66,20 @@ test('normal mode writes jsonl without body', () => { assert.equal(sum.status, 200) assert.equal(sum.api_key_id, 'key_1') assert.equal(sum.inbound_body, undefined) - const listed = store.listNormal({ limit: 5 }) + const listed = await store.listNormal({ limit: 5 }) assert.equal(listed.length, 1) assert.equal(listed[0].request_id, ctx.request_id) // no debug file - assert.equal(store.listDebug({ limit: 5 }).length, 0) + assert.equal((await store.listDebug({ limit: 5 })).length, 0) }) -test('summary persists protocol usage detail (cache/model/ttft/stop_reason)', () => { +test('summary persists protocol usage detail (cache/model/ttft/stop_reason)', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, model: 'sonnet', requested_model: 'sonnet', @@ -106,28 +106,28 @@ test('summary persists protocol usage detail (cache/model/ttft/stop_reason)', () // Sonnet 5 official: $2/$10/cache 5m $2.50/1h $4/read $0.20 per MTok assert.equal(sum.pricing_model, 'claude-sonnet-5') assert.ok(Math.abs(sum.total_cost - 0.0000926) < 1e-12) - const row = store.queryNormal({ limit: 1 }).items[0] + const row = (await store.queryNormal({ limit: 1 })).items[0] assert.equal(row.cache_creation_5m_tokens, 4) assert.equal(row.upstream_model, 'claude-sonnet-5') assert.equal(row.stop_reason, 'end_turn') assert.equal(row.total_cost, 0.0000926) }) -test('backfillMissingCosts restores actual cost with the stored group multiplier', () => { +test('backfillMissingCosts restores actual cost with the stored group multiplier', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs ( id, request_id, created_at, model, upstream_model, status, input_tokens, output_tokens, rate_multiplier - ) VALUES (?, ?, ?, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, ?) + ) VALUES ($1, $2, $3, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, $4) `) const now = new Date().toISOString() - insert.run('log_old_bill_default', 'rid_old_bill_default', now, null) - insert.run('log_old_bill_free', 'rid_old_bill_free', now, 0) - insert.run('log_old_bill_double', 'rid_old_bill_double', now, 2) + await insert.run('log_old_bill_default', 'rid_old_bill_default', now, null) + await insert.run('log_old_bill_free', 'rid_old_bill_free', now, 0) + await insert.run('log_old_bill_double', 'rid_old_bill_double', now, 2) - assert.equal(store.repo.backfillMissingCosts(), 3) - const rows = store.db + assert.equal(await store.repo.backfillMissingCosts(), 3) + const rows = await store.db .prepare( "SELECT id, total_cost, actual_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_old_bill_%' ORDER BY id", ) @@ -140,25 +140,32 @@ test('backfillMissingCosts restores actual cost with the stored group multiplier { id: 'log_old_bill_free', total_cost: 5, actual_cost: 0, pricing_model: 'claude-opus-5' }, ], ) - assert.equal(store.repo.backfillMissingCosts(), 0) + assert.equal(await store.repo.backfillMissingCosts(), 0) }) -function finishWith(store, model, usage) { +async function finishWith(store, model, usage) { const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { protocol: 'x', pathName: '/v1' }) - return store.finish(ctx, { status: 200, model, upstream_model: model, vm_id: 'vm-t', account_id: 'acc-t', usage }) + return await store.finish(ctx, { + status: 200, + model, + upstream_model: model, + vm_id: 'vm-t', + account_id: 'acc-t', + usage, + }) } -test('tier-unpriced rows stay unpriced after backfill', () => { +test('tier-unpriced rows stay unpriced after backfill', async () => { const store = tmpStore('normal') // gpt-5.5 publishes no >272K fast column; gpt-5.3-codex has no flex band. - const a = finishWith(store, 'gpt-5.5', { input_tokens: 1_000_000, output_tokens: 0, service_tier: 'priority' }) - const b = finishWith(store, 'gpt-5.3-codex', { input_tokens: 1000, output_tokens: 0, service_tier: 'flex' }) + const a = await finishWith(store, 'gpt-5.5', { input_tokens: 1_000_000, output_tokens: 0, service_tier: 'priority' }) + const b = await finishWith(store, 'gpt-5.3-codex', { input_tokens: 1000, output_tokens: 0, service_tier: 'flex' }) assert.equal(a.total_cost, null) assert.equal(a.pricing_model, 'unpriced') assert.equal(a.service_tier, 'priority') assert.equal(b.pricing_model, 'unpriced') - store.repo.billingStats() - const rows = store.db.prepare('SELECT total_cost, actual_cost, pricing_model FROM usage_logs ORDER BY id').all() + await store.repo.billingStats() + const rows = await store.db.prepare('SELECT total_cost, actual_cost, pricing_model FROM usage_logs ORDER BY id').all() for (const r of rows) { assert.equal(r.total_cost, null) assert.equal(r.actual_cost, null) @@ -166,36 +173,35 @@ test('tier-unpriced rows stay unpriced after backfill', () => { } }) -test('backfill reprices with the stored service tier and speed', () => { +test('backfill reprices with the stored service tier and speed', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs (id, request_id, created_at, model, upstream_model, status, input_tokens, output_tokens, service_tier, speed) - VALUES (?, ?, ?, ?, ?, 200, ?, 0, ?, ?) + VALUES ($1, $2, $3, $4, $5, 200, $6, 0, $7, $8) `) const now = new Date().toISOString() // 100K stays under the 272K long-context threshold → plain flex band ($2.5/MTok input). - insert.run('log_bf_flex', 'rid_bf_flex', now, 'gpt-5.5', 'gpt-5.5', 100_000, 'flex', null) - insert.run('log_bf_fast', 'rid_bf_fast', now, 'claude-opus-4-8', 'claude-opus-4-8', 1_000_000, null, 'fast') - insert.run('log_bf_weird', 'rid_bf_weird', now, 'gpt-5.5', 'gpt-5.5', 1000, 'weird', null) - store.repo.backfillMissingCosts() + await insert.run('log_bf_flex', 'rid_bf_flex', now, 'gpt-5.5', 'gpt-5.5', 100_000, 'flex', null) + await insert.run('log_bf_fast', 'rid_bf_fast', now, 'claude-opus-4-8', 'claude-opus-4-8', 1_000_000, null, 'fast') + await insert.run('log_bf_weird', 'rid_bf_weird', now, 'gpt-5.5', 'gpt-5.5', 1000, 'weird', null) + await store.repo.backfillMissingCosts() const got = Object.fromEntries( - store.db - .prepare("SELECT id, total_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_bf_%'") - .all() - .map((r) => [r.id, [r.total_cost, r.pricing_model]]), + (await store.db.prepare("SELECT id, total_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_bf_%'").all()).map( + (r) => [r.id, [r.total_cost, r.pricing_model]], + ), ) assert.deepEqual(got.log_bf_flex, [0.25, 'gpt-5.5']) assert.deepEqual(got.log_bf_fast, [10, 'claude-opus-4-8']) assert.deepEqual(got.log_bf_weird, [null, 'unpriced']) - assert.equal(store.repo.backfillMissingCosts(), 0) + assert.equal(await store.repo.backfillMissingCosts(), 0) }) -test('costByModel splits rows by billing band', () => { +test('costByModel splits rows by billing band', async () => { const store = tmpStore('normal') - finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0, speed: 'fast' }) - finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0 }) - const rows = store.repo.costByModel({ vmId: 'vm-t' }) + await finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0, speed: 'fast' }) + await finishWith(store, 'claude-opus-4-8', { input_tokens: 1_000_000, output_tokens: 0 }) + const rows = await store.repo.costByModel({ vmId: 'vm-t' }) assert.equal(rows.length, 2) const fast = rows.find((r) => r.speed === 'fast') const std = rows.find((r) => r.speed !== 'fast') @@ -205,23 +211,23 @@ test('costByModel splits rows by billing band', () => { assert.equal(std.long_context, 0) }) -test('vmUsageStats buckets by Shanghai day, isolates the slot and ranks models and paths', () => { +test('vmUsageStats buckets by Shanghai day, isolates the slot and ranks models and paths', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs (id, request_id, created_at, path, model, upstream_model, status, vm_id, input_tokens, output_tokens, total_cost, duration_ms) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 100, 50, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 100, 50, $9, $10) `) const shanghaiDay = (offsetDays) => new Date(Date.now() + 8 * 3600_000 - offsetDays * 86400_000).toISOString().slice(0, 10) // 16:30Z is 00:30 the next Shanghai day: a UTC bucket would file it under the wrong date. const at = (offsetDays) => `${shanghaiDay(offsetDays + 1)}T16:30:00.000Z` - insert.run('u1', 'r1', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 1000) - insert.run('u2', 'r2', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 3000) - insert.run('u3', 'r3', at(2), '/v1/chat/completions', 'm-b', 'm-b', 200, 'vm-x', 4, null) - insert.run('u4', 'r4', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-other', 99, 1) - insert.run('u5', 'r5', at(40), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 99, 1) - const out = store.repo.vmUsageStats({ vmId: 'vm-x', days: 30 }) + await insert.run('u1', 'r1', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 1000) + await insert.run('u2', 'r2', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 1, 3000) + await insert.run('u3', 'r3', at(2), '/v1/chat/completions', 'm-b', 'm-b', 200, 'vm-x', 4, null) + await insert.run('u4', 'r4', at(0), '/v1/messages', 'm-a', 'm-a', 200, 'vm-other', 99, 1) + await insert.run('u5', 'r5', at(40), '/v1/messages', 'm-a', 'm-a', 200, 'vm-x', 99, 1) + const out = await store.repo.vmUsageStats({ vmId: 'vm-x', days: 30 }) assert.deepEqual( out.history.map((h) => [h.day, h.requests, h.total_cost]), [ @@ -245,21 +251,21 @@ test('vmUsageStats buckets by Shanghai day, isolates the slot and ranks models a ['/v1/messages', '/v1/chat/completions'], ) assert.equal(out.endpoints[0].tokens, 300) - assert.deepEqual(store.repo.vmUsageStats({ vmId: null }).history, []) + assert.deepEqual((await store.repo.vmUsageStats({ vmId: null })).history, []) }) -test('keyUsageStats counts only that key and ranks VMs', () => { +test('keyUsageStats counts only that key and ranks VMs', async () => { const store = tmpStore('normal') const insert = store.db.prepare(` INSERT INTO usage_logs (id, request_id, created_at, path, model, upstream_model, status, vm_id, api_key_id, input_tokens, output_tokens, total_cost, duration_ms) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 100, 50, ?, ?) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 100, 50, $10, $11) `) const now = new Date().toISOString() - insert.run('k1', 'rk1', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_a', 1, 10) - insert.run('k2', 'rk2', now, '/v1/messages', 'm-b', 'm-b', 200, 'vm-b', 'key_a', 3, 20) - insert.run('k3', 'rk3', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_b', 99, 1) - const out = store.repo.keyUsageStats({ apiKeyId: 'key_a', days: 30 }) + await insert.run('k1', 'rk1', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_a', 1, 10) + await insert.run('k2', 'rk2', now, '/v1/messages', 'm-b', 'm-b', 200, 'vm-b', 'key_a', 3, 20) + await insert.run('k3', 'rk3', now, '/v1/messages', 'm-a', 'm-a', 200, 'vm-a', 'key_b', 99, 1) + const out = await store.repo.keyUsageStats({ apiKeyId: 'key_a', days: 30 }) assert.equal( out.history.reduce((n, row) => n + row.requests, 0), 2, @@ -276,22 +282,22 @@ test('keyUsageStats counts only that key and ranks VMs', () => { ], ) assert.equal(out.endpoints, undefined) - assert.deepEqual(store.repo.keyUsageStats({ apiKeyId: null }).history, []) + assert.deepEqual((await store.repo.keyUsageStats({ apiKeyId: null })).history, []) }) -test('zero group multiplier keeps usage and key counters but charges no USD', () => { +test('zero group multiplier keeps usage and key counters but charges no USD', async () => { const store = tmpStore('normal') const groups = new GroupsRepo(store.db) - groups.update(1, { rate_multiplier: 0 }) - const rateMultiplier = groups.rateMultiplier(1) - assert.equal(groups.getById(1).rate_multiplier, 0) + await groups.update(1, { rate_multiplier: 0 }) + const rateMultiplier = await groups.rateMultiplier(1) + assert.equal((await groups.getById(1)).rate_multiplier, 0) assert.equal(rateMultiplier, 0) const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - const summary = store.finish(ctx, { + const summary = await store.finish(ctx, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -302,8 +308,8 @@ test('zero group multiplier keeps usage and key counters but charges no USD', () assert.equal(summary.actual_cost, 0) const keys = new ApiKeyStore({ db: store.db, hashSecret: 'group-zero-test' }) - const key = keys.create({ name: 'free-group', quota: 100 }) - keys.recordUsage( + const key = await keys.create({ name: 'free-group', quota: 100 }) + await keys.recordUsage( key.id, { input_tokens: 1_000_000 }, { @@ -311,21 +317,21 @@ test('zero group multiplier keeps usage and key counters but charges no USD', () rateMultiplier, }, ) - const stored = keys.getById(key.id) + const stored = await keys.getById(key.id) assert.equal(stored.requests, 1) assert.equal(stored.tokens_in, 1_000_000) assert.equal(stored.quota_used, 0) assert.equal(stored.usage_5h, 0) }) -test('billingStats Extra window excludes older logs outside reset-5h', () => { +test('billingStats Extra window excludes older logs outside reset-5h', async () => { const store = tmpStore('normal') const now = Date.parse('2026-09-26T12:00:00.000Z') const ctxOld = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctxOld, { + await store.finish(ctxOld, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -333,12 +339,14 @@ test('billingStats Extra window excludes older logs outside reset-5h', () => { account_id: 'acc-1', usage: { input_tokens: 1_000_000, output_tokens: 0 }, }) - store.repo.db.prepare('UPDATE usage_logs SET created_at = ? WHERE vm_id = ?').run('2026-09-26T08:00:00.000Z', 'vm-01') + await store.repo.db + .prepare('UPDATE usage_logs SET created_at = $1 WHERE vm_id = $2') + .run('2026-09-26T08:00:00.000Z', 'vm-01') const ctxNew = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctxNew, { + await store.finish(ctxNew, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -346,30 +354,30 @@ test('billingStats Extra window excludes older logs outside reset-5h', () => { account_id: 'acc-1', usage: { input_tokens: 200_000, output_tokens: 0 }, }) - store.repo.db + await store.repo.db .prepare( "UPDATE usage_logs SET created_at = '2026-09-26T11:30:00.000Z' WHERE created_at > '2026-09-26T10:00:00.000Z'", ) .run() const reset5h = '2026-09-26T16:00:00.000Z' - const bill = store.billingStats({ + const bill = await store.billingStats({ now, accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-01', reset_5h: reset5h }], }) assert.equal(bill.accounts[0].window_5h_requests, 1) assert.ok(bill.accounts[0].window_5h_cost < 2) assert.equal(bill.accounts[0].window_5h_cost, bill.window_5h.total_cost) - const rolling = store.billingStats({ now }) + const rolling = await store.billingStats({ now }) assert.equal(rolling.accounts[0].window_5h_requests, 2) }) -test('billingStats aggregates official cost per account and today', () => { +test('billingStats aggregates official cost per account and today', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctx, { + await store.finish(ctx, { status: 200, model: 'claude-opus-5', upstream_model: 'claude-opus-5', @@ -377,7 +385,7 @@ test('billingStats aggregates official cost per account and today', () => { account_id: 'acc-1', usage: { input_tokens: 1_000_000, output_tokens: 0 }, }) - const bill = store.billingStats() + const bill = await store.billingStats() assert.equal(bill.currency, 'USD') assert.equal(bill.total.total_cost, 5) assert.equal(bill.today.total_cost, 5) @@ -393,20 +401,20 @@ test('billingStats aggregates official cost per account and today', () => { assert.equal(bill.accounts[0].window_7d_success, 1) assert.equal(bill.accounts[0].window_7d_errors, 0) assert.ok(bill.window_7d) - const models = store.costByModel({ vmId: 'vm-01' }) + const models = await store.costByModel({ vmId: 'vm-01' }) assert.equal(models.length, 1) assert.equal(models[0].model, 'claude-opus-5') assert.equal(models[0].total_cost, 5) assert.equal(models[0].input_tokens, 1_000_000) }) -test('finish writes OpenAI Responses cache read and cache write', () => { +test('finish writes OpenAI Responses cache read and cache write', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'openai.responses', pathName: '/v1/responses' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.responses', model: 'gpt-5.6-sol', @@ -422,13 +430,13 @@ test('finish writes OpenAI Responses cache read and cache write', () => { assert.equal(sum.cache_creation_tokens, 10) }) -test('finish prices OpenAI usage with top-level cached_tokens', () => { +test('finish prices OpenAI usage with top-level cached_tokens', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.chat', model: 'gpt-5.5', @@ -439,13 +447,13 @@ test('finish prices OpenAI usage with top-level cached_tokens', () => { assert.equal(sum.cache_read_tokens, 200_000) }) -test('finish prices OpenAI-shaped usage from third-party clients', () => { +test('finish prices OpenAI-shaped usage from third-party clients', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: { 'user-agent': 'OpenAI/Python 1.70.0' }, socket: {} }, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.chat', model: 'claude-sonnet-5', @@ -460,9 +468,9 @@ test('finish prices OpenAI-shaped usage from third-party clients', () => { assert.equal(sum.cache_read_tokens, 1_000_000) }) -test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h', () => { +test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h', async () => { const store = tmpStore('normal') - const five = store.finish( + const five = await store.finish( store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, @@ -477,7 +485,7 @@ test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h assert.equal(five.cache_creation_5m_tokens, 9) assert.equal(five.cache_creation_1h_tokens, 0) assert.equal(five.model_mismatch, 0) - const hour = store.finish( + const hour = await store.finish( store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, @@ -493,9 +501,9 @@ test('cache breakdown without a split bills 5m unless cache_ttl is explicitly 1h assert.equal(hour.cache_creation_1h_tokens, 9) }) -test('finish persists web_search_requests from server_tool_use', () => { +test('finish persists web_search_requests from server_tool_use', async () => { const store = tmpStore('normal') - const sum = store.finish( + const sum = await store.finish( store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, @@ -512,31 +520,32 @@ test('finish persists web_search_requests from server_tool_use', () => { }, ) assert.equal(sum.web_search_requests, 3) - const row = store.db.prepare('SELECT web_search_requests FROM usage_logs WHERE id = ?').get(sum.id) + const row = await store.db.prepare('SELECT web_search_requests FROM usage_logs WHERE id = $1').get(sum.id) assert.equal(row.web_search_requests, 3) }) -test('repriceRange rewrites total_cost and fills unpriced rows while actual_cost stays put', () => { +test('repriceRange rewrites total_cost and fills unpriced rows while actual_cost stays put', async () => { const store = tmpStore('normal') const now = new Date().toISOString() - store.db + await store.db .prepare(` INSERT INTO usage_logs ( id, request_id, created_at, model, upstream_model, status, input_tokens, output_tokens, total_cost, actual_cost, pricing_model, rate_multiplier ) VALUES - ('log_rp_old', 'rid_rp_old', ?, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, 1, 9, 'opus-5', 1), - ('log_rp_null', 'rid_rp_null', ?, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, NULL, 4, NULL, 2) + ('log_rp_old', 'rid_rp_old', $1, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, 1, 9, 'opus-5', 1), + ('log_rp_null', 'rid_rp_null', $2, 'claude-opus-5', 'claude-opus-5', 200, 1000000, 0, NULL, 4, NULL, 2) `) .run(now, now) - const out = store.repo.repriceRange({ since: now, until: now }) + const out = await store.repo.repriceRange({ since: now, until: now }) assert.equal(out.rows, 2) assert.ok(out.changed >= 1) const rows = Object.fromEntries( - store.db - .prepare("SELECT id, total_cost, actual_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_rp_%'") - .all() - .map((r) => [r.id, r]), + ( + await store.db + .prepare("SELECT id, total_cost, actual_cost, pricing_model FROM usage_logs WHERE id LIKE 'log_rp_%'") + .all() + ).map((r) => [r.id, r]), ) assert.equal(rows.log_rp_old.actual_cost, 9) assert.notEqual(rows.log_rp_old.total_cost, 1) @@ -545,13 +554,13 @@ test('repriceRange rewrites total_cost and fills unpriced rows while actual_cost assert.equal(rows.log_rp_null.total_cost, 5) }) -test('finish records washed Responses path over inbound chat path', () => { +test('finish records washed Responses path over inbound chat path', async () => { const store = tmpStore('normal') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }, ) - const sum = store.finish(ctx, { + const sum = await store.finish(ctx, { status: 200, protocol: 'openai.responses', path: '/v1/responses', @@ -561,18 +570,18 @@ test('finish records washed Responses path over inbound chat path', () => { assert.equal(sum.protocol, 'openai.responses') }) -test('debug mode stores full redacted body', () => { +test('debug mode stores full redacted body', async () => { const store = tmpStore('normal') const req = { method: 'POST', headers: { 'x-kin-debug': '1', authorization: 'Bearer sk-kin-abc' }, socket: {} } const ctx = store.start(req, { protocol: 'openai.chat', pathName: '/v1/chat/completions' }) assert.equal(ctx.mode, 'debug') - store.finish(ctx, { + await store.finish(ctx, { status: 200, model: 'm', inbound_body: { model: 'm', messages: [{ role: 'user', content: 'hi sk-ant-oat01-ABCDEFGH12345678' }] }, hop_meta: { params: { dropped: ['max_tokens'] } }, }) - const dbg = store.listDebug({ limit: 5 }) + const dbg = await store.listDebug({ limit: 5 }) assert.equal(dbg.length, 1) assert.equal(dbg[0].request_id, ctx.request_id) assert.ok(dbg[0].headers.authorization === '***REDACTED***') @@ -582,38 +591,38 @@ test('debug mode stores full redacted body', () => { assert.deepEqual(dbg[0].hop_meta.params.dropped, ['max_tokens']) }) -test('outbound bodies are stored only when the operator configured debug, not via caller headers', () => { +test('outbound bodies are stored only when the operator configured debug, not via caller headers', async () => { const outbound = { body: { system: 'gateway persona overlay' }, headers: { 'x-app': 'cli' } } - const record = (store, headers) => { + const record = async (store, headers) => { const ctx = store.start({ method: 'POST', headers, socket: {} }, { pathName: '/v1/messages' }) - store.finish(ctx, { + await store.finish(ctx, { status: 200, inbound_body: { model: 'm' }, outbound_body: outbound.body, outbound_headers: outbound.headers, }) - return store.getDebug(ctx.request_id) + return await store.getDebug(ctx.request_id) } for (const headers of [{ 'x-kin-debug': '1' }, { 'x-kin-log': 'debug' }]) { - const forced = record(tmpStore('normal'), headers) + const forced = await record(tmpStore('normal'), headers) assert.deepEqual(forced.inbound_body, { model: 'm' }) assert.equal(forced.outbound_body, null) assert.equal(forced.outbound_headers, null) } - const operator = record(tmpStore('debug'), {}) + const operator = await record(tmpStore('debug'), {}) assert.deepEqual(operator.outbound_body, outbound.body) assert.deepEqual(operator.outbound_headers, outbound.headers) }) -test('off mode writes nothing', () => { +test('off mode writes nothing', async () => { const store = tmpStore('off') const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: '/v1/messages' }) assert.equal(ctx.mode, 'off') - assert.equal(store.finish(ctx, { status: 200 }), null) - assert.equal(store.listNormal({ limit: 5 }).length, 0) + assert.equal(await store.finish(ctx, { status: 200 }), null) + assert.equal((await store.listNormal({ limit: 5 })).length, 0) }) -test('classifier debug diagnostics omit system and transcript bodies', () => { +test('classifier debug diagnostics omit system and transcript bodies', async () => { const store = tmpStore('debug') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, @@ -631,8 +640,8 @@ test('classifier debug diagnostics omit system and transcript bodies', () => { wire_observed: false, before: { max_tokens: 64 }, } - store.finish(ctx, { status: 200, classifier, inbound_body: body, outbound_body: body }) - const [debug] = store.listDebug({ limit: 1 }) + await store.finish(ctx, { status: 200, classifier, inbound_body: body, outbound_body: body }) + const [debug] = await store.listDebug({ limit: 1 }) assert.deepEqual(debug.classifier, classifier) assert.equal(debug.request_body_snapshot, null) assert.equal(debug.inbound_body, null) @@ -640,38 +649,38 @@ test('classifier debug diagnostics omit system and transcript bodies', () => { assert.equal(JSON.stringify(debug).includes('sensitive transcript'), false) }) -function logOne(store, extra = {}) { +async function logOne(store, extra = {}) { const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: extra.path || '/v1/messages' }) - return store.finish(ctx, { status: 200, ...extra }) + return await store.finish(ctx, { status: 200, ...extra }) } -test('summaries persist in sqlite across store re-open', () => { +test('summaries persist in PostgreSQL across store recreation', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rlog-')) const s1 = new RequestLogStore({ dataDir: dir, mode: 'normal' }) - logOne(s1, { model: 'claude-haiku-4-5', api_key_id: 'key_p' }) + await logOne(s1, { model: 'claude-haiku-4-5', api_key_id: 'key_p' }) const s2 = new RequestLogStore({ dataDir: dir, mode: 'normal' }) - const listed = s2.listNormal({ limit: 10 }) + const listed = await s2.listNormal({ limit: 10 }) assert.equal(listed.length, 1) assert.equal(listed[0].api_key_id, 'key_p') - assert.equal(s2.snapshot().total_rows, 1) + assert.equal((await s2.snapshot()).total_rows, 1) }) -test('gate stats count every inbound request and keep the stored verdict', () => { +test('gate stats count every inbound request and keep the stored verdict', async () => { const store = tmpStore() // Plain successes: error_code and final_state stay NULL. - logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'jev' }) }) - logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) - logOne(store, { path: '/v1/responses', intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) - logOne(store, { status: 401, error_code: 'invalid_api_key' }) - logOne(store, { + await logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'jev' }) }) + await logOne(store, { intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) + await logOne(store, { path: '/v1/responses', intercept: gateVerdict({ kind: 'pass', by: 'regex' }) }) + await logOne(store, { status: 401, error_code: 'invalid_api_key' }) + await logOne(store, { status: 403, via: 'distill-detect', final_state: 'distill_blocked', error_code: 'distill_blocked', intercept: gateVerdict({ kind: 'block', by: 'distill', keyword: 'Memory-stage-one extractor' }), }) - logOne(store, { + await logOne(store, { status: 503, via: 'refusal-guard', final_state: 'refusal_device', @@ -679,7 +688,7 @@ test('gate stats count every inbound request and keep the stored verdict', () => intercept: gateVerdict({ kind: 'block', by: 'refusal', keyword: 'device' }), }) - const stats = store.protocolEntryStats({ since: '1970-01-01T00:00:00.000Z' }) + const stats = await store.protocolEntryStats({ since: '1970-01-01T00:00:00.000Z' }) assert.equal(stats.total, 6) assert.equal(stats.blocked, 2) assert.equal(stats.passed, 4) @@ -689,27 +698,27 @@ test('gate stats count every inbound request and keep the stored verdict', () => assert.deepEqual(blockBy, { 'distill:Memory-stage-one extractor': 1, 'refusal:device': 1 }) }) -test('queryNormal filters + pagination + total', () => { +test('queryNormal filters + pagination + total', async () => { const store = tmpStore('normal') - logOne(store, { api_key_id: 'key_a', model: 'm1', input_tokens: 10, output_tokens: 5 }) - logOne(store, { api_key_id: 'key_a', model: 'm2', status: 500, error_code: 'upstream_error' }) - logOne(store, { api_key_id: 'key_b', model: 'm1', vm_id: 'vm-9' }) + await logOne(store, { api_key_id: 'key_a', model: 'm1', input_tokens: 10, output_tokens: 5 }) + await logOne(store, { api_key_id: 'key_a', model: 'm2', status: 500, error_code: 'upstream_error' }) + await logOne(store, { api_key_id: 'key_b', model: 'm1', vm_id: 'vm-9' }) - assert.equal(store.queryNormal({ api_key_id: 'key_a' }).total, 2) - assert.equal(store.queryNormal({ vm_id: 'vm-9' }).total, 1) - assert.equal(store.queryNormal({ model: 'm1' }).total, 2) - assert.equal(store.queryNormal({ status: 'error' }).total, 1) - assert.equal(store.queryNormal({ status: 'ok' }).total, 2) - assert.equal(store.queryNormal({ status: 'error', include_muted: true }).total, 1) - assert.equal(store.queryNormal({ q: 'upstream' }).total, 1) - const page = store.queryNormal({ limit: 2, offset: 2 }) + assert.equal((await store.queryNormal({ api_key_id: 'key_a' })).total, 2) + assert.equal((await store.queryNormal({ vm_id: 'vm-9' })).total, 1) + assert.equal((await store.queryNormal({ model: 'm1' })).total, 2) + assert.equal((await store.queryNormal({ status: 'error' })).total, 1) + assert.equal((await store.queryNormal({ status: 'ok' })).total, 2) + assert.equal((await store.queryNormal({ status: 'error', include_muted: true })).total, 1) + assert.equal((await store.queryNormal({ q: 'upstream' })).total, 1) + const page = await store.queryNormal({ limit: 2, offset: 2 }) assert.equal(page.total, 3) assert.equal(page.items.length, 1) }) -test('windowStats computes ttft percentiles, sla and qps', () => { +test('windowStats computes ttft percentiles, sla and qps', async () => { const store = tmpStore('normal') - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 100, duration_ms: 200, @@ -719,7 +728,7 @@ test('windowStats computes ttft percentiles, sla and qps', () => { cache_creation_tokens: 10, model: 'claude-sonnet-5', }) - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 300, duration_ms: 400, @@ -729,8 +738,8 @@ test('windowStats computes ttft percentiles, sla and qps', () => { cache_creation_tokens: 0, model: 'claude-sonnet-5', }) - logOne(store, { status: 503, error_code: 'overloaded', duration_ms: 50, model: 'claude-fable-5' }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + await logOne(store, { status: 503, error_code: 'overloaded', duration_ms: 50, model: 'claude-fable-5' }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.cache_read_tokens, 80) assert.equal(w.prompt_tokens, 110) assert.ok(Math.abs(w.cache_hit_rate - 80 / 110) < 1e-9) @@ -747,21 +756,21 @@ test('windowStats computes ttft percentiles, sla and qps', () => { assert.equal(w.by_model.length, 2) assert.equal(w.error_collection.total, 1) assert.equal(w.error_collection.by_class[0].id, 'overloaded') - const filtered = store.queryNormal({ error_class: 'overloaded' }) + const filtered = await store.queryNormal({ error_class: 'overloaded' }) assert.equal(filtered.total, 1) assert.equal(filtered.items[0].error_label, '过载排队') }) -test('windowStats treats 429 quota/rate-limit as SLA success', () => { +test('windowStats treats 429 quota/rate-limit as SLA success', async () => { const store = tmpStore('normal') - logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) + await logOne(store, { status: 429, error_code: 'upstream_rate_limit', error_message: '5h extra usage', model: 'claude-opus-5', }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.requests, 2) assert.equal(w.success, 2) assert.equal(w.errors, 0) @@ -771,22 +780,22 @@ test('windowStats treats 429 quota/rate-limit as SLA success', () => { assert.equal(w.error_collection.by_class[0].id, 'rate_limit') }) -test('windowStats treats ingress auth failures as SLA success like sub2api', () => { +test('windowStats treats ingress auth failures as SLA success like sub2api', async () => { const store = tmpStore('normal') - logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) - logOne(store, { + await logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) + await logOne(store, { status: 401, error_code: 'auth_failed', error_message: 'missing or invalid credentials', model: 'claude-sonnet-5', }) - logOne(store, { + await logOne(store, { status: 401, error_code: 'invalid_api_key', error_message: 'Invalid credentials', model: 'claude-opus-5', }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.requests, 3) assert.equal(w.success, 3) assert.equal(w.errors, 0) @@ -796,54 +805,59 @@ test('windowStats treats ingress auth failures as SLA success like sub2api', () assert.equal(w.error_collection.by_class[0].owner, 'client') }) -test('windowStats ignores client abort like sub2api ignore_context_canceled', () => { +test('windowStats ignores client abort like sub2api ignore_context_canceled', async () => { const store = tmpStore('normal') - logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) - logOne(store, { status: 200, error_code: 'client_cancelled', error_message: 'ECONNRESET', model: 'claude-sonnet-5' }) - const w = store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) + await logOne(store, { status: 200, first_token_ms: 80, duration_ms: 100, model: 'claude-sonnet-5' }) + await logOne(store, { + status: 200, + error_code: 'client_cancelled', + error_message: 'ECONNRESET', + model: 'claude-sonnet-5', + }) + const w = await store.windowStats({ since: new Date(Date.now() - 60_000).toISOString() }) assert.equal(w.requests, 2) assert.equal(w.success, 2) assert.equal(w.errors, 0) assert.equal(w.error_collection.total, 0) - assert.equal(store.queryNormal({ status: 'error' }).total, 0) + assert.equal((await store.queryNormal({ status: 'error' })).total, 0) }) -test('aggregate buckets by day with token sums', () => { +test('aggregate buckets by day with token sums', async () => { const store = tmpStore('normal') - logOne(store, { input_tokens: 10, output_tokens: 5 }) - logOne(store, { input_tokens: 20, output_tokens: 15, status: 500, error_code: 'x' }) - const rows = store.aggregate({ bucket: 'day' }) + await logOne(store, { input_tokens: 10, output_tokens: 5 }) + await logOne(store, { input_tokens: 20, output_tokens: 15, status: 500, error_code: 'x' }) + const rows = await store.aggregate({ bucket: 'day' }) assert.equal(rows.length, 1) assert.equal(rows[0].requests, 2) assert.equal(rows[0].errors, 1) assert.equal(rows[0].input_tokens, 30) assert.equal(rows[0].output_tokens, 20) - const totals = store.totals() + const totals = await store.totals() assert.equal(totals.requests, 2) assert.equal(totals.input_tokens, 30) }) -test('cleanup removes rows older than retainDays', () => { +test('cleanup removes rows older than retainDays', async () => { const store = tmpStore('normal') // one fresh row - logOne(store, {}) + await logOne(store, {}) // one stale row injected directly - store.repo.insertSummary({ + await store.repo.insertSummary({ id: 'log_old', request_id: 'rid-old', ts: new Date(Date.now() - 30 * 86400_000).toISOString(), }) - store.repo.insertDebug('rid-old', new Date(Date.now() - 30 * 86400_000).toISOString(), { old: true }) - assert.equal(store.repo.count(), 2) - store.cleanup() - assert.equal(store.repo.count(), 1) - assert.equal(store.getDebug('rid-old'), null) + await store.repo.insertDebug('rid-old', new Date(Date.now() - 30 * 86400_000).toISOString(), { old: true }) + assert.equal(await store.repo.count(), 2) + await store.cleanup() + assert.equal(await store.repo.count(), 1) + assert.equal(await store.getDebug('rid-old'), null) }) -test('jsonl mirror writes legacy format when enabled', () => { +test('jsonl mirror writes legacy format when enabled', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rlog-')) const store = new RequestLogStore({ dataDir: dir, mode: 'normal', jsonlMirror: true }) - logOne(store, { model: 'mm' }) + await logOne(store, { model: 'mm' }) const day = new Date().toISOString().slice(0, 10) const file = path.join(dir, 'request-logs', `${day}.jsonl`) assert.ok(fs.existsSync(file)) @@ -880,46 +894,46 @@ test('sanitizeRequestBodySnapshot omits image bytes', async () => { assert.equal(JSON.stringify(snap).includes(payload), false) }) -test('setConfig hot-updates mode', () => { +test('setConfig hot-updates mode', async () => { const s = tmpStore('normal') - s.setConfig({ mode: 'debug', retainDays: 3 }) - const snap = s.snapshot() + await s.setConfig({ mode: 'debug', retainDays: 3 }) + const snap = await s.snapshot() assert.equal(snap.mode, 'debug') assert.equal(snap.retain_days, 3) assert.equal(snap.debug_retain_days, 3) assert.equal(snap.max_mb, 2048) assert.deepEqual(snap.muted_error_classes, ['auth']) - s.setConfig({ mutedErrorClasses: [] }) - assert.deepEqual(s.snapshot().muted_error_classes, []) + await s.setConfig({ mutedErrorClasses: [] }) + assert.deepEqual((await s.snapshot()).muted_error_classes, []) }) -test('cleanup keeps 7-day summaries and drops 3-day-old debug', () => { +test('cleanup keeps 7-day summaries and drops 3-day-old debug', async () => { const store = tmpStore('normal') - store.setConfig({ retainDays: 7, debugRetainDays: 3, maxMb: 0 }) + await store.setConfig({ retainDays: 7, debugRetainDays: 3, maxMb: 0 }) const fiveDaysAgo = new Date(Date.now() - 5 * 86400_000).toISOString() - store.repo.insertSummary({ + await store.repo.insertSummary({ id: 'log_mid', request_id: 'rid-mid', ts: fiveDaysAgo, }) - store.repo.insertDebug('rid-mid', fiveDaysAgo, { old: true }) - store.cleanup() - assert.equal(store.repo.count(), 1) - assert.equal(store.repo.getDebug('rid-mid'), null) + await store.repo.insertDebug('rid-mid', fiveDaysAgo, { old: true }) + await store.cleanup() + assert.equal(await store.repo.count(), 1) + assert.equal(await store.repo.getDebug('rid-mid'), null) }) -test('cleanup trims oldest debug rows when over max_mb', () => { +test('cleanup trims oldest debug rows when over max_mb', async () => { const store = tmpStore('debug') - store.setConfig({ retainDays: 7, debugRetainDays: 7, maxMb: 1 }) + await store.setConfig({ retainDays: 7, debugRetainDays: 7, maxMb: 1 }) const blob = 'x'.repeat(400_000) for (let i = 0; i < 4; i++) { - store.repo.insertDebug(`rid-${i}`, new Date(Date.now() - (4 - i) * 1000).toISOString(), { blob }) + await store.repo.insertDebug(`rid-${i}`, new Date(Date.now() - (4 - i) * 1000).toISOString(), { blob }) } - assert.ok(store.repo.debugBytes() > 1024 * 1024) - store.cleanup() - assert.ok(store.repo.debugBytes() <= 1024 * 1024) - assert.equal(store.getDebug('rid-0'), null) - assert.ok(store.getDebug('rid-3')) + assert.ok((await store.repo.debugBytes()) > 1024 * 1024) + await store.cleanup() + assert.ok((await store.repo.debugBytes()) <= 1024 * 1024) + assert.equal(await store.getDebug('rid-0'), null) + assert.ok(await store.getDebug('rid-3')) }) test('normalizeLoggingConfig defaults debug retain to 3 and caps at retain_days', () => { @@ -933,7 +947,7 @@ test('normalizeLoggingConfig defaults debug retain to 3 and caps at retain_days' assert.equal(b.max_mb, 512) }) -test('ingress 401 stores presented key + ip and can be excluded from the list', () => { +test('ingress 401 stores presented key + ip and can be excluded from the list', async () => { const store = tmpStore('normal') const ctx = store.start( { @@ -943,28 +957,28 @@ test('ingress 401 stores presented key + ip and can be excluded from the list', }, { protocol: 'anthropic.messages', pathName: '/v1/messages' }, ) - store.finish(ctx, { + await store.finish(ctx, { status: 401, error_code: 'invalid_api_key', error_message: 'Invalid credentials', api_key_presented: 'sk-bad-key-plain', }) - assert.equal(store.queryNormal({ status: 'error' }).total, 0) - const listed = store.queryNormal({ status: 'error', include_muted: true }) + assert.equal((await store.queryNormal({ status: 'error' })).total, 0) + const listed = await store.queryNormal({ status: 'error', include_muted: true }) assert.equal(listed.total, 1) assert.equal(listed.items[0].api_key_presented, 'sk-bad-key-plain') assert.equal(listed.items[0].ip, '203.0.113.9') assert.equal(listed.items[0].error_class, 'auth') - assert.equal(store.queryNormal({ status: 'error', error_class: 'auth' }).total, 1) - assert.equal(store.queryNormal({ status: 'error', exclude_error_class: 'auth' }).total, 0) - const hits = store.windowStats().error_collection.ingress_hits + assert.equal((await store.queryNormal({ status: 'error', error_class: 'auth' })).total, 1) + assert.equal((await store.queryNormal({ status: 'error', exclude_error_class: 'auth' })).total, 0) + const hits = (await store.windowStats()).error_collection.ingress_hits assert.equal(hits.length, 1) assert.equal(hits[0].api_key_presented, 'sk-bad-key-plain') assert.equal(hits[0].ip, '203.0.113.9') assert.equal(hits[0].count, 1) }) -test('exportRows + csv/jsonl keep presented key and ip', () => { +test('exportRows + csv/jsonl keep presented key and ip', async () => { const store = tmpStore('normal') const ctx = store.start( { @@ -974,13 +988,13 @@ test('exportRows + csv/jsonl keep presented key and ip', () => { }, { pathName: '/v1/messages' }, ) - store.finish(ctx, { + await store.finish(ctx, { status: 401, error_code: 'invalid_api_key', error_message: 'Invalid credentials', api_key_presented: 'sk-export-plain', }) - const { items, total } = store.exportRows({ status: 'error', include_muted: true, limit: 100 }) + const { items, total } = await store.exportRows({ status: 'error', include_muted: true, limit: 100 }) assert.equal(total, 1) assert.equal(items[0].ip, '198.51.100.4') const csv = logsToCsv(items) @@ -991,14 +1005,14 @@ test('exportRows + csv/jsonl keep presented key and ip', () => { assert.match(jsonl, /sk-export-plain/) }) -test('finish persists caller and outbound sessions (capped) and only known reasoning efforts', () => { +test('finish persists caller and outbound sessions (capped) and only known reasoning efforts', async () => { const store = tmpStore('normal') - const write = (extra) => { + const write = async (extra) => { const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: '/v1/messages' }) - store.finish(ctx, { status: 200, ...extra }) - return store.repo.getByRequestId(ctx.request_id) + await store.finish(ctx, { status: 200, ...extra }) + return await store.repo.getByRequestId(ctx.request_id) } - const kept = write({ + const kept = await write({ session_id: ` ${'s'.repeat(250)} `, outbound_session_id: '11111111-2222-4333-8444-555555555555', reasoning_effort: 'xhigh', @@ -1006,7 +1020,7 @@ test('finish persists caller and outbound sessions (capped) and only known reaso assert.equal(kept.session_id, 's'.repeat(200)) assert.equal(kept.outbound_session_id, '11111111-2222-4333-8444-555555555555') assert.equal(kept.reasoning_effort, 'xhigh') - const dropped = write({ session_id: ' ', outbound_session_id: '', reasoning_effort: 'turbo' }) + const dropped = await write({ session_id: ' ', outbound_session_id: '', reasoning_effort: 'turbo' }) assert.equal(dropped.session_id, null) assert.equal(dropped.outbound_session_id, null) assert.equal(dropped.reasoning_effort, null) @@ -1037,7 +1051,7 @@ function fakeRes() { return res } -test('debug mode records the exact client response; normal mode leaves res untouched', () => { +test('debug mode records the exact client response; normal mode leaves res untouched', async () => { const store = new RequestLogStore({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rlog-')), mode: 'debug', @@ -1051,8 +1065,8 @@ test('debug mode records the exact client response; normal mode leaves res untou res.write('event: message_start\ndata: {"a":1}\n\n') res.write(Buffer.from('data: {"type":"content_block_delta","delta":{"text":"hello"}}\n\n')) res.end('data: [DONE]\n\n') - store.finish(ctx, { status: 200 }) - const rec = store.getDebug(ctx.request_id) + await store.finish(ctx, { status: 200 }) + const rec = await store.getDebug(ctx.request_id) assert.equal(rec.response.status, 200) assert.equal(rec.response.headers['content-type'], 'text/event-stream') assert.equal(rec.response.headers['set-cookie'], '***REDACTED***') @@ -1070,36 +1084,64 @@ test('debug mode records the exact client response; normal mode leaves res untou assert.equal(nctx.response, undefined) }) -test('debug response body masks credentials that fit inside the capture window', () => { +test('debug response body masks credentials that fit inside the capture window', async () => { const store = tmpStore('debug') const ctx = store.start({ method: 'POST', headers: {}, socket: {} }, { pathName: '/v1/messages' }) const res = fakeRes() store.tapResponse(ctx, res) res.end('{"echo":"sk-ant-oat01-ABCDEFGH12345678"}') - store.finish(ctx, { status: 200 }) - const body = store.getDebug(ctx.request_id).response.body + await store.finish(ctx, { status: 200 }) + const body = (await store.getDebug(ctx.request_id)).response.body assert.doesNotMatch(body, /ABCDEFGH12345678/) assert.match(body, /REDACTED/) }) -test('tenant log scope follows the slot owner at write time, not the current owner', () => { +test('tenant log scope follows the slot owner at write time, not the current owner', async () => { const store = tmpStore() - store.db.exec(` + await store.db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('alice', 'a@x', 'alice', 'x'), ('bob', 'b@x', 'bob', 'x'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'); `) - logOne(store, { user_id: 'bob', vm_id: 'vm-a', error_code: 'upstream_error', status: 500 }) - store.db.exec(`UPDATE vms SET owner_user_id = 'alice', origin = 'admin_assigned' WHERE id = 'vm-a'`) - const testChat = logOne(store, { api_key_kind: 'master', vm_id: 'vm-a', error_code: 'upstream_error', status: 500 }) + await logOne(store, { user_id: 'bob', vm_id: 'vm-a', error_code: 'upstream_error', status: 500 }) + await store.db.exec(`UPDATE vms SET owner_user_id = 'alice', origin = 'admin_assigned' WHERE id = 'vm-a'`) + const testChat = await logOne(store, { + api_key_kind: 'master', + vm_id: 'vm-a', + error_code: 'upstream_error', + status: 500, + }) - const mine = store.queryNormal({ owner_user_id: 'alice' }) + const mine = await store.queryNormal({ owner_user_id: 'alice' }) assert.deepEqual( mine.items.map((r) => r.request_id), [testChat.request_id], ) assert.deepEqual( - store.windowStats({ owner_user_id: 'alice' }).error_collection.recent.map((r) => r.request_id), + (await store.windowStats({ owner_user_id: 'alice' })).error_collection.recent.map((r) => r.request_id), [testChat.request_id], ) - assert.equal(store.queryNormal({}).total, 2) + assert.equal((await store.queryNormal({})).total, 2) +}) + +test('summary and debug write failures reject finish and response flush', async () => { + const store = tmpStore('debug') + const query = store.db.query.bind(store.db) + const failure = Object.assign(new Error('simulated persistence failure'), { code: 'TEST_PERSISTENCE_FAILED' }) + try { + store.db.query = (sql, params) => + sql.includes('INSERT INTO request_log_debug') ? Promise.reject(failure) : query(sql, params) + const debug = store.start({ method: 'POST', url: '/v1/messages', headers: {} }) + await assert.rejects(store.finish(debug, { status: 200 }), { code: 'TEST_PERSISTENCE_FAILED' }) + assert.ok(await store.repo.getByRequestId(debug.request_id), 'summary persists even when debug persistence fails') + + store.db.query = (sql, params) => + sql.includes('INSERT INTO usage_logs') ? Promise.reject(failure) : query(sql, params) + const { trackPendingWrite, flushPendingWrites } = await import('../../src/lib/db/pending-writes.mjs') + const normal = store.start({ method: 'POST', url: '/v1/messages', headers: {} }) + const pending = trackPendingWrite(store.finish(normal, { status: 200 })) + await assert.rejects(pending, { code: 'TEST_PERSISTENCE_FAILED' }) + await assert.rejects(flushPendingWrites(), { code: 'TEST_PERSISTENCE_FAILED' }) + } finally { + store.db.query = query + } }) diff --git a/test/unit/request-logs-export.test.mjs b/test/unit/request-logs-export.test.mjs index 86540db13..95b5f6461 100644 --- a/test/unit/request-logs-export.test.mjs +++ b/test/unit/request-logs-export.test.mjs @@ -9,23 +9,18 @@ import { UsageLogsView } from '../../src/lib/db/repos/usage-logs-view.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-log-export-')) -const db = openDatabase({ dbPath: path.join(root, 'logs.db') }) -test.after(() => { - closeDatabase() +let db +test.after(async () => { + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) }) -db.exec(` - INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); - INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); -`) - let seq = 0 -function seed(rows) { +async function seed(rows) { const repo = new UsageLogsRepo(db) for (const r of rows) { seq++ - repo.insertSummary({ + await repo.insertSummary({ id: `log_${String(seq).padStart(8, '0')}`, request_id: `req-${seq}`, created_at: new Date(Date.UTC(2026, 0, 1) + seq * 1000).toISOString(), @@ -43,15 +38,24 @@ function seed(rows) { } } -seed([ - { session_id: 's1', status: 200 }, - { session_id: 's1', status: 500, error_code: 'upstream_error' }, - { session_id: 's1', status: 500, error_code: 'upstream_error', path: '/v1/chat/completions' }, - { session_id: 's2', status: 500, error_code: 'upstream_error' }, - { session_id: 's1', status: 200, attempt_count: 3, log_mode: 'debug' }, - { session_id: 's1', status: 401, error_code: 'invalid_api_key', vm_id: null }, - { session_id: 's9', user_id: 'u2', api_key_id: 'k2', status: 500, error_code: 'upstream_error' }, -]) +test.beforeEach(async () => { + db = await openDatabase() + seq = 0 + await db.exec(` + INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); + INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); +`) + + await seed([ + { session_id: 's1', status: 200 }, + { session_id: 's1', status: 500, error_code: 'upstream_error' }, + { session_id: 's1', status: 500, error_code: 'upstream_error', path: '/v1/chat/completions' }, + { session_id: 's2', status: 500, error_code: 'upstream_error' }, + { session_id: 's1', status: 200, attempt_count: 3, log_mode: 'debug' }, + { session_id: 's1', status: 401, error_code: 'invalid_api_key', vm_id: null }, + { session_id: 's9', user_id: 'u2', api_key_id: 'k2', status: 500, error_code: 'upstream_error' }, + ]) +}) async function exportLogs(query, { role = 'admin', userId = null, headers = {} } = {}) { const out = {} @@ -97,9 +101,9 @@ test('export honours the list filters the page sends (session, status code, endp const { rows, headers } = await exportLogs(query) assert.deepEqual(ids(rows), ['req-2']) assert.equal(headers['x-kin-export-total'], '1') - const listed = new UsageLogsView(db) - .listBatch({ session_id: 's1', status_code: '500', endpoint: '/v1/messages' }) - .logs.map((r) => r.requestId) + const listed = ( + await new UsageLogsView(db).listBatch({ session_id: 's1', status_code: '500', endpoint: '/v1/messages' }) + ).logs.map((r) => r.requestId) assert.deepEqual(ids(rows), listed) }) diff --git a/test/unit/statistics-repo.test.mjs b/test/unit/statistics-repo.test.mjs index fca5d355b..25883fc85 100644 --- a/test/unit/statistics-repo.test.mjs +++ b/test/unit/statistics-repo.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { closeDatabase, createDatabase, openDatabase } from '../../src/lib/db/database.mjs' +import { closeDatabase, getDb, openDatabase } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' import { StatisticsRepo, resolveWindow } from '../../src/lib/db/repos/statistics-repo.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' @@ -11,26 +11,27 @@ import { codexBodyToAnthropicMessage } from '../../src/lib/protocol/codex-conver let seq = 0 -function seedUsers(db) { - db.exec(` +async function seedUsers(db) { + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'); `) } -function freshDb() { +async function freshDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-stats-')) - const db = createDatabase({ dbPath: path.join(dir, 'stats.db') }) - seedUsers(db) + const db = await getDb() + await db.exec('TRUNCATE users, api_keys, vms, usage_logs, request_attempts') + await seedUsers(db) return db } -function seed(db, rows) { +async function seed(db, rows) { const repo = new UsageLogsRepo(db) for (const r of rows) { seq++ - repo.insertSummary({ + await repo.insertSummary({ id: `log_${seq}`, request_id: `req-${seq}`, log_mode: 'normal', @@ -43,7 +44,7 @@ function seed(db, rows) { } } -test('day buckets follow local midnights across a DST change', () => { +test('day buckets follow local midnights across a DST change', async () => { const tz = 'America/New_York' const now = Date.parse('2026-03-09T15:00:00Z') // 11:00 EDT, the day after spring-forward const { buckets, resolution } = resolveWindow('7days', tz, now) @@ -53,12 +54,12 @@ test('day buckets follow local midnights across a DST change', () => { assert.equal(new Date(buckets[5]).toISOString(), '2026-03-08T05:00:00.000Z') assert.equal(new Date(buckets[6]).toISOString(), '2026-03-09T04:00:00.000Z') - const db = freshDb() - seed(db, [ + const db = await freshDb() + await seed(db, [ { created_at: '2026-03-09T03:30:00.000Z', total_cost: 1 }, // 23:30 EDT on Mar 8 { created_at: '2026-03-09T04:30:00.000Z', total_cost: 2 }, // 00:30 EDT on Mar 9 ]) - const data = new StatisticsRepo(db).statistics({ range: '7days', dimension: 'model', tz, now }) + const data = await new StatisticsRepo(db).statistics({ range: '7days', dimension: 'model', tz, now }) const cost = data.metrics.cost assert.equal(cost[5].date, '2026-03-08T05:00:00.000Z') assert.equal(cost[5].s0, 1) @@ -67,36 +68,36 @@ test('day buckets follow local midnights across a DST change', () => { assert.equal(data.since, '2026-03-03T05:00:00.000Z') }) -test('hourly buckets start at local midnight in half-hour zones; bad tz falls back to UTC', () => { - const db = freshDb() +test('hourly buckets start at local midnight in half-hour zones; bad tz falls back to UTC', async () => { + const db = await freshDb() const now = Date.parse('2026-05-01T00:00:00Z') // 05:30 in Kolkata - seed(db, [ + await seed(db, [ { created_at: '2026-04-30T18:29:59.000Z', total_cost: 100 }, // previous local day { created_at: '2026-04-30T19:29:00.000Z', total_cost: 1 }, // 00:59 local { created_at: '2026-04-30T19:30:00.000Z', total_cost: 2 }, // 01:00 local ]) const repo = new StatisticsRepo(db) - const data = repo.statistics({ range: 'today', dimension: 'model', tz: 'Asia/Kolkata', now }) + const data = await repo.statistics({ range: 'today', dimension: 'model', tz: 'Asia/Kolkata', now }) assert.equal(data.resolution, 'hour') assert.equal(data.metrics.cost.length, 6) assert.equal(data.metrics.cost[0].date, '2026-04-30T18:30:00.000Z') assert.equal(data.metrics.cost[0].s0, 1) assert.equal(data.metrics.cost[1].s0, 2) assert.equal(data.totals.cost, 3) - const utc = repo.statistics({ range: 'today', tz: 'Mars/Base', now }) + const utc = await repo.statistics({ range: 'today', tz: 'Mars/Base', now }) assert.equal(utc.tz, 'UTC') assert.equal(utc.since, '2026-05-01T00:00:00.000Z') }) -test('series keep the top 8 by cost, fold the rest into __others__, and name empty values', () => { - const db = freshDb() +test('series keep the top 8 by cost, fold the rest into __others__, and name empty values', async () => { + const db = await freshDb() const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T10:00:00.000Z' const rows = [] for (let i = 1; i <= 10; i++) rows.push({ created_at: at, model: `m${i}`, total_cost: i, input_tokens: 10 }) rows.push({ created_at: at, model: null, total_cost: 20 }) - seed(db, rows) - const data = new StatisticsRepo(db).statistics({ range: 'today', dimension: 'model', tz: 'UTC', now }) + await seed(db, rows) + const data = await new StatisticsRepo(db).statistics({ range: 'today', dimension: 'model', tz: 'UTC', now }) assert.deepEqual( data.series.map((s) => s.id), ['__none__', 'm10', 'm9', 'm8', 'm7', 'm6', 'm5', 'm4', '__others__'], @@ -114,28 +115,28 @@ test('series keep the top 8 by cost, fold the rest into __others__, and name emp assert.equal(data.totals.requests, 11) }) -test('deltas compare with the previous equal-length window', () => { - const db = freshDb() +test('deltas compare with the previous equal-length window', async () => { + const db = await freshDb() const now = Date.parse('2026-05-08T12:00:00Z') // 7days window = [2026-05-02, now); previous = same span ending 2026-05-02. - seed(db, [ + await seed(db, [ { created_at: '2026-05-05T00:00:00.000Z', total_cost: 3, input_tokens: 30 }, { created_at: '2026-05-06T00:00:00.000Z', total_cost: 3, input_tokens: 30 }, { created_at: '2026-04-28T00:00:00.000Z', total_cost: 2, input_tokens: 40 }, ]) - const data = new StatisticsRepo(db).statistics({ range: '7days', tz: 'UTC', now }) + const data = await new StatisticsRepo(db).statistics({ range: '7days', tz: 'UTC', now }) assert.equal(data.deltas.requests, 1) assert.equal(data.deltas.cost, 2) assert.equal(data.deltas.tokens, 0.5) - const empty = new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now }) + const empty = await new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now }) assert.equal(empty.deltas.requests, null) }) -test('leaderboard sorts by official cost, names entities, and caps the limit', () => { - const db = freshDb() +test('leaderboard sorts by official cost, names entities, and caps the limit', async () => { + const db = await freshDb() const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', @@ -150,7 +151,7 @@ test('leaderboard sorts by official cost, names entities, and caps the limit', ( { created_at: at, user_id: null, api_key_id: null, total_cost: 0.5 }, ]) const repo = new StatisticsRepo(db) - const board = repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now }) + const board = await repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now }) assert.deepEqual( board.entries.map((e) => [e.id, e.name]), [ @@ -166,11 +167,14 @@ test('leaderboard sorts by official cost, names entities, and caps the limit', ( assert.equal(alice.successRate, 0.5) assert.equal(alice.totalActualCost, 2) assert.equal(alice.cacheHitRate, 0.2) - assert.equal(repo.leaderboard({ range: 'today', scope: 'key', limit: 1, tz: 'UTC', now }).entries[0].name, 'bob-key') - assert.equal(repo.leaderboard({ range: 'today', scope: 'key', limit: 500, tz: 'UTC', now }).entries.length, 3) + assert.equal( + (await repo.leaderboard({ range: 'today', scope: 'key', limit: 1, tz: 'UTC', now })).entries[0].name, + 'bob-key', + ) + assert.equal((await repo.leaderboard({ range: 'today', scope: 'key', limit: 500, tz: 'UTC', now })).entries.length, 3) }) -test('cache hit rate uses disjoint input plus cache buckets for every route', () => { +test('cache hit rate uses disjoint input plus cache buckets for every route', async () => { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' const disjoint = { input_tokens: 10, cache_read_tokens: 80, cache_creation_tokens: 10 } @@ -197,27 +201,26 @@ test('cache hit rate uses disjoint input plus cache buckets for every route', () { name: 'missing input counts', row: { input_tokens: null, output_tokens: 50 }, rate: 0 }, ] for (const { name, row, rate, display } of cases) { - const db = freshDb() + const db = await freshDb() try { - seed(db, [{ created_at: at, user_id: 'u1', ...row }]) + await seed(db, [{ created_at: at, user_id: 'u1', ...row }]) const repo = new StatisticsRepo(db) - const totals = repo.statistics({ range: 'today', tz: 'UTC', now }).totals - const entry = repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now }).entries[0] + const totals = (await repo.statistics({ range: 'today', tz: 'UTC', now })).totals + const entry = (await repo.leaderboard({ range: 'today', scope: 'user', tz: 'UTC', now })).entries[0] assert.equal(totals.cacheHitRate, rate, `totals: ${name}`) assert.equal(entry.cacheHitRate, rate, `leaderboard: ${name}`) if (display) assert.equal((entry.cacheHitRate * 100).toFixed(1), display) } finally { - db.close() } } }) -test('Codex 100k input / 80k cached / 2k out stores 20k input and totals 102k tokens', () => { +test('Codex 100k input / 80k cached / 2k out stores 20k input and totals 102k tokens', async () => { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' - const db = freshDb() + const db = await freshDb() try { - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', @@ -229,20 +232,19 @@ test('Codex 100k input / 80k cached / 2k out stores 20k input and totals 102k to cache_creation_tokens: 0, }, ]) - const totals = new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now }).totals + const totals = (await new StatisticsRepo(db).statistics({ range: 'today', tz: 'UTC', now })).totals assert.equal(totals.inputTokens, 20_000) assert.equal(totals.tokens, 102_000) assert.notEqual(totals.tokens, 182_000) } finally { - db.close() } }) -test('API OpenAI replies converted to non-streaming Anthropic retain disjoint historical cache counts', () => { +test('API OpenAI replies converted to non-streaming Anthropic retain disjoint historical cache counts', async () => { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' for (const cacheWrite of [0, 10]) { - const db = freshDb() + const db = await freshDb() try { const upstream = { model: 'gpt-5.4', @@ -256,7 +258,7 @@ test('API OpenAI replies converted to non-streaming Anthropic retain disjoint hi const message = codexBodyToAnthropicMessage(upstream) // The API backend persists result.body.usage first. For an assembled // Anthropic reply it has already subtracted cache reads and writes. - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', @@ -276,25 +278,25 @@ test('API OpenAI replies converted to non-streaming Anthropic retain disjoint hi actual_cost: 0.2, }, ]) - const stored = db.prepare('SELECT * FROM usage_logs').all() + const stored = await db.prepare('SELECT * FROM usage_logs').all() assert.equal(stored[0].input_tokens, 20 - cacheWrite) assert.equal(stored[0].cache_read_tokens, 80) assert.equal(stored[0].cache_creation_tokens, cacheWrite) const repo = new StatisticsRepo(db) const options = { range: 'today', tz: 'UTC', now, owner_user_id: 'u1' } - const totals = repo.statistics(options).totals + const totals = (await repo.statistics(options)).totals assert.equal(totals.cacheHitRate, 0.8) assert.equal(totals.inputTokens, 20 - cacheWrite) assert.equal(totals.cost, 0.25) for (const scope of ['user', 'key', 'model', 'vm']) { - const entry = repo.leaderboard({ ...options, scope }).entries[0] + const entry = (await repo.leaderboard({ ...options, scope })).entries[0] assert.equal(entry.cacheHitRate, 0.8, scope) assert.equal((entry.cacheHitRate * 100).toFixed(1), '80.0', scope) assert.equal(entry.totalActualCost, 0.2, scope) } // Statistics fix the denominator without rewriting historical usage or billing. - assert.deepEqual(db.prepare('SELECT * FROM usage_logs').all(), stored) - seed(db, [ + assert.deepEqual(await db.prepare('SELECT * FROM usage_logs').all(), stored) + await seed(db, [ { created_at: at, user_id: 'u1', @@ -306,21 +308,20 @@ test('API OpenAI replies converted to non-streaming Anthropic retain disjoint hi cache_read_tokens: 400, }, ]) - assert.equal(repo.statistics(options).totals.cacheHitRate, 480 / 1100) - assert.equal(repo.leaderboard({ ...options, scope: 'user' }).entries[0].cacheHitRate, 480 / 1100) + assert.equal((await repo.statistics(options)).totals.cacheHitRate, 480 / 1100) + assert.equal((await repo.leaderboard({ ...options, scope: 'user' })).entries[0].cacheHitRate, 480 / 1100) } finally { - db.close() } } }) -test('mixed-provider cache rates weight each row by prompt tokens for every scope and owner', () => { - const db = freshDb() +test('mixed-provider cache rates weight each row by prompt tokens for every scope and owner', async () => { + const db = await freshDb() try { const now = Date.parse('2026-05-01T12:00:00Z') const at = '2026-05-01T09:00:00.000Z' const shared = { created_at: at, user_id: 'u1', api_key_id: 'k1', vm_id: 'vm-a', model: 'shared-alias' } - seed(db, [ + await seed(db, [ { ...shared, upstream_model: 'claude-sonnet-4', cache_read_tokens: 80, cache_creation_tokens: 20 }, { ...shared, via: 'codex-kernel', input_tokens: 500, cache_read_tokens: 400, cache_creation_tokens: 100 }, { created_at: at, user_id: 'u2', api_key_id: 'k2', input_tokens: 50, cache_read_tokens: 50 }, @@ -331,31 +332,31 @@ test('mixed-provider cache rates weight each row by prompt tokens for every scop const options = { range: 'today', tz: 'UTC', now } const expected = 480 / 1100 for (const scope of ['user', 'key', 'model', 'vm']) { - const entries = repo.leaderboard({ ...options, scope, owner_user_id: 'u1' }).entries + const entries = (await repo.leaderboard({ ...options, scope, owner_user_id: 'u1' })).entries assert.equal(entries.length, 1, scope) assert.equal(entries[0].cacheHitRate, expected, scope) } - const mine = repo.statistics({ ...options, owner_user_id: 'u1' }).totals + const mine = (await repo.statistics({ ...options, owner_user_id: 'u1' })).totals assert.equal(mine.cacheHitRate, expected) assert.equal(mine.cacheReadTokens, 480) assert.equal(mine.cacheCreationTokens, 120) // The existing total-token metric is intentionally separate from this fix. assert.equal(mine.tokens, 1100) - assert.equal(repo.statistics(options).totals.cacheHitRate, 530 / 1200) - const empty = repo.statistics({ ...options, owner_user_id: 'missing-user' }).totals + assert.equal((await repo.statistics(options)).totals.cacheHitRate, 530 / 1200) + const empty = (await repo.statistics({ ...options, owner_user_id: 'missing-user' })).totals assert.equal(empty.cacheHitRate, 0) } finally { - db.close() } }) test('panel routes owner-scope statistics for users and reject user/vm groupings', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-stats-route-')) - const db = openDatabase({ dbPath: path.join(dir, 'panel.db') }) + const db = await openDatabase({ dbPath: path.join(dir, 'panel.db') }) try { - seedUsers(db) + await db.exec('TRUNCATE users, api_keys, vms, usage_logs, request_attempts') + await seedUsers(db) const at = new Date(Date.now() - 1000).toISOString() - seed(db, [ + await seed(db, [ { created_at: at, user_id: 'u1', api_key_id: 'k1', total_cost: 1 }, { created_at: at, user_id: 'u2', api_key_id: 'k2', total_cost: 7 }, ]) @@ -392,6 +393,6 @@ test('panel routes owner-scope statistics for users and reject user/vm groupings assert.equal(all.status, 200) assert.equal(all.body.data.totals.cost, 8) } finally { - closeDatabase() + await closeDatabase() } }) diff --git a/test/unit/usage-logs-repo.test.mjs b/test/unit/usage-logs-repo.test.mjs index 67bc83f76..9464f1ff2 100644 --- a/test/unit/usage-logs-repo.test.mjs +++ b/test/unit/usage-logs-repo.test.mjs @@ -3,17 +3,17 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' import { codexBodyToAnthropicMessage } from '../../src/lib/protocol/codex-convert.mjs' import { estimateWindow } from '../../src/lib/admin/window-estimate.mjs' let seq = 0 -function freshDb() { +async function freshDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ulrepo-')) - const db = createDatabase({ dbPath: path.join(dir, 'logs.db') }) - db.exec(` + const db = await getDb() + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'); @@ -21,17 +21,17 @@ function freshDb() { return { dir, db, repo: new UsageLogsRepo(db) } } -function storedDebugBytes(db) { - return db - .prepare('SELECT record_json FROM request_log_debug') - .all() - .reduce((sum, row) => sum + Buffer.byteLength(row.record_json, 'utf8'), 0) +async function storedDebugBytes(db) { + return (await db.prepare('SELECT record_json FROM request_log_debug').all()).reduce( + (sum, row) => sum + Buffer.byteLength(row.record_json, 'utf8'), + 0, + ) } -function seed(repo, rows) { +async function seed(repo, rows) { for (const r of rows) { seq++ - repo.insertSummary({ + await repo.insertSummary({ id: `log_${seq}`, request_id: `req-${seq}`, created_at: new Date().toISOString(), @@ -51,65 +51,66 @@ function assertHit(stats, prompt, read, label) { assert.equal(stats.cache_hit_rate, prompt > 0 ? read / prompt : null, `${label} cache_hit_rate`) } -test('debugBytes counts stored JSON bytes for empty, ASCII, and UTF-8 records', () => { - const { dir, db, repo } = freshDb() +test('debugBytes counts stored JSON bytes for empty, ASCII, and UTF-8 records', async () => { + const { dir, db, repo } = await freshDb() try { - assert.equal(storedDebugBytes(db), 0) - assert.equal(repo.debugBytes(), 0) + assert.equal(await storedDebugBytes(db), 0) + assert.equal(await repo.debugBytes(), 0) const records = [{ body: 'ascii' }, { body: '中🙂' }, { body: 'line\nquote"' }] - records.forEach((record, i) => { - repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) - }) + await Promise.all( + records.map(async (record, i) => { + await repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) + }), + ) const expected = records.reduce((sum, record) => sum + Buffer.byteLength(JSON.stringify(record), 'utf8'), 0) - assert.equal(storedDebugBytes(db), expected) - assert.equal(repo.debugBytes(), expected) + assert.equal(await storedDebugBytes(db), expected) + assert.equal(await repo.debugBytes(), expected) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('trimDebugToMaxBytes removes oldest UTF-8 records until actual bytes fit', () => { - const { dir, db, repo } = freshDb() +test('trimDebugToMaxBytes removes oldest UTF-8 records until actual bytes fit', async () => { + const { dir, db, repo } = await freshDb() try { const record = { body: '中'.repeat(10) } const serialized = JSON.stringify(record) const maxBytes = 164 for (let i = 0; i < 5; i++) { - repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) + await repo.insertDebug(`debug-${i}`, `2026-10-15T21:00:0${i}.000Z`, record) } - const before = db.prepare('SELECT record_json FROM request_log_debug').all() + const before = await db.prepare('SELECT record_json FROM request_log_debug').all() const characterCount = before.reduce((sum, row) => sum + row.record_json.length, 0) assert.equal(characterCount, serialized.length * 5) assert.ok(characterCount < maxBytes) - assert.ok(storedDebugBytes(db) > maxBytes) + assert.ok((await storedDebugBytes(db)) > maxBytes) - const originalIds = db - .prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC') - .all() - .map((row) => row.request_id) - const deleted = repo.trimDebugToMaxBytes(maxBytes) + const originalIds = (await db.prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC').all()).map( + (row) => row.request_id, + ) + const deleted = await repo.trimDebugToMaxBytes(maxBytes) assert.ok(deleted > 0 && deleted < originalIds.length) - const remaining = db.prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC').all() + const remaining = await db.prepare('SELECT request_id FROM request_log_debug ORDER BY ts ASC').all() const remainingIds = remaining.map((row) => row.request_id) assert.equal(remainingIds.length, originalIds.length - deleted) assert.deepEqual(remainingIds, originalIds.slice(deleted)) assert.equal(remainingIds.at(-1), originalIds.at(-1)) - assert.ok(storedDebugBytes(db) <= maxBytes) - assert.equal(repo.debugBytes(), storedDebugBytes(db)) + assert.ok((await storedDebugBytes(db)) <= maxBytes) + assert.equal(await repo.debugBytes(), await storedDebugBytes(db)) } finally { - db.close() + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } }) -test('mixed Claude/Codex usage keeps a row-wise prompt denominator on totals and windows', () => { - const { db, repo } = freshDb() +test('mixed Claude/Codex usage keeps a row-wise prompt denominator on totals and windows', async () => { + const { db, repo } = await freshDb() try { - seed(repo, [ + await seed(repo, [ { user_id: 'u1', api_key_id: 'k1', @@ -135,32 +136,32 @@ test('mixed Claude/Codex usage keeps a row-wise prompt denominator on totals and cache_read_tokens: 50, }, ]) - assertHit(repo.totals(), 300, 240, 'totals') - assertHit(repo.billingStats().total, 300, 240, 'billing.total') + assertHit(await repo.totals(), 300, 240, 'totals') + assertHit((await repo.billingStats()).total, 300, 240, 'billing.total') const since = new Date(Date.now() - 60_000).toISOString() - assertHit(repo.windowStats({ since }), 300, 240, 'window') - assertHit(repo.windowStats({ since, owner_user_id: 'u1' }), 200, 190, 'owner window') - assertHit(repo.windowStats({ since, owner_user_id: 'u2' }), 100, 50, 'other owner') + assertHit(await repo.windowStats({ since }), 300, 240, 'window') + assertHit(await repo.windowStats({ since, owner_user_id: 'u1' }), 200, 190, 'owner window') + assertHit(await repo.windowStats({ since, owner_user_id: 'u2' }), 100, 50, 'other owner') } finally { - db.close() + await db.close() } }) -test('empty usage windows keep a null cache hit rate', () => { - const { db, repo } = freshDb() +test('empty usage windows keep a null cache hit rate', async () => { + const { db, repo } = await freshDb() try { - assertHit(repo.totals(), 0, 0, 'empty totals') - assertHit(repo.billingStats().total, 0, 0, 'empty billing') - const empty = repo.windowStats({ since: '2099-01-01T00:00:00.000Z' }) + assertHit(await repo.totals(), 0, 0, 'empty totals') + assertHit((await repo.billingStats()).total, 0, 0, 'empty billing') + const empty = await repo.windowStats({ since: '2099-01-01T00:00:00.000Z' }) assert.equal(empty.requests, 0) assertHit(empty, 0, 0, 'empty window') } finally { - db.close() + await db.close() } }) -test('billingStats reuses identical panel snapshots but separates account windows', () => { - const { db, repo } = freshDb() +test('billingStats reuses identical panel snapshots but separates account windows', async () => { + const { db, repo } = await freshDb() try { let computes = 0 const compute = repo._computeBillingStats.bind(repo) @@ -173,23 +174,26 @@ test('billingStats reuses identical panel snapshots but separates account window const winAReordered = [{ vm_id: 'vm-a', reset_5h: '2026-10-19T05:00:00.000Z', account_id: 'acc-1' }] const winB = [{ account_id: 'acc-1', vm_id: 'vm-a', reset_5h: '2026-10-19T06:00:00.000Z' }] - assert.equal(repo.billingStats({ accountWindows: winA }), repo.billingStats({ accountWindows: winAReordered })) + assert.equal( + await repo.billingStats({ accountWindows: winA }), + await repo.billingStats({ accountWindows: winAReordered }), + ) assert.equal(computes, 1) - repo.billingStats({ accountWindows: winB }) + await repo.billingStats({ accountWindows: winB }) assert.equal(computes, 2) // Explicit historical snapshots are deterministic test/report calls and must // not be served from the live panel cache. - repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) - repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) + await repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) + await repo.billingStats({ accountWindows: winA, now: Date.parse('2026-10-19T00:00:00.000Z') }) assert.equal(computes, 4) } finally { - db.close() + await db.close() } }) -test('billingStats TTL starts after a slow aggregation finishes', () => { - const { db, repo } = freshDb() +test('billingStats TTL starts after a slow aggregation finishes', async () => { + const { db, repo } = await freshDb() const realNow = Date.now try { let clock = 1_000_000 @@ -201,39 +205,39 @@ test('billingStats TTL starts after a slow aggregation finishes', () => { clock += 20_000 // Simulate a scan slower than the 10-second cache TTL. return compute(opts) } - const first = repo.billingStats() - assert.equal(repo.billingStats(), first) + const first = await repo.billingStats() + assert.equal(await repo.billingStats(), first) assert.equal(computes, 1) clock += 10_001 - repo.billingStats() + await repo.billingStats() assert.equal(computes, 2) } finally { Date.now = realNow - db.close() + await db.close() } }) -test('billingStats rate-limits maintenance backfill independently of cache keys', () => { - const { db, repo } = freshDb() +test('billingStats rate-limits maintenance backfill independently of cache keys', async () => { + const { db, repo } = await freshDb() try { let backfills = 0 repo.backfillMissingCosts = () => { backfills++ return 0 } - repo.billingStats() - repo.billingStats({ accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a' }] }) - repo.billingStats({ accountWindows: [{ account_id: 'acc-2', vm_id: 'vm-a' }] }) + await repo.billingStats() + await repo.billingStats({ accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a' }] }) + await repo.billingStats({ accountWindows: [{ account_id: 'acc-2', vm_id: 'vm-a' }] }) assert.equal(backfills, 1) } finally { - db.close() + await db.close() } }) -test('billingStats derives global totals from account aggregates without duplicate full scans', () => { - const { db, repo } = freshDb() +test('billingStats derives global totals from account aggregates without duplicate full scans', async () => { + const { db, repo } = await freshDb() try { - seed(repo, [ + await seed(repo, [ { account_id: 'acc-1', vm_id: 'vm-a', input_tokens: 20, cache_read_tokens: 80, total_cost: 1 }, { account_id: 'acc-2', vm_id: null, input_tokens: 40, output_tokens: 5, total_cost: 2 }, ]) @@ -243,7 +247,7 @@ test('billingStats derives global totals from account aggregates without duplica if (!cond) fullScans++ return costSelect(cond, params) } - const bill = repo.billingStats({ now: Date.now() }) + const bill = await repo.billingStats({ now: Date.now() }) assert.equal(fullScans, 0) assert.equal(bill.total.requests, 2) assert.equal(bill.total.input_tokens, 60) @@ -255,15 +259,15 @@ test('billingStats derives global totals from account aggregates without duplica bill.accounts.reduce((sum, row) => sum + row.total_cost, 0), ) } finally { - db.close() + await db.close() } }) -test('aligned account windows re-sum per-row prompt tokens instead of mixed Anthropic formula', () => { - const { db, repo } = freshDb() +test('aligned account windows re-sum per-row prompt tokens instead of mixed Anthropic formula', async () => { + const { db, repo } = await freshDb() try { const now = Date.parse('2026-09-26T12:00:00.000Z') - seed(repo, [ + await seed(repo, [ { created_at: '2026-09-26T10:00:00.000Z', vm_id: 'vm-a', @@ -289,22 +293,22 @@ test('aligned account windows re-sum per-row prompt tokens instead of mixed Anth cache_read_tokens: 95, }, ]) - const aligned = repo.billingStats({ + const aligned = await repo.billingStats({ now, accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a', reset_5h: '2026-09-26T16:00:00.000Z' }], }) assertHit(aligned.window_5h, 200, 190, 'aligned 5h') assert.equal(Number(aligned.accounts[0].window_5h.prompt_tokens), 200) assert.equal(Number(aligned.accounts[0].window_5h.cache_read_tokens), 190) - const rolling = repo.billingStats({ now }) + const rolling = await repo.billingStats({ now }) assertHit(rolling.window_5h, 10_205, 190, 'rolling 5h') } finally { - db.close() + await db.close() } }) -test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRepo totals', () => { - const { db, repo } = freshDb() +test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRepo totals', async () => { + const { db, repo } = await freshDb() try { const upstream = { model: 'gpt-5.4', @@ -316,7 +320,7 @@ test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRep }, } const message = codexBodyToAnthropicMessage(upstream) - seed(repo, [ + await seed(repo, [ { user_id: 'u1', via: 'api-kernel', @@ -340,17 +344,17 @@ test('converted non-streaming Anthropic API usage stays disjoint in UsageLogsRep ]) assert.equal(message.usage.input_tokens, 20) assert.equal(message.usage.cache_read_input_tokens, 80) - assertHit(repo.totals(), 200, 160, 'converted+codex totals') + assertHit(await repo.totals(), 200, 160, 'converted+codex totals') } finally { - db.close() + await db.close() } }) -test('billingStats window tokens are total including cache', () => { - const { db, repo } = freshDb() +test('billingStats window tokens are total including cache', async () => { + const { db, repo } = await freshDb() try { const now = Date.parse('2026-09-26T12:00:00.000Z') - seed(repo, [ + await seed(repo, [ { created_at: '2026-09-26T11:00:00.000Z', vm_id: 'vm-a', @@ -360,53 +364,53 @@ test('billingStats window tokens are total including cache', () => { cache_read_tokens: 80, }, ]) - const bill = repo.billingStats({ now }) + const bill = await repo.billingStats({ now }) assert.equal(bill.accounts[0].window_5h_tokens, 102) - const window = repo.windowStats({ since: '2026-09-26T00:00:00.000Z', until: '2026-09-26T12:00:00.000Z' }) + const window = await repo.windowStats({ since: '2026-09-26T00:00:00.000Z', until: '2026-09-26T12:00:00.000Z' }) assert.equal(window.input_tokens + window.output_tokens + window.cache_read_tokens, 102) } finally { - db.close() + await db.close() } }) -test('log coverage follows recorded prunes, not the oldest row or the current retention', () => { - const { db, repo } = freshDb() +test('log coverage follows recorded prunes, not the oldest row or the current retention', async () => { + const { db, repo } = await freshDb() try { const now = Date.now() const iso = (msAgo) => new Date(now - msAgo).toISOString() - db.prepare("UPDATE schema_migrations SET applied_at = ? WHERE version = '001'").run(iso(30 * 86400_000)) - seed(repo, [ + await db.prepare("UPDATE schema_migrations SET applied_at = $1 WHERE version = '001'").run(iso(30 * 86400_000)) + await seed(repo, [ { created_at: iso(3 * 86400_000), vm_id: 'vm-a', account_id: 'acc-1', input_tokens: 1 }, { created_at: iso(3600_000), vm_id: 'vm-a', account_id: 'acc-1', input_tokens: 1 }, ]) // Never pruned: a quiet stretch before the oldest row is still complete history. - assert.equal(repo.billingStats().log_coverage_start, iso(30 * 86400_000)) + assert.equal((await repo.billingStats()).log_coverage_start, iso(30 * 86400_000)) - assert.equal(repo.cleanup({ retainDays: 1 }).usage_logs, 1) - const afterPrune = Date.parse(repo.logCoverageStart()) + assert.equal((await repo.cleanup({ retainDays: 1 })).usage_logs, 1) + const afterPrune = Date.parse(await repo.logCoverageStart()) assert.ok(afterPrune >= now - 86400_000 && afterPrune <= Date.now() - 86400_000 + 1000) // Raising retention later does not bring the deleted day back. - assert.equal(repo.cleanup({ retainDays: 7 }).usage_logs, 0) - assert.equal(Date.parse(repo.logCoverageStart()), afterPrune) + assert.equal((await repo.cleanup({ retainDays: 7 })).usage_logs, 0) + assert.equal(Date.parse(await repo.logCoverageStart()), afterPrune) const week = estimateWindow({ spentUsd: 5, usedRatio: 0.5, reset: iso(-86400_000), windowStart: iso(6 * 86400_000), - coveredSince: repo.logCoverageStart(), + coveredSince: await repo.logCoverageStart(), }) assert.equal(week.status, 'log_gap') } finally { - db.close() + await db.close() } }) -test('window_fable only counts fable rows since reset_7d_oi', () => { - const { db, repo } = freshDb() +test('window_fable only counts fable rows since reset_7d_oi', async () => { + const { db, repo } = await freshDb() try { const now = Date.parse('2026-09-26T12:00:00.000Z') - seed(repo, [ + await seed(repo, [ { created_at: '2026-09-20T11:00:00.000Z', vm_id: 'vm-a', @@ -439,7 +443,7 @@ test('window_fable only counts fable rows since reset_7d_oi', () => { total_cost: 8, }, ]) - const bill = repo.billingStats({ + const bill = await repo.billingStats({ now, accountWindows: [ { @@ -455,29 +459,29 @@ test('window_fable only counts fable rows since reset_7d_oi', () => { assert.equal(fable.cost, 3) assert.equal(fable.tokens, 22) assert.equal(bill.accounts[0].window_fable, fable) - const none = repo.billingStats({ + const none = await repo.billingStats({ now, accountWindows: [{ account_id: 'acc-1', vm_id: 'vm-a' }], }) assert.equal(none.accounts[0].window_fable, null) } finally { - db.close() + await db.close() } }) -test('unpricedModels lists unpriced rows', () => { - const { db, repo } = freshDb() +test('unpricedModels lists unpriced rows', async () => { + const { db, repo } = await freshDb() try { - seed(repo, [ + await seed(repo, [ { model: 'mystery', pricing_model: 'unpriced', input_tokens: 10 }, { model: 'mystery', pricing_model: 'unpriced', input_tokens: 4 }, { model: 'claude-sonnet-5', pricing_model: 'sonnet-5', input_tokens: 8 }, ]) - const rows = repo.unpricedModels() + const rows = await repo.unpricedModels() assert.equal(rows.length, 1) assert.equal(rows[0].model, 'mystery') assert.equal(rows[0].requests, 2) } finally { - db.close() + await db.close() } }) diff --git a/test/unit/usage-logs-view.test.mjs b/test/unit/usage-logs-view.test.mjs index db6491c4c..2896927fb 100644 --- a/test/unit/usage-logs-view.test.mjs +++ b/test/unit/usage-logs-view.test.mjs @@ -3,16 +3,16 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb, withTransaction } from '../../src/lib/db/database.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' import { UsageLogsView } from '../../src/lib/db/repos/usage-logs-view.mjs' let seq = 0 -function freshDb() { +async function freshDb() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ulview-')) - const db = createDatabase({ dbPath: path.join(dir, 'logs.db') }) - db.exec(` + const db = await getDb() + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); INSERT INTO vms (id, name, vm_json) VALUES ('vm-a', 'Slot A', '{}'), ('vm-b', 'Slot B', '{}'); @@ -22,37 +22,37 @@ function freshDb() { } /** Insert rows newest-last; created_at defaults to a strictly increasing clock. */ -function seed(db, rows) { +async function seed(db, rows) { const repo = new UsageLogsRepo(db) const out = [] - db.exec('BEGIN') - for (const r of rows) { - seq++ - const rec = { - id: `log_${String(seq).padStart(8, '0')}`, - request_id: `req-${seq}`, - created_at: new Date(Date.UTC(2026, 0, 1) + seq * 1000).toISOString(), - log_mode: 'normal', - method: 'POST', - path: '/v1/messages', - protocol: 'anthropic.messages', - model: 'claude-sonnet-4', - status: 200, - vm_id: 'vm-a', - ...r, + await withTransaction(db, async () => { + for (const r of rows) { + seq++ + const rec = { + id: `log_${String(seq).padStart(8, '0')}`, + request_id: `req-${seq}`, + created_at: new Date(Date.UTC(2026, 0, 1) + seq * 1000).toISOString(), + log_mode: 'normal', + method: 'POST', + path: '/v1/messages', + protocol: 'anthropic.messages', + model: 'claude-sonnet-4', + status: 200, + vm_id: 'vm-a', + ...r, + } + await repo.insertSummary(rec) + out.push(rec) } - repo.insertSummary(rec) - out.push(rec) - } - db.exec('COMMIT') + }) return out } -function pageAll(view, filters) { +async function pageAll(view, filters) { const seen = [] let cursor = null for (let i = 0; i < 50; i++) { - const page = view.listBatch({ + const page = await view.listBatch({ ...filters, cursor_created_at: cursor?.createdAt, cursor_id: cursor?.id, @@ -65,20 +65,20 @@ function pageAll(view, filters) { throw new Error('paging did not terminate') } -test('keyset paging returns every row once, newest first', () => { - const db = freshDb() - const rows = seed( +test('keyset paging returns every row once, newest first', async () => { + const db = await freshDb() + const rows = await seed( db, Array.from({ length: 5 }, () => ({})), ) const view = new UsageLogsView(db) - const first = view.listBatch({ limit: 2 }) + const first = await view.listBatch({ limit: 2 }) assert.deepEqual( first.logs.map((r) => r.id), [rows[4].id, rows[3].id], ) assert.equal(first.hasMore, true) - const { seen, pages } = pageAll(view, { limit: 2 }) + const { seen, pages } = await pageAll(view, { limit: 2 }) assert.equal(pages, 3) assert.deepEqual( seen.map((r) => r.id), @@ -86,24 +86,24 @@ test('keyset paging returns every row once, newest first', () => { ) }) -test('JS-filtered paging hands back the scan cursor instead of ending early', () => { - const db = freshDb() - const old = seed(db, [ +test('JS-filtered paging hands back the scan cursor instead of ending early', async () => { + const db = await freshDb() + const old = await seed(db, [ { status: 504, error_code: 'upstream_timeout' }, { status: 504, error_code: 'upstream_timeout' }, { status: 504, error_code: 'upstream_timeout' }, ]) // 2100 newer errors of another class: more than one call's scan budget. - seed( + await seed( db, Array.from({ length: 2100 }, () => ({ status: 500, error_code: 'upstream_error' })), ) const view = new UsageLogsView(db) - const first = view.listBatch({ error_class: 'timeout', limit: 50 }) + const first = await view.listBatch({ error_class: 'timeout', limit: 50 }) assert.equal(first.logs.length, 0) assert.equal(first.hasMore, true) assert.ok(first.nextCursor) - const { seen } = pageAll(view, { error_class: 'timeout', limit: 50 }) + const { seen } = await pageAll(view, { error_class: 'timeout', limit: 50 }) assert.deepEqual( seen.map((r) => r.id), old.map((r) => r.id).reverse(), @@ -111,79 +111,75 @@ test('JS-filtered paging hands back the scan cursor instead of ending early', () assert.ok(seen.every((r) => r.errorClass === 'timeout')) }) -test('user owner scope keeps only own key / own user rows and ignores other tenants', () => { - const db = freshDb() - seed(db, [ +test('user owner scope keeps only own key / own user rows and ignores other tenants', async () => { + const db = await freshDb() + await seed(db, [ { user_id: 'u1', api_key_id: 'k1' }, { user_id: null, api_key_id: 'k1' }, { user_id: 'u2', api_key_id: 'k2' }, ]) const view = new UsageLogsView(db) - const mine = view.listBatch({ owner_user_id: 'u1' }).logs + const mine = (await view.listBatch({ owner_user_id: 'u1' })).logs assert.equal(mine.length, 2) assert.ok(mine.every((r) => r.keyId === 'k1')) - assert.equal(view.summary({ owner_user_id: 'u1' }).totalRequests, 2) - const opts = view.filterOptions({ owner_user_id: 'u1' }) + assert.equal((await view.summary({ owner_user_id: 'u1' })).totalRequests, 2) + const opts = await view.filterOptions({ owner_user_id: 'u1' }) assert.deepEqual(opts.users, [{ id: 'u1', name: 'alice' }]) assert.deepEqual(opts.keys, [{ id: 'k1', name: 'alice-key', userId: 'u1' }]) - assert.equal(view.listBatch({ user_id: 'u2' }).logs.length, 1) + assert.equal((await view.listBatch({ user_id: 'u2' })).logs.length, 1) }) -test('muting follows RequestLogStore: settings default, explicit list, include_muted, error_class override', () => { - const db = freshDb() - seed(db, [ +test('muting follows RequestLogStore: settings default, explicit list, include_muted, error_class override', async () => { + const db = await freshDb() + await seed(db, [ { status: 401, error_code: 'invalid_api_key', vm_id: null }, { status: 429, error_code: 'quota_5h_safety' }, { status: 200 }, ]) - const classes = (view, f = {}) => - view - .listBatch(f) - .logs.map((r) => r.errorClass ?? 'ok') - .sort() + const classes = async (view, f = {}) => (await view.listBatch(f)).logs.map((r) => r.errorClass ?? 'ok').sort() const byDefault = new UsageLogsView(db) - assert.deepEqual(classes(byDefault), ['ok', 'quota']) + assert.deepEqual(await classes(byDefault), ['ok', 'quota']) const settings = new UsageLogsView(db, { mutedErrorClasses: ['auth', 'quota'] }) - assert.deepEqual(classes(settings), ['ok']) - assert.equal(settings.summary({}).totalRequests, 1) - assert.deepEqual(classes(settings, { include_muted: true }), ['auth', 'ok', 'quota']) - assert.deepEqual(classes(settings, { error_class: 'auth' }), ['auth']) - assert.deepEqual(classes(settings, { exclude_error_class: 'quota' }), ['auth', 'ok']) + assert.deepEqual(await classes(settings), ['ok']) + assert.equal((await settings.summary({})).totalRequests, 1) + assert.deepEqual(await classes(settings, { include_muted: true }), ['auth', 'ok', 'quota']) + assert.deepEqual(await classes(settings, { error_class: 'auth' }), ['auth']) + assert.deepEqual(await classes(settings, { exclude_error_class: 'quota' }), ['auth', 'ok']) }) -test('summary cache hit rate counts Claude writes and Codex disjoint input on both summary paths', () => { - const db = freshDb() - seed(db, [ +test('summary cache hit rate counts Claude writes and Codex disjoint input on both summary paths', async () => { + const db = await freshDb() + await seed(db, [ { input_tokens: 20, cache_read_tokens: 80_000, cache_creation_tokens: 20_000 }, { via: 'codex-kernel', model: 'gpt-5.4', input_tokens: 20, cache_read_tokens: 80 }, ]) const want = 80_080 / 100_120 - const sqlPath = new UsageLogsView(db).summary({}) - const scanPath = new UsageLogsView(db, { mutedErrorClasses: ['auth', 'quota'] }).summary({}) + const sqlPath = await new UsageLogsView(db).summary({}) + const scanPath = await new UsageLogsView(db, { mutedErrorClasses: ['auth', 'quota'] }).summary({}) for (const summary of [sqlPath, scanPath]) { assert.equal(summary.totalRequests, 2) assert.ok(Math.abs(summary.cacheHitRate - want) < 1e-9, String(summary.cacheHitRate)) } }) -test('provider chains for a page come from one request_attempts query, ordered by attempt', () => { - const db = freshDb() - const [a, b, c] = seed(db, [ +test('provider chains for a page come from one request_attempts query, ordered by attempt', async () => { + const db = await freshDb() + const [a, b, c] = await seed(db, [ { account_id: 'acc-a' }, { final_account_id: 'acc-b', vm_id: 'vm-b', duration_ms: 2000 }, {}, ]) const before = (row, ms) => new Date(Date.parse(row.created_at) - ms).toISOString() const ins = db.prepare( - 'INSERT INTO request_attempts (request_id, attempt_no, vm_id, account_id, started_at, selection_reason, downstream_committed) VALUES (?, ?, ?, ?, ?, ?, ?)', + 'INSERT INTO request_attempts (request_id, attempt_no, vm_id, account_id, started_at, selection_reason, downstream_committed) VALUES ($1, $2, $3, $4, $5, $6, $7)', ) - ins.run(a.request_id, 2, 'vm-b', 'acc-b', before(a, 200), 'failover', 1) - ins.run(a.request_id, 1, 'vm-a', 'acc-a', before(a, 500), 'sticky', 0) - ins.run(b.request_id, 1, 'vm-b', 'acc-b', before(b, 1500), 'roundrobin', 1) + await ins.run(a.request_id, 2, 'vm-b', 'acc-b', before(a, 200), 'failover', 1) + await ins.run(a.request_id, 1, 'vm-a', 'acc-a', before(a, 500), 'sticky', 0) + await ins.run(b.request_id, 1, 'vm-b', 'acc-b', before(b, 1500), 'roundrobin', 1) // Client-chosen x-request-id reused by later / earlier requests: their attempts // fall outside the row's own lifetime and must not be attached. - ins.run(b.request_id, 2, 'vm-a', 'acc-a', before(b, 60_000), 'sticky', 1) - ins.run(c.request_id, 1, 'vm-b', 'acc-b', before(c, -3600_000), 'sticky', 1) + await ins.run(b.request_id, 2, 'vm-a', 'acc-a', before(b, 60_000), 'sticky', 1) + await ins.run(c.request_id, 1, 'vm-b', 'acc-b', before(c, -3600_000), 'sticky', 1) const attemptQueries = [] const spy = { @@ -192,7 +188,7 @@ test('provider chains for a page come from one request_attempts query, ordered b return db.prepare(sql) }, } - const logs = new UsageLogsView(spy).listBatch({}).logs + const logs = (await new UsageLogsView(spy).listBatch({})).logs assert.equal(attemptQueries.length, 1) const rowA = logs.find((r) => r.requestId === a.request_id) assert.deepEqual( @@ -219,40 +215,39 @@ test('provider chains for a page come from one request_attempts query, ordered b assert.deepEqual(logs.find((r) => r.requestId === c.request_id).providerChain, []) }) -test('filters: model/requested_model, status, attempts, session, debug, time, q', () => { - const db = freshDb() - const rows = seed(db, [ +test('filters: model/requested_model, status, attempts, session, debug, time, q', async () => { + const db = await freshDb() + const rows = await seed(db, [ { model: 'claude-opus-4', requested_model: 'opus-alias', session_id: 's-1', outbound_session_id: 'out-1' }, { status: 500, error_code: 'upstream_error', attempt_count: 3, session_id: 's-2', outbound_session_id: 'out-2' }, { status: 204, log_mode: 'debug', path: '/v1/chat/completions' }, { status: 200, error_message: 'needle here', attempt_count: 1 }, ]) const view = new UsageLogsView(db) - const ids = (f) => - view - .listBatch(f) - .logs.map((r) => r.id) - .sort() - assert.deepEqual(ids({ model: 'opus-alias' }), [rows[0].id]) - assert.deepEqual(ids({ status_code: '500' }), [rows[1].id]) - assert.deepEqual(ids({ exclude_status_200: true }), [rows[1].id]) - assert.deepEqual(ids({ exclude_status_200: true, status_code: '204' }), [rows[2].id]) - assert.deepEqual(ids({ min_attempt_count: '2' }), [rows[1].id]) + const ids = async (f) => (await view.listBatch(f)).logs.map((r) => r.id).sort() + assert.deepEqual(await ids({ model: 'opus-alias' }), [rows[0].id]) + assert.deepEqual(await ids({ status_code: '500' }), [rows[1].id]) + assert.deepEqual(await ids({ exclude_status_200: true }), [rows[1].id]) + assert.deepEqual(await ids({ exclude_status_200: true, status_code: '204' }), [rows[2].id]) + assert.deepEqual(await ids({ min_attempt_count: '2' }), [rows[1].id]) // Either the caller's id or the id the gateway sent upstream finds the row. - assert.deepEqual(ids({ session_id: 's-2' }), [rows[1].id]) - assert.deepEqual(ids({ session_id: 'out-2' }), [rows[1].id]) - const [first] = view.listBatch({ session_id: 'out-1' }).logs + assert.deepEqual(await ids({ session_id: 's-2' }), [rows[1].id]) + assert.deepEqual(await ids({ session_id: 'out-2' }), [rows[1].id]) + const [first] = (await view.listBatch({ session_id: 'out-1' })).logs assert.equal(first.sessionId, 'out-1') assert.equal(first.clientSessionId, 's-1') - assert.deepEqual(ids({ debug_only: true }), [rows[2].id]) - assert.deepEqual(ids({ endpoint: '/v1/chat/completions' }), [rows[2].id]) - assert.deepEqual(ids({ q: 'needle' }), [rows[3].id]) - assert.deepEqual(ids({ start_time: rows[1].created_at, end_time: rows[3].created_at }), [rows[1].id, rows[2].id]) + assert.deepEqual(await ids({ debug_only: true }), [rows[2].id]) + assert.deepEqual(await ids({ endpoint: '/v1/chat/completions' }), [rows[2].id]) + assert.deepEqual(await ids({ q: 'needle' }), [rows[3].id]) + assert.deepEqual(await ids({ start_time: rows[1].created_at, end_time: rows[3].created_at }), [ + rows[1].id, + rows[2].id, + ]) }) -test('row badges: via only when non-default, context1m from long_context, fast/priority', () => { - const db = freshDb() - const [plain, special] = seed(db, [ +test('row badges: via only when non-default, context1m from long_context, fast/priority', async () => { + const db = await freshDb() + const [plain, special] = await seed(db, [ { via: 'go-worker-pool', stream: true }, { via: 'api-kernel', @@ -265,7 +260,7 @@ test('row badges: via only when non-default, context1m from long_context, fast/p workspace: 'client', }, ]) - const logs = new UsageLogsView(db).listBatch({}).logs + const logs = (await new UsageLogsView(db).listBatch({})).logs const keys = (id) => logs .find((r) => r.id === id) @@ -277,17 +272,17 @@ test('row badges: via only when non-default, context1m from long_context, fast/p assert.equal(logs.find((r) => r.id === plain.id).context1mApplied, null) }) -test('overview splits today/yesterday-same-time in the requested zone', () => { - const db = freshDb() +test('overview splits today/yesterday-same-time in the requested zone', async () => { + const db = await freshDb() const now = Date.parse('2026-03-10T04:00:00Z') // 12:00 in Shanghai - seed(db, [ + await seed(db, [ { created_at: '2026-03-09T02:00:00.000Z', total_cost: 1 }, // yesterday 10:00 local { created_at: '2026-03-09T06:00:00.000Z', total_cost: 2 }, // yesterday 14:00 local, after same-time { created_at: '2026-03-09T17:00:00.000Z', total_cost: 4, status: 500, error_code: 'upstream_error' }, { created_at: '2026-03-10T03:59:30.000Z', total_cost: 8, session_id: 's-live', outbound_session_id: 'out-live' }, ]) const view = new UsageLogsView(db) - const sh = view.overview({ tz: 'Asia/Shanghai', now }) + const sh = await view.overview({ tz: 'Asia/Shanghai', now }) assert.equal(sh.todayRequests, 2) assert.equal(sh.todayCost, 12) assert.equal(sh.todayErrors, 1) @@ -296,17 +291,17 @@ test('overview splits today/yesterday-same-time in the requested zone', () => { assert.equal(sh.rpm, 1) assert.equal(sh.activeSessions, 1) // Unknown zone falls back to UTC: today starts 2026-03-10T00:00Z. - const utc = view.overview({ tz: 'Not/AZone', now }) + const utc = await view.overview({ tz: 'Not/AZone', now }) assert.equal(utc.todayRequests, 1) assert.equal(utc.yesterdayRequests, 1) }) -test('active sessions and session suggestions group by outbound session, owner-scoped, newest first', () => { - const db = freshDb() +test('active sessions and session suggestions group by outbound session, owner-scoped, newest first', async () => { + const db = await freshDb() const now = Date.now() const iso = (ago) => new Date(now - ago).toISOString() // One client session that the gateway rebuilt into two upstream sessions (abc-1 / abc-1b). - seed(db, [ + await seed(db, [ { created_at: iso(60_000), session_id: 'client-1', @@ -329,7 +324,7 @@ test('active sessions and session suggestions group by outbound session, owner-s { created_at: iso(5_000), session_id: 'client-legacy', api_key_id: 'k1' }, ]) const view = new UsageLogsView(db) - const active = view.activeSessions({ owner_user_id: 'u1', now }) + const active = await view.activeSessions({ owner_user_id: 'u1', now }) assert.equal(active.total, 2) assert.deepEqual( active.sessions.map((s) => [s.sessionId, s.clientSessionId, s.requests]), @@ -340,19 +335,19 @@ test('active sessions and session suggestions group by outbound session, owner-s ) assert.equal(active.sessions[1].lastStatus, 200) assert.equal(active.sessions[1].keyName, 'alice-key') - assert.equal(view.activeSessions({ now }).total, 3) - assert.deepEqual(view.sessionSuggestions({ q: 'abc', owner_user_id: 'u1' }), ['abc-1b', 'abc-1', 'abc-3']) - assert.deepEqual(view.sessionSuggestions({ q: 'abc-2' }), ['abc-2']) - assert.deepEqual(view.sessionSuggestions({ q: 'client' }), []) - assert.equal(view.overview({ now }).activeSessions, 3) + assert.equal((await view.activeSessions({ now })).total, 3) + assert.deepEqual(await view.sessionSuggestions({ q: 'abc', owner_user_id: 'u1' }), ['abc-1b', 'abc-1', 'abc-3']) + assert.deepEqual(await view.sessionSuggestions({ q: 'abc-2' }), ['abc-2']) + assert.deepEqual(await view.sessionSuggestions({ q: 'client' }), []) + assert.equal((await view.overview({ now })).activeSessions, 3) }) -test('active sessions never resolve the last row from another tenant sharing the session id', () => { - const db = freshDb() +test('active sessions never resolve the last row from another tenant sharing the session id', async () => { + const db = await freshDb() const now = Date.now() const at = new Date(now - 30_000).toISOString() // Same client-chosen session id and completion instant; bob's row sorts later by id. - seed(db, [ + await seed(db, [ { created_at: at, session_id: 'shared', @@ -371,15 +366,15 @@ test('active sessions never resolve the last row from another tenant sharing the error_code: 'upstream_error', }, ]) - const [s] = new UsageLogsView(db).activeSessions({ owner_user_id: 'u1', now }).sessions + const [s] = (await new UsageLogsView(db).activeSessions({ owner_user_id: 'u1', now })).sessions assert.equal(s.keyName, 'alice-key') assert.equal(s.vmId, 'vm-a') assert.equal(s.lastStatus, 200) }) -test('tenant error collection hides other users and anonymous ingress failures', () => { - const db = freshDb() - seed(db, [ +test('tenant error collection hides other users and anonymous ingress failures', async () => { + const db = await freshDb() + await seed(db, [ { user_id: 'u2', api_key_id: 'k2', @@ -393,13 +388,27 @@ test('tenant error collection hides other users and anonymous ingress failures', { user_id: 'u1', api_key_id: 'k1', status: 500, error_code: 'upstream_error', error_message: 'boom' }, ]) const repo = new UsageLogsRepo(db) - const mine = repo.windowStats({ owner_user_id: 'u1' }).error_collection + const mine = (await repo.windowStats({ owner_user_id: 'u1' })).error_collection assert.deepEqual( mine.recent.map((r) => r.error_code), ['upstream_error'], ) assert.deepEqual(mine.ingress_hits, []) - const all = repo.windowStats().error_collection + const all = (await repo.windowStats()).error_collection assert.equal(all.recent.length, 3) assert.deepEqual(all.ingress_hits.map((h) => h.ip).sort(), ['198.51.100.7', '203.0.113.9']) }) + +test('search keeps ASCII case folding and literal backslashes', async () => { + const db = await freshDb() + await seed(db, [ + { id: 'search-upper', request_id: 'SearchABC\\literal', path: '/v1/messages', status: 200 }, + { id: 'search-nonascii', request_id: 'search-Ä', path: '/v1/messages', status: 200 }, + ]) + const view = new UsageLogsView(db, { mutedErrorClasses: [] }) + assert.deepEqual( + (await view.listBatch({ q: 'searchabc\\lit' })).logs.map((r) => r.id), + ['search-upper'], + ) + assert.deepEqual((await view.listBatch({ q: 'search-ä' })).logs, []) +}) From 86f03351c923f977396d84c3ab04e30f521139e0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:32:20 +0800 Subject: [PATCH 006/473] test(cluster): require host discovery to await platform persistence --- test/unit/cluster-native-node.test.mjs | 77 +++++++++++++++++++++----- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/test/unit/cluster-native-node.test.mjs b/test/unit/cluster-native-node.test.mjs index 289fd1909..4f9110f7c 100644 --- a/test/unit/cluster-native-node.test.mjs +++ b/test/unit/cluster-native-node.test.mjs @@ -3,7 +3,9 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' +import { EventEmitter } from 'node:events' +import { getDb } from '../../src/lib/db/database.mjs' +import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' import { describeNativePayload, isDarwinArm64MachO, @@ -183,13 +185,10 @@ test('node platform comes from uname -s and the darwin host stats parse like /pr assert.equal(stats.containers_mem, 1048576) }) -test('platform is stored per node without touching the edit version', () => { - const db = new DatabaseSync(':memory:') - for (const name of ['026_cluster_nodes.sql', '040_cluster_node_platform.sql']) { - db.exec(fs.readFileSync(new URL(`../../src/lib/db/migrations/${name}`, import.meta.url), 'utf8')) - } +test('platform is stored per node without touching the edit version', async () => { + const db = getDb() const repo = new ClusterNodesRepo(db) - const node = repo.insert({ + const node = await repo.insert({ id: 'node-m', label: 'mini', host: 'mini.example.test', @@ -200,12 +199,64 @@ test('platform is stored per node without touching the edit version', () => { host_key_alg: 'ssh-ed25519', host_key_sha256: 'SHA256:x', }) - assert.equal(repo.platform('node-m'), 'linux') - assert.equal(repo.setPlatform('node-m', 'darwin'), true) - assert.equal(repo.platform('node-m'), 'darwin') - assert.equal(repo.get('node-m').updated_at, node.updated_at) - assert.equal(repo.platform('node-gone'), null) - db.close() + assert.equal(await repo.platform('node-m'), 'linux') + assert.equal(await repo.setPlatform('node-m', 'darwin'), true) + assert.equal(await repo.platform('node-m'), 'darwin') + assert.equal((await repo.get('node-m')).updated_at, node.updated_at) + assert.equal(await repo.platform('node-gone'), null) + await db.close() +}) + +test('remote host discovery waits for the persisted platform and preserves one in-flight read', async () => { + let releaseSave + const savePending = new Promise((resolve) => { + releaseSave = resolve + }) + let signalSave + const saveStarted = new Promise((resolve) => { + signalSave = resolve + }) + const manager = new ClusterManager({ + repo: { + setPlatform: async () => { + signalSave() + await savePending + }, + }, + dataDir: '/unused', + logger: { log() {}, warn() {} }, + }) + manager.nodes.set('node-mac', { id: 'node-mac' }) + manager.platforms.set('node-mac', 'linux') + const client = { + exec(_command, callback) { + const stream = new EventEmitter() + stream.stderr = new EventEmitter() + stream.close = () => {} + callback(null, stream) + queueMicrotask(() => { + stream.emit('data', Buffer.from('501\n20\n/Users/kin\nsudo\nDarwin\n')) + stream.emit('exit', 0) + stream.emit('close') + }) + }, + } + manager.links.set('node-mac', { state: 'ready', client }) + const pending = manager.remoteHost('node-mac') + assert.equal(manager.remoteHost('node-mac'), pending) + let settled = false + pending.then(() => { + settled = true + }) + try { + await saveStarted + await Promise.resolve() + assert.equal(settled, false) + } finally { + releaseSave() + } + assert.equal((await pending).platform, 'darwin') + assert.equal(manager.nodePlatform('node-mac'), 'darwin') }) function bindMac(t, { ready = true, root = '/Users/kin/.vm2api', projectRoot = '/unused' } = {}) { From 440cb47d12d799f52f7d2d0348135b2a4869e2c3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:34:28 +0800 Subject: [PATCH 007/473] test(logging): require maintenance to drain scheduled cleanup --- test/unit/request-log.test.mjs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/unit/request-log.test.mjs b/test/unit/request-log.test.mjs index 1c0542724..38aff8991 100644 --- a/test/unit/request-log.test.mjs +++ b/test/unit/request-log.test.mjs @@ -1123,6 +1123,39 @@ test('tenant log scope follows the slot owner at write time, not the current own assert.equal((await store.queryNormal({})).total, 2) }) +test('stopping retention scheduling still waits for its running cleanup during flush', async (t) => { + const store = tmpStore() + let run + t.mock.method(globalThis, 'setInterval', (callback) => { + run = callback + return { unref() {} } + }) + t.mock.method(globalThis, 'clearInterval', () => {}) + let releaseCleanup + const cleanupPending = new Promise((resolve) => { releaseCleanup = resolve }) + let signalCleanup + const cleanupStarted = new Promise((resolve) => { signalCleanup = resolve }) + store.cleanup = async () => { + signalCleanup() + await cleanupPending + } + store.startCleanupScheduler() + run() + await cleanupStarted + store.stopCleanupScheduler() + let settled = false + let pending + try { + pending = store.flush().then(() => { settled = true }) + await Promise.resolve() + assert.equal(settled, false) + } finally { + releaseCleanup() + } + await pending + assert.equal(settled, true) +}) + test('summary and debug write failures reject finish and response flush', async () => { const store = tmpStore('debug') const query = store.db.query.bind(store.db) From 052dcd9c36ba1242206d86bca1ac0ba550dadd6a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:38:01 +0800 Subject: [PATCH 008/473] test(keys): cover bigint request quotas beyond the safe integer range --- test/unit/api-keys.test.mjs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/unit/api-keys.test.mjs b/test/unit/api-keys.test.mjs index f5911b9aa..fa2c16524 100644 --- a/test/unit/api-keys.test.mjs +++ b/test/unit/api-keys.test.mjs @@ -118,6 +118,19 @@ test('canAccept rejects disabled / expired / quota / concurrency', async () => { assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_quota_exhausted') }) +test('request quotas compare imported bigint counters exactly', async () => { + const store = tmpStore() + const rec = await store.create({ name: 'large quota' }) + await store.db + .prepare('UPDATE api_keys SET quota_requests = $1, quota_requests_used = $2 WHERE id = $3') + .run('10000000000000000', '9007199254740993', rec.id) + assert.equal((await store.canAccept(await store.getById(rec.id))).ok, true) + await store.db.prepare('UPDATE api_keys SET status = $1 WHERE id = $2').run('quota_exhausted', rec.id) + assert.equal((await store.update(rec.id, { name: 'renamed' })).status, 'active') + await store.db.prepare('UPDATE api_keys SET quota_requests_used = quota_requests WHERE id = $1').run(rec.id) + assert.equal((await store.canAccept(await store.getById(rec.id))).code, 'api_key_quota_exhausted') +}) + test('rpm window blocks then recovers', async () => { const store = tmpStore() const rec = await store.create({ name: 'rpm', max_concurrency: 0, quota_requests: 0, rpm: 2 }) From aa112094d443419d85b10703b03bbb89aa196b59 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:39:12 +0800 Subject: [PATCH 009/473] test(quota): expose bigint counter concatenation after PostgreSQL migration --- test/unit/account-quota.test.mjs | 343 +++++++++++++++++-------------- 1 file changed, 193 insertions(+), 150 deletions(-) diff --git a/test/unit/account-quota.test.mjs b/test/unit/account-quota.test.mjs index 744dead6d..df8f20fe3 100644 --- a/test/unit/account-quota.test.mjs +++ b/test/unit/account-quota.test.mjs @@ -10,12 +10,12 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-')) } -test('ensure seeds account and is idempotent', () => { - const q = new AccountQuota({ +test('ensure seeds account and is idempotent', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {}, accounts: [{ account_id: 'a1', vm_id: 'vm-1', email: 'a@x' }], - }) + }).init()) const acc = q.ensure({ account_id: 'a1' }) assert.equal(acc.vm_id, 'vm-1') assert.equal(acc.email, 'a@x') @@ -23,8 +23,8 @@ test('ensure seeds account and is idempotent', () => { assert.equal(acc.unified['5h'].status, 'active') }) -test('a model entitlement rejection is persisted for that account and model', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('a model entitlement rejection is persisted for that account and model', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ensure({ account_id: 'a-denied' }) q.markModelUnsupported('a-denied', 'claude-fable-5') const until = q.repo.get('a-denied').unified.model_denied_until['claude-fable-5'] @@ -38,8 +38,8 @@ function futureReset(msFromNow) { return new Date(Date.now() + msFromNow).toISOString() } -test('ingestHeaders updates unified windows + counters + allocations', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders updates unified windows + counters + allocations', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestHeaders( 'a2', { @@ -61,8 +61,8 @@ test('ingestHeaders updates unified windows + counters + allocations', () => { assert.ok(acc.unified.headers.sampled_at) }) -test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestHeaders('a-health', { 'anthropic-ratelimit-unified-5h-utilization': '0.91', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -86,8 +86,8 @@ test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', assert.equal(acc.recent_allocations.length, 1) }) -test('ingestHeaders accumulates cache tokens and mirrors the session window', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders accumulates cache tokens and mirrors the session window', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) const windows = [] q.attachRuntimeRepo({ updateWindow: (accountId, patch) => windows.push({ accountId, ...patch }) }) q.ingestHeaders( @@ -115,8 +115,8 @@ test('ingestHeaders accumulates cache tokens and mirrors the session window', () assert.equal(windows[0].sessionWindowStatus, 'allowed') }) -test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestHeaders( 'a-oa', {}, @@ -133,8 +133,8 @@ test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', () => { assert.equal(acc.cache_creation_tokens, 3) }) -test('test-chat last_probe does not overwrite a successful usage probe', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('test-chat last_probe does not overwrite a successful usage probe', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.recordLastProbe('a-keep', { ok: true, source: 'vm-oauth-usage' }) q.recordLastProbe('a-keep', { ok: false, @@ -146,11 +146,11 @@ test('test-chat last_probe does not overwrite a successful usage probe', () => { assert.equal(acc.unified.last_probe.ok, true) }) -test('canAccept blocks at safety ratio and records last_blocked', () => { - const q = new AccountQuota({ +test('canAccept blocks at safety ratio and records last_blocked', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestHeaders('a3', { 'anthropic-ratelimit-unified-5h-utilization': '0.96' }) const gate = q.canAccept('a3') assert.equal(gate.ok, false) @@ -159,19 +159,19 @@ test('canAccept blocks at safety ratio and records last_blocked', () => { assert.equal(acc.last_blocked.window, '5h') }) -test('cli rate_limit_event blocks via status', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('cli rate_limit_event blocks via status', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestCliRateLimit('a4', { rateLimitType: 'five_hour', status: 'rejected', resetsAt: 1893456000 }) const gate = q.canAccept('a4') assert.equal(gate.ok, false) assert.equal(gate.reason, 'quota_5h_cli') }) -test('official 26 percent does not trip the quota gate', () => { - const q = new AccountQuota({ +test('official 26 percent does not trip the quota gate', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestHeaders('pct-26', { 'anthropic-ratelimit-unified-5h-utilization': '26', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -200,8 +200,8 @@ test('extraHeadersFromLimitError fills Extra 5h + reset from the text when wrap assert.equal(kept['anthropic-ratelimit-unified-5h-status'], 'allowed_warning') }) -test('limit error without a fresh reset does not inherit the elapsed reset', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('limit error without a fresh reset does not inherit the elapsed reset', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) const elapsed = String(Math.floor((Date.now() - 3_600_000) / 1000)) q.ingestHeaders('elapsed-5h', { 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -218,8 +218,8 @@ test('limit error without a fresh reset does not inherit the elapsed reset', () assert.ok(acc.unified.headers.exhausted_at) }) -test('cli rate_limit_event copies unifiedWindows utilization', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('cli rate_limit_event copies unifiedWindows utilization', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestCliRateLimit('a4-util', { rateLimitType: 'five_hour', status: 'allowed', @@ -231,11 +231,11 @@ test('cli rate_limit_event copies unifiedWindows utilization', () => { assert.equal(acc.unified.headers['5h'].status, 'allowed') }) -test('tryAcquire enforces and releases the configured concurrency cap', () => { - const q = new AccountQuota({ +test('tryAcquire enforces and releases the configured concurrency cap', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), accounts: [{ account_id: 'acc-reserve', max_concurrency: 1, concurrency_override: 1 }], - }) + }).init()) assert.equal(q.tryAcquire('acc-reserve').ok, true) assert.equal(q.tryAcquire('acc-reserve').reason, 'concurrency_limit') q.release('acc-reserve') @@ -243,12 +243,12 @@ test('tryAcquire enforces and releases the configured concurrency cap', () => { q.release('acc-reserve') }) -test('state persists across re-open (same dataDir)', () => { +test('state persists across re-open (same dataDir)', async () => { const dir = tmpDir() - const q1 = new AccountQuota({ dataDir: dir, config: {} }) + const q1 = (await new AccountQuota({ dataDir: dir, config: {} }).init()) q1.ingestHeaders('a5', { 'anthropic-ratelimit-unified-7d-utilization': '0.5' }, { input_tokens: 7, output_tokens: 3 }) - const q2 = new AccountQuota({ dataDir: dir, config: {} }) + const q2 = (await new AccountQuota({ dataDir: dir, config: {} }).init()) const acc = q2.snapshot().accounts.find((a) => a.account_id === 'a5') assert.ok(acc) assert.equal(acc.unified['7d'].utilization, 0.5) @@ -257,8 +257,8 @@ test('state persists across re-open (same dataDir)', () => { assert.equal(acc.recent_allocations.length, 1) }) -test('allocations trimmed to 50 per account', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('allocations trimmed to 50 per account', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) for (let i = 0; i < 60; i++) { q.ingestHeaders('a6', { 'anthropic-ratelimit-unified-5h-utilization': String(i / 100) }) } @@ -268,8 +268,8 @@ test('allocations trimmed to 50 per account', () => { assert.equal(recent[4].util_5h, 0.59) }) -test('max_concurrency 0 blocks instead of treating as unlimited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 20 } } }) +test('max_concurrency 0 blocks instead of treating as unlimited', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 20 } } }).init()) q.ensure({ account_id: 'zero', max_concurrency: 0 }) q.setMaxConcurrency('zero', 0, { override: true }) const gate = q.canAccept('zero') @@ -278,8 +278,8 @@ test('max_concurrency 0 blocks instead of treating as unlimited', () => { assert.equal(gate.detail.max, 0) }) -test('rebindToVm moves the UUID row onto the new slot', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('rebindToVm moves the UUID row onto the new slot', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ensure({ account_id: '1c5a7a73', vm_id: 'vm-02', email: 'old@x' }) q.rebindToVm('1c5a7a73', 'vm-04', { email: 'new@x' }) const acc = q.snapshot().accounts.find((a) => a.account_id === '1c5a7a73') @@ -287,8 +287,8 @@ test('rebindToVm moves the UUID row onto the new slot', () => { assert.equal(acc.email, 'new@x') }) -test('concurrency inflight gate stays in memory', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 1 } } }) +test('concurrency inflight gate stays in memory', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 1 } } }).init()) q.ensure({ account_id: 'a7' }) q.setMaxConcurrency('a7', 1, { override: true }) q.acquire('a7') @@ -299,8 +299,8 @@ test('concurrency inflight gate stays in memory', () => { assert.equal(q.canAccept('a7').ok, true) }) -test('rpm window blocks then recovers', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('rpm window blocks then recovers', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ensure({ account_id: 'rpm-acc', max_concurrency: 8 }) q.setMaxRpm('rpm-acc', 2, { override: true }) assert.equal(q.tryAcquire('rpm-acc').ok, true) @@ -314,8 +314,8 @@ test('rpm window blocks then recovers', () => { assert.ok(gate.detail.reset_at > Date.now()) }) -test('max_rpm 0 is unlimited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('max_rpm 0 is unlimited', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ensure({ account_id: 'rpm-off', max_concurrency: 8, max_rpm: 0 }) for (let i = 0; i < 5; i++) { assert.equal(q.tryAcquire('rpm-off').ok, true) @@ -324,11 +324,11 @@ test('max_rpm 0 is unlimited', () => { assert.equal(q.canAccept('rpm-off').ok, true) }) -test('quota gate still blocks before RPM wait', () => { - const q = new AccountQuota({ +test('quota gate still blocks before RPM wait', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ensure({ account_id: 'rpm-quota', max_concurrency: 8, max_rpm: 60 }) q.ingestHeaders('rpm-quota', { 'anthropic-ratelimit-unified-5h-utilization': '1', @@ -340,11 +340,11 @@ test('quota gate still blocks before RPM wait', () => { assert.equal(gate.reason, 'quota_5h_cli') }) -test('ingestOAuthUsage stores 5h/7d and isolated fable limit', () => { - const q = new AccountQuota({ +test('ingestOAuthUsage stores 5h/7d and isolated fable limit', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-fable', { ok: true, five_hour: { utilization: 0.2, resets_at: '2026-08-18T20:00:00Z', status: 'allowed' }, @@ -374,11 +374,11 @@ test('ingestOAuthUsage stores 5h/7d and isolated fable limit', () => { assert.equal(gate.ok, true, 'fable weekly limit must not block the whole account') }) -test('official /usage 429 keeps the last successful probe', () => { - const q = new AccountQuota({ +test('official /usage 429 keeps the last successful probe', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-rl', { ok: true, source: 'official-cc-usage', @@ -404,11 +404,11 @@ test('official /usage 429 keeps the last successful probe', () => { assert.ok(Date.parse(acc.unified.usage_rate_limited_until) > Date.now()) }) -test('usage ok plus fable 401 does not poison last_probe with revoke text', () => { - const q = new AccountQuota({ +test('usage ok plus fable 401 does not poison last_probe with revoke text', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-pro-401', { ok: true, usage_status: 200, @@ -436,8 +436,8 @@ test('usage ok plus fable 401 does not poison last_probe with revoke text', () = assert.equal(after.unified.account_tier, undefined) }) -test('Fable 403 does not classify Pro when official usage cannot verify the grant', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('Fable 403 does not classify Pro when official usage cannot verify the grant', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestOAuthUsage('acc-uncertain-403', { ok: false, usage_status: 500, @@ -449,8 +449,8 @@ test('Fable 403 does not classify Pro when official usage cannot verify the gran assert.equal(acc.unified.fable.plan_denied, false) }) -test('official usage without Fable evidence does not classify an unprobed account as Pro', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('official usage without Fable evidence does not classify an unprobed account as Pro', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestOAuthUsage('acc-unprobed', { ok: true, usage_status: 200, @@ -463,8 +463,8 @@ test('official usage without Fable evidence does not classify an unprobed accoun assert.equal(acc.unified.fable, undefined) }) -test('complete official usage without Fable classifies Pro', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('complete official usage without Fable classifies Pro', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestOAuthUsage('acc-complete-pro', { ok: true, usage_status: 200, @@ -479,8 +479,8 @@ test('complete official usage without Fable classifies Pro', () => { assert.equal(acc.unified.account_tier, 'pro') }) -test('complete official usage overrides a stale profile tier', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('complete official usage overrides a stale profile tier', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ensure({ account_id: 'acc-stale-profile' }) q.setAccountTier('acc-stale-profile', 'max', { source: 'profile' }) q.ingestOAuthUsage('acc-stale-profile', { @@ -497,8 +497,8 @@ test('complete official usage overrides a stale profile tier', () => { assert.equal(acc.unified.account_tier_source, 'usage') }) -test('setup-token usage scope failure preserves historical tier and does not mark grant dead', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('setup-token usage scope failure preserves historical tier and does not mark grant dead', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ensure({ account_id: 'acc-setup-scope' }) q.setAccountTier('acc-setup-scope', 'max', { source: 'usage' }) q.ingestOAuthUsage('acc-setup-scope', { @@ -515,8 +515,8 @@ test('setup-token usage scope failure preserves historical tier and does not mar assert.match(acc.unified.last_probe.error, /user:profile/) }) -test('incomplete official usage without Fable does not downgrade a stored Max', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('incomplete official usage without Fable does not downgrade a stored Max', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ensure({ account_id: 'acc-incomplete-max' }) q.setAccountTier('acc-incomplete-max', 'max', { source: 'usage' }) q.ingestOAuthUsage('acc-incomplete-max', { @@ -531,11 +531,11 @@ test('incomplete official usage without Fable does not downgrade a stored Max', assert.equal(acc.unified.account_tier, 'max') }) -test('clearGrantRevokeLeftover drops stale revoke after refresh', () => { - const q = new AccountQuota({ +test('clearGrantRevokeLeftover drops stale revoke after refresh', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-stale', { ok: false, usage_status: 401, @@ -552,11 +552,11 @@ test('clearGrantRevokeLeftover drops stale revoke after refresh', () => { assert.equal(acc.unified.account_tier, undefined) }) -test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', () => { - const q = new AccountQuota({ +test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-max', { ok: true, usage_status: 200, @@ -583,11 +583,11 @@ test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', () = assert.equal(acc.unified.fable.plan_denied, false) }) -test('fable 429 without 7d_oi window does not invent a full Fable quota', () => { - const q = new AccountQuota({ +test('fable 429 without 7d_oi window does not invent a full Fable quota', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-pro', { ok: true, five_hour: { utilization: 0.1, status: 'allowed' }, @@ -604,11 +604,11 @@ test('fable 429 without 7d_oi window does not invent a full Fable quota', () => assert.equal(q.fableWindowLimited('acc-pro'), false) }) -test('official usage without Fable hop does not turn leftover 429 into Pro', () => { - const q = new AccountQuota({ +test('official usage without Fable hop does not turn leftover 429 into Pro', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-pro-skip', { ok: true, five_hour: { utilization: 0.1, status: 'allowed' }, @@ -640,11 +640,11 @@ test('official usage without Fable hop does not turn leftover 429 into Pro', () assert.equal(acc.unified.account_tier, undefined) }) -test('Fable hop alone does not classify Pro; complete usage can flip stored Pro to Max', () => { - const q = new AccountQuota({ +test('Fable hop alone does not classify Pro; complete usage can flip stored Pro to Max', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-mispro', { ok: true, usage_status: 200, @@ -673,11 +673,11 @@ test('Fable hop alone does not classify Pro; complete usage can flip stored Pro assert.equal(acc.unified.fable.ok, true) }) -test('7d_oi window is Fable-only and does not block the account', () => { - const q = new AccountQuota({ +test('7d_oi window is Fable-only and does not block the account', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const reset = futureReset(7 * 24 * 3600_000) q.ingestHeaders('acc-oi', { 'anthropic-ratelimit-unified-7d_oi-utilization': '1', @@ -690,11 +690,11 @@ test('7d_oi window is Fable-only and does not block the account', () => { assert.equal(q.fableWindowResetAt('acc-oi'), Date.parse(reset)) }) -test('7d_oi rejected with elapsed second-level reset is not Fable-limited', () => { - const q = new AccountQuota({ +test('7d_oi rejected with elapsed second-level reset is not Fable-limited', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const now = Date.parse('2026-10-19T09:00:00Z') const resetSec = '1792206000' const acc = q.ensure({ account_id: 'acc-oi-elapsed' }) @@ -710,8 +710,8 @@ test('7d_oi rejected with elapsed second-level reset is not Fable-limited', () = assert.equal(q.fableWindowResetAt('acc-oi-elapsed'), parseResetMs(resetSec)) }) -test('7d_oi rejected without reset stays Fable-limited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('7d_oi rejected without reset stays Fable-limited', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) const acc = q.ensure({ account_id: 'acc-oi-no-reset' }) acc.unified['7d_oi'] = { utilization: 1, status: 'rejected' } q.repo.save(acc) @@ -719,8 +719,8 @@ test('7d_oi rejected without reset stays Fable-limited', () => { assert.equal(q.fableWindowResetAt('acc-oi-no-reset'), null) }) -test('7d_oi rejected with future second-level reset stays Fable-limited', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('7d_oi rejected with future second-level reset stays Fable-limited', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) const now = Date.parse('2026-10-19T09:00:00Z') const resetSec = '1792810800' const acc = q.ensure({ account_id: 'acc-oi-future' }) @@ -731,8 +731,8 @@ test('7d_oi rejected with future second-level reset stays Fable-limited', () => assert.equal(q.fableWindowResetAt('acc-oi-future'), parseResetMs(resetSec)) }) -test('persistEffectiveWindows clears elapsed unified 7d_oi', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('persistEffectiveWindows clears elapsed unified 7d_oi', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) const now = Date.parse('2026-10-19T09:00:00Z') const acc = q.ensure({ account_id: 'acc-wipe-oi' }) acc.unified['7d_oi'] = { utilization: 1, reset: '1792206000', status: 'rejected' } @@ -745,11 +745,11 @@ test('persistEffectiveWindows clears elapsed unified 7d_oi', () => { assert.equal(oi.stale_reason, 'reset_elapsed') }) -test('probe 5h rejected with no last_used stays a hard block until reset', () => { - const q = new AccountQuota({ +test('probe 5h rejected with no last_used stays a hard block until reset', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-sticky', { ok: true, @@ -763,8 +763,8 @@ test('probe 5h rejected with no last_used stays a hard block until reset', () => assert.equal(acc.unified['5h'].status, 'rejected') }) -test('official probe fills Extra reset when Extra has usage but no reset', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('official probe fills Extra reset when Extra has usage but no reset', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestHeaders('acc-no-reset', { 'anthropic-ratelimit-unified-5h-utilization': '0.44', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -783,11 +783,11 @@ test('official probe fills Extra reset when Extra has usage but no reset', () => assert.equal(acc.unified.headers['5h'].reset, reset) }) -test('official leftover 100% does not overwrite live Extra allowed', () => { - const q = new AccountQuota({ +test('official leftover 100% does not overwrite live Extra allowed', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestHeaders('acc-live', { 'anthropic-ratelimit-unified-5h-utilization': '0.36', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -807,8 +807,8 @@ test('official leftover 100% does not overwrite live Extra allowed', () => { assert.equal(q.canAccept('acc-live').ok, true) }) -test('successful probe overwrites leftover rejected when official percent is low', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('successful probe overwrites leftover rejected when official percent is low', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestHeaders( 'acc-stale-status', { @@ -831,11 +831,11 @@ test('successful probe overwrites leftover rejected when official percent is low assert.equal(q.canAccept('acc-stale-status').ok, true) }) -test('probe 5h rejected stays blocked when last_used is missing', () => { - const q = new AccountQuota({ +test('probe 5h rejected stays blocked when last_used is missing', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-no-used', { ok: true, source: 'official-cc-usage', @@ -852,11 +852,11 @@ test('probe 5h rejected stays blocked when last_used is missing', () => { assert.equal(gate.reason, 'quota_5h_cli') }) -test('probe 5h rejected stays blocked when the slot was used in this window', () => { - const q = new AccountQuota({ +test('probe 5h rejected stays blocked when the slot was used in this window', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.attachRuntimeRepo({ get: () => ({ last_used_at: Date.now() - 60_000 }), @@ -872,8 +872,8 @@ test('probe 5h rejected stays blocked when the slot was used in this window', () assert.equal(gate.reason, 'quota_5h_cli') }) -test('persistEffectiveWindows writes a cleared 5h after reset elapsed', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('persistEffectiveWindows writes a cleared 5h after reset elapsed', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestOAuthUsage('acc-persist', { ok: true, source: 'vm-oauth-usage', @@ -892,11 +892,11 @@ test('persistEffectiveWindows writes a cleared 5h after reset elapsed', () => { assert.equal(q.canAccept('acc-persist').ok, true) }) -test('elapsed 5h reset is not a hard block even after a live header write', () => { - const q = new AccountQuota({ +test('elapsed 5h reset is not a hard block even after a live header write', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestHeaders('acc-elapsed', { 'anthropic-ratelimit-unified-5h-utilization': '1', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -905,8 +905,8 @@ test('elapsed 5h reset is not a hard block even after a live header write', () = assert.equal(q.canAccept('acc-elapsed').ok, true) }) -test('429 unified exhausted marks headers rejected without inventing official 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('429 unified exhausted marks headers rejected without inventing official 100%', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestHeaders( 'acc-429', { @@ -923,8 +923,8 @@ test('429 unified exhausted marks headers rejected without inventing official 10 assert.equal(q.canAccept('acc-429').ok, false) }) -test('local 429 without quota evidence does not reject the 5h window', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('local 429 without quota evidence does not reject the 5h window', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) q.ingestHeaders('acc-local-429', { 'anthropic-ratelimit-unified-5h-utilization': '0.23', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -956,11 +956,11 @@ test('local 429 without quota evidence does not reject the 5h window', () => { assert.equal(q.canAccept('acc-header-429').ok, false) }) -test('stuck 5h header with growing tokens does not lift official utilization', () => { - const q = new AccountQuota({ +test('stuck 5h header with growing tokens does not lift official utilization', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true }, tiers: { default: { limit_5h: 0.85 } } }, - }) + }).init()) q.ingestOAuthUsage('acc-stall', { ok: true, source: 'official-cc-usage', @@ -985,11 +985,11 @@ test('stuck 5h header with growing tokens does not lift official utilization', ( assert.equal(q.canAccept('acc-stall').ok, true) }) -test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', () => { - const q = new AccountQuota({ +test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) q.ingestOAuthUsage('acc-40', { ok: true, source: 'official-cc-usage', @@ -1006,11 +1006,11 @@ test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', ( assert.equal(q.canAccept('acc-40').ok, true) }) -test('persistEffectiveWindows clears leftover quota cooldown when official is under the gate', () => { - const q = new AccountQuota({ +test('persistEffectiveWindows clears leftover quota cooldown when official is under the gate', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, @@ -1043,11 +1043,11 @@ test('persistEffectiveWindows clears leftover quota cooldown when official is un assert.equal(q.canAccept('acc-reconcile').ok, true) }) -test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', () => { - const q = new AccountQuota({ +test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, @@ -1085,8 +1085,8 @@ test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', () assert.equal(runtime.get('acc-clear').cooldown_reason, null) }) -test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) q.ingestOAuthUsage('acc-scale', { ok: true, source: 'official-cc-usage', @@ -1104,11 +1104,11 @@ test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', () => { assert.notEqual(acc.unified.headers['7d'].utilization, 1) }) -test('official 0.90 plus header 0.40 is schedulable', () => { - const q = new AccountQuota({ +test('official 0.90 plus header 0.40 is schedulable', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const acc = q.ensure({ account_id: 'acc-mix-open' }) acc.unified.official = { '5h': { utilization: 0.9, status: 'allowed', reset: futureReset(4 * 3600_000) }, @@ -1125,11 +1125,11 @@ test('official 0.90 plus header 0.40 is schedulable', () => { assert.equal(q.canAccept('acc-mix-open').ok, true) }) -test('header 0.90 plus official 0.40 hits the 5h gate', () => { - const q = new AccountQuota({ +test('header 0.90 plus official 0.40 hits the 5h gate', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }) + }).init()) const acc = q.ensure({ account_id: 'acc-mix-shut' }) acc.unified.official = { '5h': { utilization: 0.4, status: 'allowed', reset: futureReset(4 * 3600_000) }, @@ -1144,8 +1144,8 @@ test('header 0.90 plus official 0.40 hits the 5h gate', () => { assert.equal(gate.reason, 'quota_5h_safety') }) -test('elapsed Extra is not restored from leftover official 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('elapsed Extra is not restored from leftover official 100%', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) const nextReset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-restore', { ok: true, @@ -1167,8 +1167,8 @@ test('elapsed Extra is not restored from leftover official 100%', () => { assert.equal(q.canAccept('acc-restore').ok, true) }) -test('already-wiped Extra stays empty when official leftover is still 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('already-wiped Extra stays empty when official leftover is still 100%', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) const nextReset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-stale-wipe', { ok: true, @@ -1192,8 +1192,8 @@ test('already-wiped Extra stays empty when official leftover is still 100%', () assert.equal(q.canAccept('acc-stale-wipe').ok, true) }) -test('persistEffectiveWindows heals Extra overwritten by leftover official 100%', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('persistEffectiveWindows heals Extra overwritten by leftover official 100%', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) const reset = futureReset(4 * 3600_000) q.ingestHeaders('acc-heal', { 'anthropic-ratelimit-unified-5h-utilization': '0.36', @@ -1217,8 +1217,8 @@ test('persistEffectiveWindows heals Extra overwritten by leftover official 100%' assert.equal(q.canAccept('acc-heal').ok, true) }) -test('elapsed header reset wipes Extra and becomes schedulable', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }) +test('elapsed header reset wipes Extra and becomes schedulable', async () => { + const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) q.ingestHeaders('acc-wipe', { 'anthropic-ratelimit-unified-5h-utilization': '0.91', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -1231,7 +1231,7 @@ test('elapsed header reset wipes Extra and becomes schedulable', () => { }) test('complete usage may replace a profile-sourced tier', async () => { - const quota = new AccountQuota({ dataDir: tmpDir(), config: {} }) + const quota = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) quota.ensure({ account_id: 'acct-profile', vm_id: 'vm-01' }) quota.setAccountTier('acct-profile', 'pro', { source: 'profile' }) quota.setAccountTier('acct-profile', 'max') @@ -1248,3 +1248,46 @@ test('complete usage may replace a profile-sourced tier', async () => { quota.setAccountTier('acct-profile', 'max', { source: 'profile' }) assert.equal(quota.repo.get('acct-profile').unified.account_tier, 'max') }) + +test('concurrent completed hops persist every quota counter update', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() + q.ensure({ account_id: 'parallel-account' }) + await Promise.all(Array.from({ length: 20 }, () => Promise.resolve().then(() => { + q.ingestHeaders('parallel-account', {}, { input_tokens: 3, output_tokens: 2 }) + }))) + await q.flush() + const stored = await q.persistence.get('parallel-account') + assert.equal(stored.requests, 20) + assert.equal(stored.tokens_in, 60) + assert.equal(stored.tokens_out, 40) +}) + +test('restore reload does not recreate accounts from the original startup seed', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {}, accounts: [{ account_id: 'old-startup-account' }] }).init() + await q.persistence.remove('old-startup-account') + await q.rebind(q.db) + assert.equal(q.repo.get('old-startup-account'), null) + assert.equal(await q.persistence.get('old-startup-account'), null) +}) + +for (const source of ['headers', 'cli']) { + test(`${source} quota counters add exactly beyond JavaScript safe integers`, async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() + q.ensure({ account_id: 'large-counters' }) + await q.flush() + await q.db.prepare(`UPDATE accounts SET requests = $1, tokens_in = $1, tokens_out = $1, + cache_read_tokens = $1, cache_creation_tokens = $1 WHERE id = $2`).run('9007199254740993', 'large-counters') + await q.reload() + if (source === 'headers') q.ingestHeaders('large-counters', {}, { + input_tokens: 2, output_tokens: 3, cache_read_input_tokens: 4, cache_creation_input_tokens: 5, + }) + else q.ingestCliRateLimit('large-counters', null, { input_tokens: 2, output_tokens: 3 }) + await q.flush() + const account = await q.persistence.get('large-counters') + assert.equal(account.requests, '9007199254740994') + assert.equal(account.tokens_in, '9007199254740995') + assert.equal(account.tokens_out, '9007199254740996') + assert.equal(account.cache_read_tokens, source === 'headers' ? '9007199254740997' : '9007199254740993') + assert.equal(account.cache_creation_tokens, source === 'headers' ? '9007199254740998' : '9007199254740993') + }) +} From d2c2b8aa046a72dafc7ff8be19d77ac9730c3767 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:40:22 +0800 Subject: [PATCH 010/473] test(migration): preserve fractional scheduling weights from SQLite --- test/unit/sqlite-import.test.mjs | 73 ++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 test/unit/sqlite-import.test.mjs diff --git a/test/unit/sqlite-import.test.mjs b/test/unit/sqlite-import.test.mjs new file mode 100644 index 000000000..17ff42e18 --- /dev/null +++ b/test/unit/sqlite-import.test.mjs @@ -0,0 +1,73 @@ +import { test, beforeEach, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import crypto from 'node:crypto' +import { createDatabase as sqlite } from '../../src/lib/db/sqlite-legacy.mjs' +import { importSqlite } from '../../src/lib/db/sqlite-import.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +let dir, source +beforeEach(() => { + dir = fs.mkdtempSync('/tmp/vm2api-import-test-') + source = dir + '/source.db' +}) +afterEach(() => fs.rmSync(dir, { recursive: true, force: true })) +function digest() { + return crypto.createHash('sha256').update(fs.readFileSync(source)).digest('hex') +} + +test('offline import preserves source, credentials, bigint, runtime and legacy data, and sequence high-water', async () => { + const old = sqlite({ dbPath: source }) + old.exec( + `CREATE TABLE thinking_signature_owners(key TEXT PRIMARY KEY,account_id TEXT NOT NULL,created_at INTEGER NOT NULL)`, + ) + old.prepare('INSERT INTO settings(key,value) VALUES(?,?)').run('encrypted_fixture', '"enc:synthetic-ciphertext"') + old.prepare('INSERT INTO thinking_signature_owners VALUES(?,?,?)').run('signature', 'account', 9007199254740993n) + old.prepare("INSERT INTO redeem_codes(id,code,value) VALUES(100,'deleted-code',1)").run() + old.prepare('DELETE FROM redeem_codes WHERE id=100').run() + old.close() + const before = digest() + const db = getDb() + const report = await importSqlite({ source, db }) + assert.equal(digest(), before) + assert.equal(report.tables.thinking_signature_owners.count, 1) + assert.equal( + (await db.prepare('SELECT created_at FROM thinking_signature_owners').get()).created_at, + '9007199254740993', + ) + assert.equal( + (await db.prepare("SELECT value FROM settings WHERE key='encrypted_fixture'").get()).value, + '"enc:synthetic-ciphertext"', + ) + const next = await db.prepare("INSERT INTO redeem_codes(code,value) VALUES('new-code',1) RETURNING id").get() + assert.equal(next.id, 101) + await assert.rejects(importSqlite({ source, db }), /target is not empty/) +}) + +test('unmapped source schema is rejected without writing target', async () => { + const old = sqlite({ dbPath: source }) + old.exec('CREATE TABLE unknown_business(id TEXT)') + old.close() + await assert.rejects(importSqlite({ source, db: getDb() }), /unmapped SQLite table/) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM settings').get()).n, 0) +}) + +test('invalid source row rolls back all target data and seeds', async () => { + const old = sqlite({ dbPath: source }) + old.exec("INSERT INTO settings(key,value) VALUES(NULL,'bad')") + old.close() + await assert.rejects(importSqlite({ source, db: getDb() }), { code: '23502' }) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM settings').get()).n, 0) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM groups').get()).n, 2) +}) + +test('fractional scheduling weights survive SQLite import and PostgreSQL updates', async () => { + const old = sqlite({dbPath:source}) + old.prepare('INSERT INTO accounts(id,load_factor) VALUES(?,?)').run('fractional',1.25) + old.close() + const db=getDb() + await importSqlite({source,db}) + assert.equal((await db.prepare("SELECT load_factor FROM accounts WHERE id='fractional'").get()).load_factor,1.25) + const {AccountRuntimeRepo}=await import('../../src/lib/db/repos/account-runtime-repo.mjs') + const record=await new AccountRuntimeRepo(db).upsert({account_id:'fractional',vm_id:'vm-01',weight:2.75}) + assert.equal(record.weight,2.75) +}) From 99f32e18f2d40b30ba100d713973c7eb4474fd6e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:47:21 +0800 Subject: [PATCH 011/473] test(db): expose precision loss in PostgreSQL counters and summaries --- test/unit/bigint-counters.test.mjs | 188 +++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 test/unit/bigint-counters.test.mjs diff --git a/test/unit/bigint-counters.test.mjs b/test/unit/bigint-counters.test.mjs new file mode 100644 index 000000000..ba6101e2a --- /dev/null +++ b/test/unit/bigint-counters.test.mjs @@ -0,0 +1,188 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { runLegacyImport } from '../../src/lib/db/legacy-import.mjs' +import { AccountsRepo } from '../../src/lib/db/repos/accounts-repo.mjs' +import { ApiKeysRepo } from '../../src/lib/db/repos/api-keys-repo.mjs' +import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' +import { UsageLogsView } from '../../src/lib/db/repos/usage-logs-view.mjs' +import { StatisticsRepo } from '../../src/lib/db/repos/statistics-repo.mjs' + +const LARGE = '9007199254740993' +const INPUT_TOTAL = '9007199254740995' +const PROMPT_TOTAL = '9007199254741006' +const TOKEN_TOTAL = '9007199254741011' + +async function seedLogs(db) { + const now = Date.now() + for (const [id, input, output, read, write] of [ + ['large', LARGE, 4, 5, 6], + ['small', 2, 1, 0, 0], + ]) { + await db + .prepare(` + INSERT INTO usage_logs (id, request_id, created_at, model, status, input_tokens, output_tokens, + cache_read_tokens, cache_creation_tokens, vm_id, account_id, api_key_id, outbound_session_id, + protocol, path, total_cost, actual_cost, pricing_model) + VALUES ($1,$2,$3,$4,200,$5,$6,$7,$8,'vm-a','account-a','key-a','session-a', + 'anthropic.messages','/v1/messages',1,1,'synthetic-price') + `) + .run(id, 'request-' + id, new Date(now - 1).toISOString(), 'model-a', input, output, read, write) + } + return now + 1 +} + +test('legacy JSONL and summary inserts preserve exact bigint token fields', async (t) => { + const db = getDb() + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-bigint-import-')) + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + const ts = new Date().toISOString() + const record = { + id: 'legacy-large', + request_id: 'legacy-large', + ts, + status: 200, + input_tokens: LARGE, + output_tokens: LARGE, + cache_read_tokens: LARGE, + cache_creation_tokens: LARGE, + cache_creation_5m_tokens: LARGE, + cache_creation_1h_tokens: LARGE, + web_search_requests: LARGE, + } + const logDir = path.join(dir, 'request-logs') + fs.mkdirSync(logDir) + fs.writeFileSync(path.join(logDir, ts.slice(0, 10) + '.jsonl'), JSON.stringify(record) + '\n') + assert.equal((await runLegacyImport({ dataDir: dir, projectRoot: dir, db })).counts.request_logs, 1) + const logs = new UsageLogsRepo(db) + const imported = await logs.getByRequestId('legacy-large') + for (const field of Object.keys(record).filter((key) => key.endsWith('_tokens') || key === 'web_search_requests')) { + assert.equal(imported[field], LARGE, field) + } + await logs.insertSummary({ ...imported, id: 'copied-large', request_id: 'copied-large' }) + assert.equal((await logs.getByRequestId('copied-large')).input_tokens, LARGE) +}) + +test('bigint counter writers reject an already rounded unsafe Number', async () => { + const db = getDb() + const logs = new UsageLogsRepo(db) + await assert.rejects( + logs.insertSummary({ + id: 'unsafe', + request_id: 'unsafe', + ts: new Date().toISOString(), + input_tokens: Number(LARGE), + }), + RangeError, + ) + const keys = new ApiKeysRepo(db) + await keys.insert({ id: 'key-a', key: 'synthetic-key-a', status: 'active' }) + await assert.rejects(keys.recordUsage('key-a', { tokens_in: Number(LARGE) }), RangeError) + assert.equal((await keys.getById('key-a')).tokens_in, 0) +}) + +test('API key usage preserves both large stored counters and exact string deltas', async () => { + const keys = new ApiKeysRepo(getDb()) + await keys.insert({ id: 'key-a', key: 'synthetic-key-a', status: 'active', requests: LARGE, tokens_in: LARGE }) + const smallDelta = await keys.recordUsage('key-a', { tokens_in: 2 }) + assert.equal(smallDelta.requests, '9007199254740994') + assert.equal(smallDelta.tokens_in, INPUT_TOTAL) + await keys.insert({ id: 'key-b', key: 'synthetic-key-b', status: 'active' }) + const largeDelta = await keys.recordUsage('key-b', { + tokens_in: LARGE, + tokens_out: LARGE, + cache_read_tokens: LARGE, + cache_creation_tokens: LARGE, + }) + for (const field of ['tokens_in', 'tokens_out', 'cache_read_tokens', 'cache_creation_tokens']) { + assert.equal(largeDelta[field], LARGE, field) + } +}) + +test('account updates and allocation history preserve bigint counters', async () => { + const accounts = new AccountsRepo(getDb()) + await accounts.insert({ account_id: 'account-a', vm_id: 'vm-a', requests: LARGE, tokens_in: LARGE }) + const account = await accounts.get('account-a') + account.name = 'renamed' + await accounts.save(account) + await accounts.addAllocation('account-a', { tokens_in: LARGE }) + assert.equal((await accounts.get('account-a')).tokens_in, LARGE) + assert.equal((await accounts.get('account-a')).requests, LARGE) + assert.equal((await accounts.recentAllocations('account-a'))[0].tokens_in, LARGE) +}) + +test('usage totals and billing windows return exact integer sums', async () => { + const db = getDb() + const now = await seedLogs(db) + const logs = new UsageLogsRepo(db) + const totals = await logs.totals() + assert.equal(totals.input_tokens, INPUT_TOTAL) + assert.equal(totals.prompt_tokens, PROMPT_TOTAL) + assert.equal(totals.requests, 2) + assert.equal((await logs.aggregate())[0].input_tokens, INPUT_TOTAL) + const owner = await logs.ownerBilling() + assert.equal(owner.totals.tokens_in, PROMPT_TOTAL) + assert.equal(owner.items[0].tokens_in, PROMPT_TOTAL) + assert.equal((await logs.costByAccount())[0].input_tokens, INPUT_TOTAL) + assert.equal((await logs.costByModel())[0].input_tokens, INPUT_TOTAL) + const rolling = await logs.billingStats({ now }) + assert.equal(rolling.total.input_tokens, INPUT_TOTAL) + assert.equal(rolling.window_5h.input_tokens, INPUT_TOTAL) + assert.equal(rolling.accounts[0].window_5h_tokens, TOKEN_TOTAL) + const aligned = await logs.billingStats({ + now, + accountWindows: [ + { + account_id: 'account-a', + vm_id: 'vm-a', + reset_5h: new Date(now + 60_000).toISOString(), + reset_7d: new Date(now + 60_000).toISOString(), + }, + ], + }) + assert.equal(aligned.window_5h.input_tokens, INPUT_TOTAL) + assert.equal(aligned.accounts[0].window_5h_tokens, TOKEN_TOTAL) + assert.equal((await logs.vmUsageStats({ vmId: 'vm-a' })).history[0].input_tokens, INPUT_TOTAL) + assert.equal((await logs.keyUsageStats({ apiKeyId: 'key-a' })).models[0].tokens, TOKEN_TOTAL) + const window = await logs.windowStats() + assert.equal(window.input_tokens, INPUT_TOTAL) + assert.equal(window.prompt_tokens, PROMPT_TOTAL) +}) + +test('SQL and classified usage summaries return the same exact integers', async () => { + const db = getDb() + const now = await seedLogs(db) + for (const mutedErrorClasses of [[], ['distill']]) { + const view = new UsageLogsView(db, { mutedErrorClasses }) + const summary = await view.summary() + assert.equal(summary.totalInputTokens, INPUT_TOTAL) + assert.equal(summary.totalTokens, TOKEN_TOTAL) + assert.equal(summary.totalRequests, 2) + assert.equal((await view.activeSessions({ now })).sessions[0].totalTokens, TOKEN_TOTAL) + assert.equal( + (await view.exportRows({ include_muted: true })).items.find((row) => row.id === 'large').input_tokens, + LARGE, + ) + } +}) + +test('statistics totals, series, and leaderboards preserve exact integer sums', async () => { + const db = getDb() + const now = await seedLogs(db) + const repo = new StatisticsRepo(db) + const stats = await repo.statistics({ range: 'today', now }) + assert.equal(stats.totals.inputTokens, INPUT_TOTAL) + assert.equal(stats.totals.tokens, TOKEN_TOTAL) + assert.equal(stats.seriesTotals.s0.tokens, TOKEN_TOTAL) + assert.equal( + stats.metrics.tokens.flatMap((point) => point.s0).find((value) => value !== 0), + TOKEN_TOTAL, + ) + const entry = (await repo.leaderboard({ range: 'today', now })).entries[0] + assert.equal(entry.inputTokens, INPUT_TOTAL) + assert.equal(entry.totalTokens, TOKEN_TOTAL) + assert.equal(entry.totalRequests, 2) +}) From 65c545af8719b65def372849202e9c407378e9e5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:50:03 +0800 Subject: [PATCH 012/473] test(panel): require async mutations to remain handled --- test/unit/panel-persistence.test.mjs | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 test/unit/panel-persistence.test.mjs diff --git a/test/unit/panel-persistence.test.mjs b/test/unit/panel-persistence.test.mjs new file mode 100644 index 000000000..b21a8a95f --- /dev/null +++ b/test/unit/panel-persistence.test.mjs @@ -0,0 +1,71 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' + +function setup({ failBefore = false } = {}) { + const sent = [] + let writes = 0 + let changed = 0 + let maintenance = 0 + const handler = createPanelHandler({ + cfg: { paths: { project: '/missing-project' } }, + json: (_res, status, body) => sent.push({ status, body }), + readBody: async () => ({ enabled: false }), + requireAuth: () => true, + clusterRoutes: { handle: (_req, res, _url, respond) => { + changed += 1 + return respond(res, 201, { ok: true }) + } }, + onPersistenceFailure: () => { maintenance += 1 }, + proxyPool: { + writes: { flush: async () => { + writes += 1 + if (failBefore || writes > 1) throw new Error('synthetic persistence failure') + } }, + snapshot: () => ({ config: {}, proxies: [] }), + updateConfig: () => { changed += 1; return { ok: true, config: { enabled: false } } }, + }, + }) + return { handler, sent, changed: () => changed, maintenance: () => maintenance } +} + +test('panel cache mutations send failure when persistence fails before a success response', async () => { + const state = setup() + await state.handler({ method: 'PUT', apiKeyKind: 'master' }, {}, new URL('http://test/api/panel/proxies/config')) + assert.equal(state.changed(), 1) + assert.equal(state.maintenance(), 1) + assert.equal(state.sent.length, 1) + assert.equal(state.sent[0].status, 503) + assert.equal(state.sent[0].body.error.code, 'database_persistence_failed') +}) + +test('an already failed cache queue rejects a new panel mutation before changing state', async () => { + const state = setup({ failBefore: true }) + await state.handler({ method: 'PUT', apiKeyKind: 'master' }, {}, new URL('http://test/api/panel/proxies/config')) + assert.equal(state.changed(), 0) + assert.equal(state.maintenance(), 1) + assert.equal(state.sent[0].status, 503) +}) + +test('delegated cluster responses wait for the same persistence boundary', async () => { + const state = setup() + await state.handler({ method: 'POST', apiKeyKind: 'master' }, {}, new URL('http://test/api/panel/cluster/nodes')) + assert.equal(state.changed(), 1) + assert.equal(state.maintenance(), 1) + assert.equal(state.sent.length, 1) + assert.equal(state.sent[0].status, 503) +}) + +test('a staged panel response remains handled after persistence succeeds', async () => { + const sent = [] + const handler = createPanelHandler({ + cfg: {}, + requireAuth: () => true, + json: (_res, status, body) => { sent.push({ status, body }) }, + }) + + const handled = await handler({ method: 'POST', apiKeyKind: 'managed' }, {}, new URL('http://test/admin/models/refresh')) + assert.equal(handled, true) + assert.equal(sent.length, 1) + assert.equal(sent[0].status, 403) +}) From e712c0b97e584372e2cc5d0d46d3c8e434dcc625 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:52:52 +0800 Subject: [PATCH 013/473] test(stores): preserve bigint values through usage preprocessing --- test/unit/bigint-counters.test.mjs | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/unit/bigint-counters.test.mjs b/test/unit/bigint-counters.test.mjs index ba6101e2a..7efcfd76f 100644 --- a/test/unit/bigint-counters.test.mjs +++ b/test/unit/bigint-counters.test.mjs @@ -10,6 +10,8 @@ import { ApiKeysRepo } from '../../src/lib/db/repos/api-keys-repo.mjs' import { UsageLogsRepo } from '../../src/lib/db/repos/usage-logs-repo.mjs' import { UsageLogsView } from '../../src/lib/db/repos/usage-logs-view.mjs' import { StatisticsRepo } from '../../src/lib/db/repos/statistics-repo.mjs' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { RequestLogStore } from '../../src/lib/admin/request-log.mjs' const LARGE = '9007199254740993' const INPUT_TOTAL = '9007199254740995' @@ -79,9 +81,19 @@ test('bigint counter writers reject an already rounded unsafe Number', async () RangeError, ) const keys = new ApiKeysRepo(db) + await assert.rejects( + keys.insert({ id: 'unsafe-key', key: 'synthetic-unsafe-key', requests: Number(LARGE) }), + RangeError, + ) await keys.insert({ id: 'key-a', key: 'synthetic-key-a', status: 'active' }) await assert.rejects(keys.recordUsage('key-a', { tokens_in: Number(LARGE) }), RangeError) assert.equal((await keys.getById('key-a')).tokens_in, 0) + await assert.rejects(keys.update({ ...(await keys.getById('key-a')), requests: Number(LARGE) }), RangeError) + const accounts = new AccountsRepo(db) + await assert.rejects(accounts.insert({ account_id: 'unsafe-account', requests: Number(LARGE) }), RangeError) + await accounts.insert({ account_id: 'account-a', requests: 0 }) + await assert.rejects(accounts.save({ ...(await accounts.get('account-a')), tokens_in: Number(LARGE) }), RangeError) + await assert.rejects(accounts.addAllocation('account-a', { tokens_in: Number(LARGE) }), RangeError) }) test('API key usage preserves both large stored counters and exact string deltas', async () => { @@ -114,6 +126,35 @@ test('account updates and allocation history preserve bigint counters', async () assert.equal((await accounts.recentAllocations('account-a'))[0].tokens_in, LARGE) }) +test('API key store forwards exact string token deltas to persistence', async () => { + const store = new ApiKeyStore({ db: getDb() }) + const key = await store.create({ name: 'large usage' }) + await store.recordUsage(key.id, { input_tokens: LARGE, cache_creation_input_tokens: LARGE }) + const saved = await store.getById(key.id) + assert.equal(saved.tokens_in, LARGE) + assert.equal(saved.cache_creation_tokens, LARGE) +}) + +test('request logging preserves exact cache subdivisions and web search counts', async (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-bigint-request-')) + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + const store = new RequestLogStore({ db: getDb(), dataDir: dir }) + const ctx = store.start({ method: 'POST', headers: {} }) + await store.finish(ctx, { + status: 200, + usage: { + input_tokens: LARGE, + cache_creation_input_tokens: LARGE, + cache_creation: { ephemeral_5m_input_tokens: LARGE }, + web_search_requests: LARGE, + }, + }) + const saved = await store.repo.getByRequestId(ctx.request_id) + assert.equal(saved.input_tokens, LARGE) + assert.equal(saved.cache_creation_5m_tokens, LARGE) + assert.equal(saved.web_search_requests, LARGE) +}) + test('usage totals and billing windows return exact integer sums', async () => { const db = getDb() const now = await seedLogs(db) From 720a047417d130e0b1bdaec1403aa7b54d0ed58f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:53:33 +0800 Subject: [PATCH 014/473] feat(migration): import SQLite snapshots and convert legacy backups --- package.json | 8 +- scripts/convert-sqlite-backup.mjs | 96 ++++++ scripts/import-sqlite.mjs | 15 + src/lib/db/database.mjs | 14 + src/lib/db/legacy-import.mjs | 291 +++++++----------- .../db/postgres-migrations/001_baseline.sql | 2 +- src/lib/db/sqlite-backup.mjs | 54 ++++ src/lib/db/sqlite-import.mjs | 198 ++++++++++++ test/unit/legacy-import.test.mjs | 46 +-- test/unit/sqlite-backup-conversion.test.mjs | 55 ++++ test/unit/sqlite-import.test.mjs | 16 +- 11 files changed, 579 insertions(+), 216 deletions(-) create mode 100644 scripts/convert-sqlite-backup.mjs create mode 100644 scripts/import-sqlite.mjs create mode 100644 src/lib/db/sqlite-backup.mjs create mode 100644 src/lib/db/sqlite-import.mjs create mode 100644 test/unit/sqlite-backup-conversion.test.mjs diff --git a/package.json b/package.json index a72b1d4bc..dbf054d0d 100644 --- a/package.json +++ b/package.json @@ -19,16 +19,18 @@ "fetch:payload": "node scripts/fetch-payload.mjs", "test:web": "pnpm -C web test", "format": "biome format --write && gofmt -w worker && pnpm -C web format", - "format:check": "biome format && node scripts/check-gofmt.mjs && pnpm -C web format:check" + "format:check": "biome format && node scripts/check-gofmt.mjs && pnpm -C web format:check", + "db:import:sqlite": "node scripts/import-sqlite.mjs", + "db:convert:sqlite-backup": "node scripts/convert-sqlite-backup.mjs" }, "dependencies": { "croner": "^10.0.1", "https-proxy-agent": "^9.1.0", "node-fetch": "^2.7.0", + "pg": "^8.23.1", "socks-proxy-agent": "^10.1.0", "ssh2": "^1.17.0", - "ws": "^8.22.0", - "pg": "^8.23.1" + "ws": "^8.22.0" }, "devDependencies": { "@biomejs/biome": "2.5.11" diff --git a/scripts/convert-sqlite-backup.mjs b/scripts/convert-sqlite-backup.mjs new file mode 100644 index 000000000..16063ba45 --- /dev/null +++ b/scripts/convert-sqlite-backup.mjs @@ -0,0 +1,96 @@ +import fs from 'node:fs' +import path from 'node:path' +import os from 'node:os' +import crypto from 'node:crypto' +import { DatabaseSync } from 'node:sqlite' +import { parseArgs } from 'node:util' +import { createDatabase } from '../src/lib/db/database.mjs' +import { importSqlite } from '../src/lib/db/sqlite-import.mjs' +import { materializeSqliteBackup } from '../src/lib/db/sqlite-backup.mjs' +import { runBackupJob } from '../src/lib/admin/backup-core.mjs' +const { values } = parseArgs({ + options: { + source: { type: 'string' }, + output: { type: 'string' }, + 'vm-dir': { type: 'string' }, + 'config-dir': { type: 'string' }, + }, +}) +if (!values.source || !values.output) + throw new Error( + 'usage: npm run db:convert:sqlite-backup -- --source --output ; target PostgreSQL must be empty', + ) +const output = path.resolve(values.output) +if (fs.existsSync(output)) throw new Error('output already exists') +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-convert-backup-')) +let db, client +try { + const source = await materializeSqliteBackup(path.resolve(values.source), tmp) + for (const [option, folder] of [ + ['vm-dir', 'vms'], + ['config-dir', 'config'], + ]) { + const destination = path.join(tmp, folder) + if (values[option]) { + const directory = path.resolve(values[option]) + if (!fs.statSync(directory).isDirectory()) throw new Error(`${option} must be a directory`) + fs.mkdirSync(destination, { recursive: true, mode: 0o700 }) + for (const name of fs.readdirSync(directory)) { + const file = path.join(directory, name) + if (!name.endsWith('.json') || !fs.lstatSync(file).isFile()) continue + fs.copyFileSync(file, path.join(destination, name)) + fs.chmodSync(path.join(destination, name), 0o600) + } + } + if (!fs.existsSync(destination)) throw new Error(`legacy backup requires --${option} to restore companion files`) + } + const legacy = new DatabaseSync(source, { readOnly: true }) + try { + for (const { id } of legacy.prepare('SELECT id FROM vms').all()) { + if (typeof id !== 'string' || path.basename(id) !== id || !fs.existsSync(path.join(tmp, 'vms', id + '.json'))) + throw new Error('companion VM files do not cover the SQLite VM records') + } + } finally { + legacy.close() + } + db = await createDatabase({ allowLegacySource: true }) + await importSqlite({ source, db }) + client = await db.pool.connect() + await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') + const snapshot = (await client.query('SELECT pg_export_snapshot() AS id')).rows[0].id + const schemaVersion = (await db.prepare('SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1').get()) + .version + const record = await runBackupJob( + { + connectionString: db.connectionString, + schema: db.schema, + schemaVersion, + snapshot, + backupDir: path.dirname(output), + tmpDir: tmp, + projectRoot: tmp, + configDir: path.join(tmp, 'config'), + }, + { + id: 'converted_' + crypto.randomBytes(6).toString('hex'), + createdAt: new Date().toISOString(), + kind: 'converted', + }, + ) + if (record.file_path !== output) { + fs.linkSync(record.file_path, output) + fs.unlinkSync(record.file_path) + } + await client.query('COMMIT') + console.log(JSON.stringify({ ok: true, output, sha256: record.sha256, includes: record.includes })) +} catch (error) { + console.error(error.message) + process.exitCode = 1 +} finally { + if (client) { + await client.query('ROLLBACK').catch(() => {}) + client.release() + } + if (db) await db.close() + fs.rmSync(tmp, { recursive: true, force: true }) +} diff --git a/scripts/import-sqlite.mjs b/scripts/import-sqlite.mjs new file mode 100644 index 000000000..d3050c933 --- /dev/null +++ b/scripts/import-sqlite.mjs @@ -0,0 +1,15 @@ +import { parseArgs } from 'node:util' +import { createDatabase } from '../src/lib/db/database.mjs' +import { importSqlite } from '../src/lib/db/sqlite-import.mjs' +const { values } = parseArgs({ options: { source: { type: 'string' } } }) +if (!values.source) throw new Error('usage: npm run db:import:sqlite -- --source ') +const db = await createDatabase({ allowLegacySource: true }) +try { + const manifest = await importSqlite({ source: values.source, db }) + console.log(JSON.stringify({ ok: true, source_sha256: manifest.source_sha256, tables: manifest.tables })) +} catch (error) { + console.error(error.message) + process.exitCode = 1 +} finally { + await db.close() +} diff --git a/src/lib/db/database.mjs b/src/lib/db/database.mjs index 3566279ef..aee9eba94 100644 --- a/src/lib/db/database.mjs +++ b/src/lib/db/database.mjs @@ -9,6 +9,7 @@ import pg from 'pg' const MIGRATIONS_DIR = fileURLToPath(new URL('./postgres-migrations/', import.meta.url)) const transactions = new AsyncLocalStorage() const maintenance = new AsyncLocalStorage() +const instances = new Set() let singleton = null let opening = null @@ -57,6 +58,7 @@ export class PostgresDatabase { }, }, }) + instances.add(this) this.closed = false this.active = 0 this.maintaining = false @@ -136,6 +138,7 @@ export class PostgresDatabase { } finally { this.closed = true await this.pool.end() + instances.delete(this) } if (failure) throw failure })() @@ -232,6 +235,9 @@ export async function applyMigrations(db, { migrationsDir = MIGRATIONS_DIR } = { } export async function createDatabase(options = {}) { + if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'restore-maintenance.json'))) { + throw new Error('restore maintenance marker exists; complete offline recovery before startup') + } const db = new PostgresDatabase({ ...options, schema: options.schema || process.env.KIN_DATABASE_SCHEMA || 'public' }) try { const version = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) @@ -277,3 +283,11 @@ export async function closeDatabase() { singleton = null if (db) await db.close() } + +/** Drain only explicitly identified test connections before deleting their schema. */ +export async function closeDatabasesForSchema(schema, connectionString) { + const matches = [...instances].filter((db) => db.schema === schema && db.connectionString === connectionString) + const results = await Promise.allSettled(matches.map((db) => db.close())) + const failed = results.find((result) => result.status === 'rejected') + if (failed) throw failed.reason +} diff --git a/src/lib/db/legacy-import.mjs b/src/lib/db/legacy-import.mjs index 5a4f5b0ac..0fd4d4fa1 100644 --- a/src/lib/db/legacy-import.mjs +++ b/src/lib/db/legacy-import.mjs @@ -1,24 +1,8 @@ -/** - * One-time import of legacy JSON stores into SQLite. - * - * Sources (all optional — skipped when missing/corrupt): - * data/api-keys.json → api_keys - * data/account-stats.json → accounts + account_allocations - * data/sticky-map.json → sticky_sessions - * data/proxy-pool.json → proxies + settings.proxy_pool_config - * data/request-logs/*.jsonl → request_logs (recent retainDays) - * data/request-logs/debug/** → request_log_debug - * vms/*.json + active.json → vms + settings.active_vm - * - * Idempotent: guarded by settings.legacy_import_done. Original files are - * left untouched as a rollback safety net (they are no longer read/written, - * except vms/*.json which stay the runtime mirror surface). - */ - +/** One-time import of pre-database JSON files. Originals are never modified. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' -import { getDb, withTransaction } from './database.mjs' +import { getDb, withTransaction, withSavepoint } from './database.mjs' import { SettingsRepo } from './repos/settings-repo.mjs' import { ApiKeysRepo } from './repos/api-keys-repo.mjs' import { AccountsRepo } from './repos/accounts-repo.mjs' @@ -28,23 +12,10 @@ import { VmsRepo } from './repos/vms-repo.mjs' import { UsageLogsRepo } from './repos/usage-logs-repo.mjs' import { isVmRecordFile } from '../vm/vm-file.mjs' -const IMPORT_FLAG = 'legacy_import_done' - -function readJson(file) { - try { - if (!fs.existsSync(file)) return null - return JSON.parse(fs.readFileSync(file, 'utf8')) - } catch { - return undefined // corrupt (≠ missing) - } -} - -export function runLegacyImport({ dataDir, projectRoot, db = null, retainDays = 7 } = {}) { - const d = db || getDb() - const settings = new SettingsRepo(d) - const existing = settings.get(IMPORT_FLAG) +export async function runLegacyImport({ dataDir, projectRoot, db = getDb(), retainDays = 7 } = {}) { + const settings = new SettingsRepo(db) + const existing = await settings.get('legacy_import_done') if (existing) return { imported: false, already: existing } - const counts = { api_keys: 0, accounts: 0, @@ -56,161 +27,119 @@ export function runLegacyImport({ dataDir, projectRoot, db = null, retainDays = vms: 0, errors: 0, } - - withTransaction(d, () => { - // --- api keys --- - try { - const raw = readJson(path.join(dataDir, 'api-keys.json')) - if (raw === undefined) counts.errors += 1 - const repo = new ApiKeysRepo(d) - for (const k of raw?.keys || []) { - if (!k?.id || !k?.key || repo.getById(k.id) || repo.getByKey(k.key)) continue - // legacy `quota_used` was a request counter; the column is USD now - const { quota_used, ...rest } = k - repo.insert({ - ...rest, - quota_requests_used: k.quota_requests_used ?? quota_used ?? 0, - category: String(k.category || 'oauth').toLowerCase() === 'api' ? 'api' : 'oauth', - }) - counts.api_keys += 1 + return withTransaction(db, async () => { + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:legacy-import:${db.schema}`, + ]) + const already = await settings.get('legacy_import_done') + if (already) return { imported: false, already } + const safe = async (fn) => { + try { + return await withSavepoint(db, fn) + } catch { + counts.errors++ + return null } - } catch { - counts.errors += 1 } - - // --- account stats --- - try { - const raw = readJson(path.join(dataDir, 'account-stats.json')) - if (raw === undefined) counts.errors += 1 - const repo = new AccountsRepo(d) - for (const acc of Object.values(raw?.accounts || {})) { - if (!acc?.account_id || repo.get(acc.account_id)) continue - repo.insert(acc) - counts.accounts += 1 - for (const alloc of acc.allocations || []) { - repo.addAllocation(acc.account_id, alloc) - counts.allocations += 1 - } - } - } catch { - counts.errors += 1 - } - - // --- sticky sessions --- - try { - const raw = readJson(path.join(dataDir, 'sticky-map.json')) - if (raw === undefined) counts.errors += 1 - const repo = new StickyRepo(d) - for (const [key, ent] of Object.entries(raw?.sessions || {})) { - if (!key || repo.get(key)) continue - repo.upsert(key, ent) - counts.sticky += 1 + const read = (file) => { + if (!fs.existsSync(file)) return null + try { + return JSON.parse(fs.readFileSync(file, 'utf8')) + } catch { + counts.errors++ + return null } - } catch { - counts.errors += 1 } - - // --- proxy pool --- - try { - const raw = readJson(path.join(dataDir, 'proxy-pool.json')) - if (raw === undefined) counts.errors += 1 - if (raw?.proxies?.length || raw?.config) { - const repo = new ProxiesRepo(d) - if (raw.config) repo.setConfig(raw.config) - const current = repo.loadAll() - if (!current.length && Array.isArray(raw.proxies)) { - repo.replaceAll(raw.proxies) - counts.proxies = raw.proxies.length + const keys = new ApiKeysRepo(db) + for (const record of read(path.join(dataDir, 'api-keys.json'))?.keys || []) + await safe(async () => { + if (!record?.id || !record?.key || (await keys.getById(record.id)) || (await keys.getByKey(record.key))) return + const { quota_used, ...rest } = record + await keys.insert({ + ...rest, + quota_requests_used: record.quota_requests_used ?? quota_used ?? 0, + category: String(record.category || 'oauth').toLowerCase() === 'api' ? 'api' : 'oauth', + }) + counts.api_keys++ + }) + const accounts = new AccountsRepo(db) + for (const account of Object.values(read(path.join(dataDir, 'account-stats.json'))?.accounts || {})) + await safe(async () => { + if (!account?.account_id || (await accounts.get(account.account_id))) return + await accounts.insert(account) + for (const allocation of account.allocations || []) await accounts.addAllocation(account.account_id, allocation) + counts.accounts++ + counts.allocations += (account.allocations || []).length + }) + const sticky = new StickyRepo(db) + for (const [key, record] of Object.entries(read(path.join(dataDir, 'sticky-map.json'))?.sessions || {})) + await safe(async () => { + if (!key || (await sticky.get(key))) return + await sticky.upsert(key, record) + counts.sticky++ + }) + const proxies = read(path.join(dataDir, 'proxy-pool.json')) + if (proxies) + await safe(async () => { + const repo = new ProxiesRepo(db) + if (proxies.config) await repo.setConfig(proxies.config) + if (!(await repo.loadAll()).length && Array.isArray(proxies.proxies)) { + await repo.replaceAll(proxies.proxies) + counts.proxies = proxies.proxies.length } - } - } catch { - counts.errors += 1 + }) + const logs = new UsageLogsRepo(db) + const root = path.join(dataDir, 'request-logs') + const cutoff = Date.now() - retainDays * 86400_000 + const recent = (day) => { + const ms = Date.parse(day + 'T00:00:00Z') + return Number.isFinite(ms) && ms >= cutoff } - - // --- request logs (normal summaries, recent days) --- - try { - const repo = new UsageLogsRepo(d) - const root = path.join(dataDir, 'request-logs') - if (fs.existsSync(root)) { - const cutoff = Date.now() - retainDays * 86400_000 - for (const f of fs.readdirSync(root)) { - if (!/^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)) continue - const dayMs = Date.parse(f.slice(0, 10) + 'T00:00:00Z') - if (Number.isFinite(dayMs) && dayMs < cutoff) continue - try { - const lines = fs.readFileSync(path.join(root, f), 'utf8').split('\n').filter(Boolean) - for (const line of lines) { - try { - const rec = JSON.parse(line) - if (!rec?.id) rec.id = 'log_' + crypto.randomBytes(6).toString('hex') - if (repo.insertSummaryIfAbsent(rec)) counts.request_logs += 1 - } catch { - counts.errors += 1 - } - } - } catch { - counts.errors += 1 - } - } - // --- debug records --- - const dbgRoot = path.join(root, 'debug') - if (fs.existsSync(dbgRoot)) { - for (const day of fs.readdirSync(dbgRoot)) { - if (!/^\d{4}-\d{2}-\d{2}$/.test(day)) continue - const dayMs = Date.parse(day + 'T00:00:00Z') - if (Number.isFinite(dayMs) && dayMs < cutoff) continue - const dir = path.join(dbgRoot, day) - let files = [] - try { - files = fs.readdirSync(dir).filter((x) => x.endsWith('.json')) - } catch { - continue - } - for (const f of files) { - try { - const rec = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) - const reqId = rec?.request_id || f.replace(/\.json$/, '') - if (repo.insertDebugIfAbsent(reqId, rec?.ts || new Date(dayMs).toISOString(), rec)) { - counts.request_log_debug += 1 - } - } catch { - counts.errors += 1 - } - } - } - } + if (fs.existsSync(root)) { + for (const file of fs.readdirSync(root)) { + if (!/^\d{4}-\d{2}-\d{2}\.jsonl$/.test(file) || !recent(file.slice(0, 10))) continue + for (const line of fs.readFileSync(path.join(root, file), 'utf8').split('\n').filter(Boolean)) + await safe(async () => { + const record = JSON.parse(line) + record.id ||= 'log_' + crypto.randomBytes(6).toString('hex') + if (await logs.insertSummaryIfAbsent(record)) counts.request_logs++ + }) } - } catch { - counts.errors += 1 - } - - // --- vms + active --- - try { - const repo = new VmsRepo(d) - const vmsDir = projectRoot ? path.join(projectRoot, 'vms') : null - if (vmsDir && fs.existsSync(vmsDir)) { - for (const f of fs.readdirSync(vmsDir)) { - if (!isVmRecordFile(f)) continue - try { - const file = path.join(vmsDir, f) - const vm = JSON.parse(fs.readFileSync(file, 'utf8')) - if (!vm?.id || repo.get(vm.id)) continue - const st = fs.statSync(file) - repo.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) - counts.vms += 1 - } catch { - counts.errors += 1 - } + const debug = path.join(root, 'debug') + if (fs.existsSync(debug)) + for (const day of fs.readdirSync(debug)) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(day) || !recent(day)) continue + const dir = path.join(debug, day) + if (!fs.statSync(dir).isDirectory()) continue + for (const file of fs.readdirSync(dir).filter((f) => f.endsWith('.json'))) + await safe(async () => { + const record = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) + if ( + await logs.insertDebugIfAbsent( + record.request_id || file.slice(0, -5), + record.ts || day + 'T00:00:00Z', + record, + ) + ) + counts.request_log_debug++ + }) } - const active = readJson(path.join(vmsDir, 'active.json')) - if (active?.active_vm) repo.setActiveVmId(active.active_vm) - } - } catch { - counts.errors += 1 } - - settings.set(IMPORT_FLAG, { at: new Date().toISOString(), counts }) + const dir = projectRoot && path.join(projectRoot, 'vms') + if (dir && fs.existsSync(dir)) { + const vms = new VmsRepo(db) + for (const file of fs.readdirSync(dir).filter(isVmRecordFile)) + await safe(async () => { + const name = path.join(dir, file) + const vm = JSON.parse(fs.readFileSync(name, 'utf8')) + if (!vm?.id || (await vms.get(vm.id))) return + await vms.upsertFromVmJson(vm, { mtimeMs: fs.statSync(name).mtimeMs }) + counts.vms++ + }) + const active = read(path.join(dir, 'active.json')) + if (active?.active_vm) await vms.setActiveVmId(active.active_vm) + } + await settings.set('legacy_import_done', { at: new Date().toISOString(), counts }) + return { imported: true, counts } }) - - return { imported: true, counts } } diff --git a/src/lib/db/postgres-migrations/001_baseline.sql b/src/lib/db/postgres-migrations/001_baseline.sql index 5763dd16e..348169650 100644 --- a/src/lib/db/postgres-migrations/001_baseline.sql +++ b/src/lib/db/postgres-migrations/001_baseline.sql @@ -95,7 +95,7 @@ CREATE TABLE "accounts" ( "credentials" text COLLATE "C" NOT NULL DEFAULT '{}', "extra" text COLLATE "C" NOT NULL DEFAULT '{}', "proxy_id" text COLLATE "C", - "load_factor" bigint, + "load_factor" double precision, "priority" bigint NOT NULL DEFAULT 50, "rate_multiplier" double precision NOT NULL DEFAULT 1.0, "status" text COLLATE "C" NOT NULL DEFAULT 'active', diff --git a/src/lib/db/sqlite-backup.mjs b/src/lib/db/sqlite-backup.mjs new file mode 100644 index 000000000..f947a8104 --- /dev/null +++ b/src/lib/db/sqlite-backup.mjs @@ -0,0 +1,54 @@ +/** Materialize old SQLite archives without extracting archive paths or symlinks. */ +import fs from 'node:fs' +import path from 'node:path' +import { spawn } from 'node:child_process' +import { pipeline } from 'node:stream/promises' +import { createGunzip } from 'node:zlib' +import { runProcess, sha256File } from '../admin/backup-core.mjs' + +async function memberToFile(archive, member, destination) { + fs.mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 }) + const child = spawn('tar', ['-xOzf', archive, '--', member], { stdio: ['ignore', 'pipe', 'ignore'] }) + const done = new Promise((resolve, reject) => { + child.once('error', () => reject(new Error('tar could not start'))) + child.once('close', (code) => (code === 0 ? resolve() : reject(new Error('legacy archive extraction failed')))) + }) + await Promise.all([pipeline(child.stdout, fs.createWriteStream(destination, { mode: 0o600, flags: 'wx' })), done]) +} +export async function materializeSqliteBackup(source, root) { + const database = path.join(root, 'db', 'kin.db') + fs.mkdirSync(path.dirname(database), { recursive: true, mode: 0o700 }) + if (source.endsWith('.db.gz')) { + await pipeline( + fs.createReadStream(source), + createGunzip(), + fs.createWriteStream(database, { mode: 0o600, flags: 'wx' }), + ) + return database + } + if (!source.endsWith('.tar.gz')) throw new Error('legacy backup must be .tar.gz or .db.gz') + const entries = (await runProcess('tar', ['-tzf', source])).split('\n').filter(Boolean) + const seen = new Set() + for (const entry of entries) { + const name = entry.replace(/^\.\//, '') + if (name === '.' || name === '') continue + if (/^(db|vms|config)\/$/.test(name)) { + fs.mkdirSync(path.join(root, name), { recursive: true, mode: 0o700 }) + continue + } + if (!/^(db\/kin\.db|manifest\.json|(vms|config)\/[A-Za-z0-9._-]+\.json)$/.test(name) || seen.has(name)) + throw new Error('legacy archive contains an unsupported path') + seen.add(name) + await memberToFile(source, entry, path.join(root, name)) + } + if (!fs.existsSync(database)) throw new Error('legacy archive has no SQLite snapshot') + const manifest = path.join(root, 'manifest.json') + if (fs.existsSync(manifest)) { + const info = JSON.parse(fs.readFileSync(manifest, 'utf8')) + for (const [name, hash] of Object.entries(info.sha256s || {})) { + if (!seen.has(name) || sha256File(path.join(root, name)) !== hash) + throw new Error('legacy archive checksum mismatch') + } + } + return database +} diff --git a/src/lib/db/sqlite-import.mjs b/src/lib/db/sqlite-import.mjs new file mode 100644 index 000000000..f8da35827 --- /dev/null +++ b/src/lib/db/sqlite-import.mjs @@ -0,0 +1,198 @@ +/** Offline, source-preserving SQLite -> PostgreSQL import. */ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import crypto from 'node:crypto' +import { DatabaseSync, backup } from 'node:sqlite' +import { applyMigrations as upgradeSqlite, createDatabase as createSqlite } from './sqlite-legacy.mjs' +import { withTransaction, quoteIdentifier as q } from './database.mjs' + +const FLAG = 'sqlite_import_done' +const sqliteMigrations = new URL('./migrations/', import.meta.url) +function tables(db) { + return db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name") + .all() + .map((r) => r.name) +} +function columns(db, table) { + return db.prepare(`PRAGMA table_info(${q(table)})`).all() +} +function canonical(row, cols) { + return JSON.stringify( + cols.map((c) => { + const v = row[c.name] + if (v == null) return null + if (/INT/i.test(c.type)) return String(v) + if (v instanceof Uint8Array) return Buffer.from(v).toString('base64') + return v + }), + ) +} +function digestRows(rows, cols) { + const hashes = [] + for (const row of rows) hashes.push(crypto.createHash('sha256').update(canonical(row, cols)).digest('hex')) + hashes.sort() + const digest = crypto.createHash('sha256') + for (const hash of hashes) digest.update(hash) + return { count: hashes.length, sha256: digest.digest('hex') } +} + +export async function importSqlite({ source, db }) { + if (!source || !db) throw new Error('source and PostgreSQL database required') + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-sqlite-import-')) + let original, snapshot, template + try { + original = new DatabaseSync(path.resolve(source), { readOnly: true }) + const file = path.join(tmp, 'snapshot.db') + await backup(original, file) + original.close() + original = null + fs.chmodSync(file, 0o600) + const sourceHash = crypto.createHash('sha256') + for await (const chunk of fs.createReadStream(file)) sourceHash.update(chunk) + const sourceSha = sourceHash.digest('hex') + snapshot = new DatabaseSync(file) + if (!tables(snapshot).includes('schema_migrations')) + throw new Error('source has no recognized SQLite migration history') + const sourceVersions = snapshot.prepare('SELECT version,checksum FROM schema_migrations ORDER BY version').all() + const known = new Set( + fs + .readdirSync(sqliteMigrations) + .filter((f) => /^\d+.*\.sql$/.test(f)) + .map((f) => f.split('_')[0]), + ) + if (sourceVersions.some((r) => !known.has(r.version))) + throw new Error('source contains an unknown SQLite migration') + upgradeSqlite(snapshot) + template = createSqlite({ dbPath: path.join(tmp, 'template.db') }) + template.exec( + 'CREATE TABLE thinking_signature_owners(key TEXT PRIMARY KEY,account_id TEXT NOT NULL,created_at INTEGER NOT NULL)', + ) + const expected = tables(template).filter((t) => t !== 'schema_migrations') + const actual = tables(snapshot).filter((t) => t !== 'schema_migrations') + if (actual.some((t) => !expected.includes(t))) throw new Error('source contains an unmapped SQLite table') + if (!actual.includes('thinking_signature_owners')) + snapshot.exec( + 'CREATE TABLE thinking_signature_owners(key TEXT PRIMARY KEY,account_id TEXT NOT NULL,created_at INTEGER NOT NULL)', + ) + const metadata = new Map(expected.map((t) => [t, columns(template, t)])) + for (const table of expected) { + const sourceCols = columns(snapshot, table).map((c) => c.name) + if (JSON.stringify(sourceCols) !== JSON.stringify(metadata.get(table).map((c) => c.name))) + throw new Error(`source schema mismatch for ${table}`) + } + const sequenceStatement = snapshot.prepare('SELECT name,seq FROM sqlite_sequence') + sequenceStatement.setReadBigInts(true) + const sequences = new Map(sequenceStatement.all().map((r) => [r.name, BigInt(r.seq)])) + const report = await withTransaction(db, async () => { + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:sqlite-import:${db.schema}`, + ]) + await db.exec(`LOCK TABLE ${expected.map(q).join(',')} IN ACCESS EXCLUSIVE MODE`) + for (const table of expected) { + const count = (await db.prepare(`SELECT count(*) AS n FROM ${q(table)}`).get()).n + if (table === 'groups') { + const rows = await db.prepare('SELECT * FROM groups ORDER BY id').all() + const seedCols = columns(template, 'groups') + const seedRows = template.prepare('SELECT * FROM groups ORDER BY id').all() + const seedNames = seedCols.filter((c) => !['created_at', 'updated_at'].includes(c.name)) + if (digestRows(rows, seedNames).sha256 !== digestRows(seedRows, seedNames).sha256) + throw new Error('target groups are not the unchanged baseline seeds') + } else if (Number(count) !== 0) throw new Error(`target is not empty (${table})`) + } + await db.exec('DELETE FROM groups') + const results = {} + // All baseline FKs are deferred, including self-referencing jump nodes. + for (const table of expected) { + const cols = metadata.get(table) + const statement = snapshot.prepare(`SELECT * FROM ${q(table)}`) + statement.setReadBigInts(true) + let batch = [] + const sourceHashes = [] + const flush = async () => { + if (!batch.length) return + const params = [] + const values = batch.map( + (row) => + '(' + + cols + .map((c) => { + const value = row[c.name] + params.push( + typeof value === 'bigint' + ? value.toString() + : value instanceof Uint8Array + ? Buffer.from(value) + : value, + ) + return '$' + params.length + }) + .join(',') + + ')', + ) + await db.query( + `INSERT INTO ${q(table)} (${cols.map((c) => q(c.name)).join(',')}) VALUES ${values.join(',')}`, + params, + ) + batch = [] + } + for (const row of statement.iterate()) { + sourceHashes.push(crypto.createHash('sha256').update(canonical(row, cols)).digest('hex')) + batch.push(row) + if (batch.length >= 100) await flush() + } + await flush() + const sourceDigest = crypto.createHash('sha256') + for (const hash of sourceHashes.sort()) sourceDigest.update(hash) + const before = { count: sourceHashes.length, sha256: sourceDigest.digest('hex') } + sourceHashes.length = 0 + const targetHashes = [] + await db.exec(`DECLARE import_validation NO SCROLL CURSOR FOR SELECT * FROM ${q(table)}`) + try { + for (;;) { + const batch = await db.prepare('FETCH FORWARD 100 FROM import_validation').all() + if (!batch.length) break + for (const row of batch) + targetHashes.push(crypto.createHash('sha256').update(canonical(row, cols)).digest('hex')) + } + } finally { + await db.exec('CLOSE import_validation') + } + const targetDigest = crypto.createHash('sha256') + for (const hash of targetHashes.sort()) targetDigest.update(hash) + const after = { count: targetHashes.length, sha256: targetDigest.digest('hex') } + if (before.count !== after.count || before.sha256 !== after.sha256) + throw new Error(`import validation failed for ${table}`) + results[table] = before + for (const col of cols) { + const seq = await db + .prepare( + `SELECT c.relname,n.nspname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.oid=pg_get_serial_sequence($1,$2)::regclass`, + ) + .get(`${q(db.schema)}.${q(table)}`, col.name) + if (!seq) continue + const maximum = await db.prepare(`SELECT max(${q(col.name)}) AS n FROM ${q(table)}`).get() + const high = [sequences.get(table) || 0n, BigInt(maximum.n || 0)].reduce((a, b) => (a > b ? a : b)) + await db.exec(`ALTER SEQUENCE ${q(seq.nspname)}.${q(seq.relname)} RESTART WITH ${high + 1n}`) + } + } + const manifest = { + at: new Date().toISOString(), + source_sha256: sourceSha, + source_versions: sourceVersions, + tables: results, + } + await db + .prepare('INSERT INTO settings(key,value,updated_at) VALUES($1,$2,$3)') + .run(FLAG, JSON.stringify(manifest), manifest.at) + return manifest + }) + return report + } finally { + original?.close() + snapshot?.close() + template?.close() + fs.rmSync(tmp, { recursive: true, force: true }) + } +} diff --git a/test/unit/legacy-import.test.mjs b/test/unit/legacy-import.test.mjs index 94196db76..d27f5f77f 100644 --- a/test/unit/legacy-import.test.mjs +++ b/test/unit/legacy-import.test.mjs @@ -149,20 +149,20 @@ function seedLegacyFiles() { fs.writeFileSync(path.join(vmsDir, 'active.json'), JSON.stringify({ active_vm: 'vm-l1' })) } -beforeEach(() => { +beforeEach(async () => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-import-')) dataDir = path.join(root, 'data') - openDatabase({ dataDir }) + await openDatabase({ dataDir }) }) -afterEach(() => { - closeDatabase() +afterEach(async () => { + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) }) -test('imports every legacy store and is idempotent', () => { +test('imports every legacy store and is idempotent', async () => { seedLegacyFiles() - const res = runLegacyImport({ dataDir, projectRoot: root }) + const res = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res.imported, true) assert.deepEqual( { ...res.counts }, @@ -179,33 +179,33 @@ test('imports every legacy store and is idempotent', () => { }, ) const db = getDb() - assert.equal(new ApiKeysRepo(db).getById('key_legacy1').tokens_in, 300) - assert.equal(new AccountsRepo(db).get('acct-legacy').requests, 7) - assert.equal(new AccountsRepo(db).allocationCount('acct-legacy'), 2) - assert.equal(new StickyRepo(db).get('conv-legacy').hits, 4) - assert.equal(new ProxiesRepo(db).loadAll()[0].bound_vm_id, 'vm-l1') - assert.equal(new ProxiesRepo(db).getConfig().probe_interval_min, 30) - assert.equal(new UsageLogsRepo(db).count(), 2) - assert.ok(new UsageLogsRepo(db).getDebug('rid-l1')) - const vmRow = new VmsRepo(db).get('vm-l1') + assert.equal((await new ApiKeysRepo(db).getById('key_legacy1')).tokens_in, 300) + assert.equal((await new AccountsRepo(db).get('acct-legacy')).requests, 7) + assert.equal(await new AccountsRepo(db).allocationCount('acct-legacy'), 2) + assert.equal((await new StickyRepo(db).get('conv-legacy')).hits, 4) + assert.equal((await new ProxiesRepo(db).loadAll())[0].bound_vm_id, 'vm-l1') + assert.equal((await new ProxiesRepo(db).getConfig()).probe_interval_min, 30) + assert.equal(await new UsageLogsRepo(db).count(), 2) + assert.ok(await new UsageLogsRepo(db).getDebug('rid-l1')) + const vmRow = await new VmsRepo(db).get('vm-l1') assert.equal(vmRow.access_token, null) assert.equal(vmRow.vm.claude.has_access, true) assert.equal(vmRow.vm.claude.access_token, undefined) - assert.equal(new VmsRepo(db).getActiveVmId(), 'vm-l1') + assert.equal(await new VmsRepo(db).getActiveVmId(), 'vm-l1') // second run skips - const res2 = runLegacyImport({ dataDir, projectRoot: root }) + const res2 = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res2.imported, false) assert.ok(res2.already?.at) - assert.equal(new ApiKeysRepo(db).count(), 1) + assert.equal(await new ApiKeysRepo(db).count(), 1) // original files untouched assert.ok(fs.existsSync(path.join(dataDir, 'api-keys.json'))) assert.ok(fs.existsSync(path.join(dataDir, 'proxy-pool.json'))) }) -test('missing files import cleanly with zero counts', () => { - const res = runLegacyImport({ dataDir, projectRoot: root }) +test('missing files import cleanly with zero counts', async () => { + const res = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res.imported, true) assert.equal( Object.values(res.counts).reduce((a, b) => a + b, 0), @@ -213,14 +213,14 @@ test('missing files import cleanly with zero counts', () => { ) }) -test('corrupt file is skipped, counted as error, others still import', () => { +test('corrupt file is skipped, counted as error, others still import', async () => { seedLegacyFiles() fs.writeFileSync(path.join(dataDir, 'api-keys.json'), '{ not json') - const res = runLegacyImport({ dataDir, projectRoot: root }) + const res = await runLegacyImport({ dataDir, projectRoot: root }) assert.equal(res.imported, true) assert.equal(res.counts.api_keys, 0) assert.ok(res.counts.errors >= 1) assert.equal(res.counts.accounts, 1) assert.equal(res.counts.vms, 1) - assert.ok(new SettingsRepo(getDb()).get('legacy_import_done')) + assert.ok(await new SettingsRepo(getDb()).get('legacy_import_done')) }) diff --git a/test/unit/sqlite-backup-conversion.test.mjs b/test/unit/sqlite-backup-conversion.test.mjs new file mode 100644 index 000000000..19671790a --- /dev/null +++ b/test/unit/sqlite-backup-conversion.test.mjs @@ -0,0 +1,55 @@ +import { test, afterEach } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import zlib from 'node:zlib' +import { spawnSync } from 'node:child_process' +import { createDatabase as sqlite } from '../../src/lib/db/sqlite-legacy.mjs' +let root +afterEach(() => { + if (root) fs.rmSync(root, { recursive: true, force: true }) +}) +function fixture() { + root = fs.mkdtempSync('/tmp/vm2api-convert-test-') + const old = sqlite({ dbPath: root + '/source.db' }) + old.prepare('INSERT INTO vms(id,vm_json) VALUES(?,?)').run('vm-fixture', '{"id":"vm-fixture"}') + old.close() + fs.writeFileSync(root + '/source.db.gz', zlib.gzipSync(fs.readFileSync(root + '/source.db'))) + fs.mkdirSync(root + '/vms') + fs.mkdirSync(root + '/config') + fs.writeFileSync(root + '/vms/vm-fixture.json', '{"id":"vm-fixture","name":"backup fixture"}') + fs.writeFileSync(root + '/config/routing.json', '{}') +} +test('old SQLite db-only backup converts to PG v2 only with companion files', async () => { + fixture() + const output = root + '/converted.tar.gz' + const missing = spawnSync( + process.execPath, + ['scripts/convert-sqlite-backup.mjs', '--source', root + '/source.db.gz', '--output', output], + { encoding: 'utf8', env: process.env }, + ) + assert.equal(missing.status, 1) + assert.match(missing.stderr, /requires --vm-dir/) + assert.equal(fs.existsSync(output), false) + const converted = spawnSync( + process.execPath, + [ + 'scripts/convert-sqlite-backup.mjs', + '--source', + root + '/source.db.gz', + '--output', + output, + '--vm-dir', + root + '/vms', + '--config-dir', + root + '/config', + ], + { encoding: 'utf8', env: process.env }, + ) + assert.equal(converted.status, 0, converted.stderr) + const manifest = JSON.parse(spawnSync('tar', ['-xOzf', output, './manifest.json'], { encoding: 'utf8' }).stdout) + assert.equal(manifest.engine, 'postgresql') + assert.equal(manifest.version, 2) + assert.deepEqual(manifest.includes, { db: true, vms: 1, config: 1 }) + assert.ok(manifest.sha256s['db/postgres.dump']) + assert.ok(manifest.sha256s['vms/vm-fixture.json']) +}) diff --git a/test/unit/sqlite-import.test.mjs b/test/unit/sqlite-import.test.mjs index 17ff42e18..efbbec726 100644 --- a/test/unit/sqlite-import.test.mjs +++ b/test/unit/sqlite-import.test.mjs @@ -61,13 +61,13 @@ test('invalid source row rolls back all target data and seeds', async () => { }) test('fractional scheduling weights survive SQLite import and PostgreSQL updates', async () => { - const old = sqlite({dbPath:source}) - old.prepare('INSERT INTO accounts(id,load_factor) VALUES(?,?)').run('fractional',1.25) + const old = sqlite({ dbPath: source }) + old.prepare('INSERT INTO accounts(id,load_factor) VALUES(?,?)').run('fractional', 1.25) old.close() - const db=getDb() - await importSqlite({source,db}) - assert.equal((await db.prepare("SELECT load_factor FROM accounts WHERE id='fractional'").get()).load_factor,1.25) - const {AccountRuntimeRepo}=await import('../../src/lib/db/repos/account-runtime-repo.mjs') - const record=await new AccountRuntimeRepo(db).upsert({account_id:'fractional',vm_id:'vm-01',weight:2.75}) - assert.equal(record.weight,2.75) + const db = getDb() + await importSqlite({ source, db }) + assert.equal((await db.prepare("SELECT load_factor FROM accounts WHERE id='fractional'").get()).load_factor, 1.25) + const { AccountRuntimeRepo } = await import('../../src/lib/db/repos/account-runtime-repo.mjs') + const record = await new AccountRuntimeRepo(db).upsert({ account_id: 'fractional', vm_id: 'vm-01', weight: 2.75 }) + assert.equal(record.weight, 2.75) }) From 1a5a752042dda278154c529b79233608f7fa0aa9 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:53:49 +0800 Subject: [PATCH 015/473] feat(backup): use PostgreSQL snapshots and guarded restoration --- src/lib/admin/backup-core.mjs | 232 ++++++------ src/lib/admin/backup-service.mjs | 566 +++++++++++++++++------------- src/lib/admin/backup-worker.mjs | 29 +- test/e2e/backup.e2e.test.mjs | 17 +- test/unit/backup-core.test.mjs | 27 ++ test/unit/backup-service.test.mjs | 200 ++++++++--- 6 files changed, 646 insertions(+), 425 deletions(-) create mode 100644 test/unit/backup-core.test.mjs diff --git a/src/lib/admin/backup-core.mjs b/src/lib/admin/backup-core.mjs index 2913b9b80..f78732bec 100644 --- a/src/lib/admin/backup-core.mjs +++ b/src/lib/admin/backup-core.mjs @@ -1,27 +1,9 @@ -/** - * Backup pipeline primitives. - * - * Why this module exists: the whole pipeline (VACUUM INTO of a multi-GB - * database, sha256 of the snapshot, `tar -czf`, sha256 of the archive, staging - * cleanup) is synchronous. Running it on the gateway's main thread stops the - * HTTP server from answering any request for the entire backup — a 3-5 minute - * hole that turns into Cloudflare 524s for /v1 traffic, the panel and /healthz. - * - * So the pipeline lives here and is executed by backup-worker.mjs on its own - * thread with its own database connection, while BackupService stays a thin - * orchestrator on the main thread. - * - * Fallback: when `tar` is unavailable the snapshot is gzipped in memory - * (only sane below ~2 GiB) — unchanged from the original implementation. - */ - +/** PostgreSQL backup primitives. Child processes keep dump/compression off the HTTP thread. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' -import zlib from 'node:zlib' -import { spawnSync } from 'node:child_process' -import { getDb, vacuumInto } from '../db/database.mjs' -import { BackupRepo } from '../db/repos/backup-repo.mjs' +import { spawn, spawnSync } from 'node:child_process' +import { pipeline } from 'node:stream/promises' export function stamp(d = new Date()) { return d.toISOString().replace(/[:.]/g, '-') @@ -63,31 +45,8 @@ function mkdtempOnDataVolume({ tmpDir }) { return fs.mkdtempSync(path.join(base, 'kin-backup-')) } -/** Drop backups past the retention window (pre_restore keeps the newest 3). */ -function applyRetention(repo, retention) { - const drop = [...repo.beyondRetention(retention), ...repo.preRestoreBeyond(3)] - for (const rec of drop) { - try { - if (rec.file_path && fs.existsSync(rec.file_path)) fs.rmSync(rec.file_path, { force: true }) - } catch {} - try { - repo.remove(rec.id) - } catch {} - } - return drop.length -} - -/** - * Run one backup job end to end. Blocking by design — call it from a worker - * thread (or the test process), never from the gateway's main thread. - * - * The caller owns the database connection: open it (openDatabase/openDatabase - * singleton) before calling and close it afterwards. - * - * @returns backup record (status 'ok') - */ -export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, retention = 7 }) { - const repo = new BackupRepo(getDb()) +/** The parent holds the exported snapshot until this asynchronous worker job completes. */ +export async function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null }, onFilesCopied = () => {}) { let tmp = null try { fs.mkdirSync(ctx.backupDir, { recursive: true, mode: 0o700 }) @@ -96,14 +55,14 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, } catch {} tmp = mkdtempOnDataVolume(ctx) - // 1. online-consistent DB snapshot (WAL-safe) - const dbSnap = path.join(tmp, 'db', 'kin.db') - vacuumInto(getDb(), dbSnap) - const dbBytes = fs.statSync(dbSnap).size + const dbSnap = path.join(tmp, 'db', 'postgres.dump') + fs.mkdirSync(path.dirname(dbSnap), { recursive: true, mode: 0o700 }) // 2. vm files + config files const includes = { db: true, vms: 0, config: 0 } - const sha256s = { 'db/kin.db': sha256File(dbSnap) } + const sha256s = {} + if (ctx.projectRoot) fs.mkdirSync(path.join(tmp, 'vms'), { recursive: true }) + if (ctx.configDir) fs.mkdirSync(path.join(tmp, 'config'), { recursive: true }) const vmsDir = ctx.projectRoot ? path.join(ctx.projectRoot, 'vms') : null if (vmsDir && fs.existsSync(vmsDir)) { const outVms = path.join(tmp, 'vms') @@ -111,13 +70,11 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, for (const f of fs.readdirSync(vmsDir)) { if (!/\.json$/.test(f)) continue const src = path.join(vmsDir, f) - try { - if (!fs.statSync(src).isFile()) continue - fs.copyFileSync(src, path.join(outVms, f)) - fs.chmodSync(path.join(outVms, f), 0o600) - sha256s[`vms/${f}`] = sha256File(src) - includes.vms += 1 - } catch {} + if (!fs.statSync(src).isFile()) continue + fs.copyFileSync(src, path.join(outVms, f)) + fs.chmodSync(path.join(outVms, f), 0o600) + sha256s[`vms/${f}`] = sha256File(path.join(outVms, f)) + includes.vms += 1 } } if (ctx.configDir && fs.existsSync(ctx.configDir)) { @@ -126,18 +83,37 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, for (const f of fs.readdirSync(ctx.configDir)) { if (!/\.json$/.test(f)) continue const src = path.join(ctx.configDir, f) - try { - if (!fs.statSync(src).isFile()) continue - fs.copyFileSync(src, path.join(outCfg, f)) - sha256s[`config/${f}`] = sha256File(src) - includes.config += 1 - } catch {} + if (!fs.statSync(src).isFile()) continue + fs.copyFileSync(src, path.join(outCfg, f)) + sha256s[`config/${f}`] = sha256File(path.join(outCfg, f)) + includes.config += 1 } } - // 3. manifest + // Files and exported DB snapshot refer to the same write-free window. + await onFilesCopied() + await runPgTool( + 'pg_dump', + [ + '--format=custom', + '--no-owner', + '--no-privileges', + `--schema=${ctx.schema}`, + `--snapshot=${ctx.snapshot}`, + '--file', + dbSnap, + ], + ctx.connectionString, + ) + fs.chmodSync(dbSnap, 0o600) + const dbBytes = fs.statSync(dbSnap).size + sha256s['db/postgres.dump'] = sha256File(dbSnap) + const manifest = { - version: 1, + version: 2, + engine: 'postgresql', + schema: ctx.schema, + schema_version: ctx.schemaVersion, tool: 'kin-gateway', created_at: createdAt, kind, @@ -146,30 +122,12 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, } fs.writeFileSync(path.join(tmp, 'manifest.json'), JSON.stringify(manifest, null, 2)) - // 4. archive (tar.gz preferred; gzipped db fallback) - let filePath, - fileName, - noteOut = note - if (tarAvailable()) { - fileName = `kin-backup-${stamp(new Date(createdAt))}.tar.gz` - filePath = path.join(ctx.backupDir, fileName) - const r = spawnSync('tar', ['-czf', filePath, '-C', tmp, '.'], { stdio: 'pipe' }) - if (r.status !== 0) { - throw new Error(`tar failed: ${String(r.stderr || '').slice(0, 300)}`) - } - } else { - fileName = `kin-backup-${stamp(new Date(createdAt))}.db.gz` - filePath = path.join(ctx.backupDir, fileName) - fs.writeFileSync(filePath, zlib.gzipSync(fs.readFileSync(dbSnap))) - noteOut = [note, 'tar unavailable — db-only gzip snapshot (vm files rebuilt from DB on restore)'] - .filter(Boolean) - .join(' | ') - } - try { - fs.chmodSync(filePath, 0o600) - } catch {} + const fileName = `kin-backup-${stamp(new Date(createdAt))}.tar.gz` + const filePath = path.join(ctx.backupDir, fileName) + await runProcess('tar', ['-czf', filePath, '-C', tmp, '.']) + fs.chmodSync(filePath, 0o600) - const rec = repo.insert({ + return { id, created_at: createdAt, kind, @@ -181,28 +139,8 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, db_bytes: dbBytes, includes, error: null, - note: noteOut, - }) - applyRetention(repo, retention) - return rec - } catch (e) { - try { - repo.insert({ - id, - created_at: createdAt, - kind, - status: 'failed', - file_path: null, - file_name: null, - size_bytes: null, - sha256: null, - db_bytes: null, - includes: null, - error: String(e.message || e).slice(0, 500), - note, - }) - } catch {} - throw e + note, + } } finally { if (tmp) { try { @@ -211,3 +149,75 @@ export function runBackupJob(ctx, { id, createdAt, kind = 'manual', note = null, } } } + +/** Secrets are passed only through the child environment; stderr may contain data and is never returned. */ +export function pgEnvironment(connectionString) { + let url + try { + url = new URL(connectionString) + } catch { + throw new Error('invalid PostgreSQL connection configuration') + } + if (!['postgres:', 'postgresql:'].includes(url.protocol)) throw new Error('PostgreSQL connection required') + const env = { + ...process.env, + PGHOST: url.searchParams.get('host') || url.hostname.replace(/^\[(.+)\]$/, '$1'), + PGPORT: url.searchParams.get('port') || url.port || '5432', + PGUSER: url.searchParams.get('user') || decodeURIComponent(url.username), + PGPASSWORD: url.searchParams.get('password') || decodeURIComponent(url.password), + PGDATABASE: decodeURIComponent(url.pathname.slice(1)), + PGCONNECT_TIMEOUT: '10', + } + for (const [key, name] of Object.entries({ + sslmode: 'PGSSLMODE', + sslcert: 'PGSSLCERT', + sslkey: 'PGSSLKEY', + sslrootcert: 'PGSSLROOTCERT', + connect_timeout: 'PGCONNECT_TIMEOUT', + application_name: 'PGAPPNAME', + options: 'PGOPTIONS', + })) { + if (url.searchParams.has(key)) env[name] = url.searchParams.get(key) + } + return env +} + +export function runProcess(command, args, env = process.env) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'ignore'] }) + let stdout = '' + child.stdout.on('data', (chunk) => { + stdout += chunk.toString() + if (stdout.length > 1024 * 1024) { + child.kill() + reject(new Error(`${command} output exceeds the archive metadata limit`)) + } + }) + child.once('error', () => reject(new Error(`${command} could not start`))) + child.once('close', (code) => + code === 0 ? resolve(stdout) : reject(new Error(`${command} failed (exit ${code})`)), + ) + }) +} + +export function runPgTool(command, args, connectionString) { + return runProcess(command, args, pgEnvironment(connectionString)) +} + +/** Hash large archives without blocking health checks on the main thread. */ +export async function sha256FileAsync(file) { + const hash = crypto.createHash('sha256') + for await (const chunk of fs.createReadStream(file)) hash.update(chunk) + return hash.digest('hex') +} + +/** Extract only a selected member's bytes; archive symlinks are never installed on disk. */ +export async function extractArchiveMember(archive, member, destination) { + fs.mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 }) + const child = spawn('tar', ['-xOzf', archive, '--', member], { stdio: ['ignore', 'pipe', 'ignore'] }) + const done = new Promise((resolve, reject) => { + child.once('error', () => reject(new Error('tar could not start'))) + child.once('close', (code) => (code === 0 ? resolve() : reject(new Error('backup archive extraction failed')))) + }) + await Promise.all([pipeline(child.stdout, fs.createWriteStream(destination, { mode: 0o600, flags: 'wx' })), done]) +} diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index 448ea8355..ae1da8455 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -1,53 +1,13 @@ -/** - * Local backup service — sub2api BackupService counterpart (no S3, per user). - * - * Backup artifact: data/backups/kin-backup-.tar.gz (0600) containing - * manifest.json {version, created_at, kind, includes, sha256s} - * db/kin.db VACUUM INTO online-consistent snapshot (credentials - * already mirrored into the `vms` table) - * vms/ slot *.json + active.json (double safety) - * config/ routing.json / intercept-rules.json … - * - * When `tar` is unavailable the service degrades to a gzipped DB snapshot - * (kin-backup-.db.gz) — still fully restorable because the DB holds - * the credential mirror (vm files are rebuilt from vms.vm_json). That fallback - * buffers the whole snapshot in memory, so it only works below ~2 GiB; the - * tar path has no such limit. - * - * Staging lives on the data volume (`/tmp`, override with - * KIN_BACKUP_TMPDIR), not os.tmpdir(): /tmp is a tmpfs sized to a fraction of - * RAM and a multi-GB `VACUUM INTO` snapshot fails there with SQLITE_FULL - * ("database or disk is full") while the real disk still has room. - * - * Scheduling (default ON, user requirement): - * settings.backup_schedule = { enabled: true, interval_hours: 24, retention: 7 } - * env overrides: KIN_BACKUP_DISABLED=1, KIN_BACKUP_INTERVAL_HOURS, KIN_BACKUP_RETENTION - * - checker runs every 10 min + a catch-up check shortly after boot - * - the job itself runs in a worker thread (backup-worker.mjs): the - * pipeline is synchronous for minutes, so running it in-process - * would stop the gateway from answering (Cloudflare 524) - * - retention prunes old ok backups (pre_restore keeps newest 3) - * - * Restore: - * verify manifest/sha256 → automatic pre_restore backup → close DB → - * swap kin.db (old one kept as kin.db.pre-restore) → restore vms/ + - * config/ files (or rebuild from DB) → reopen + migrate → rebind stores. - * `isRestoring` flag lets the server 503 protocol traffic meanwhile. - */ - +/** PostgreSQL backup scheduling, write maintenance and transactional restore. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' -import zlib from 'node:zlib' -import { spawnSync } from 'node:child_process' import { Worker } from 'node:worker_threads' -import { getDb, getDbPath, closeDatabase, openDatabase } from '../db/database.mjs' -import { sha256File } from './backup-core.mjs' +import { getDb, applyMigrations } from '../db/database.mjs' +import { sha256FileAsync, runPgTool, runProcess, extractArchiveMember } from './backup-core.mjs' import { SettingsRepo } from '../db/repos/settings-repo.mjs' import { BackupRepo } from '../db/repos/backup-repo.mjs' -import { reconcileVms } from '../vm/vm-db-sync.mjs' -import { repairSlotSmbios, repairVmProxyCliEnabled } from '../vm/vm-registry.mjs' -import { invalidateReadCache, isVmRecordFile } from '../vm/vm-file.mjs' +import { invalidateReadCache } from '../vm/vm-file.mjs' // tarAvailability / sha256File / the runnable pipeline live in backup-core.mjs; // keep the historical export surface for tests and callers. @@ -69,7 +29,11 @@ export class BackupService { this.backupDir = backupDir || path.join(dataDir, 'backups') this.tmpDir = tmpDir || path.join(dataDir, 'tmp') this.checkIntervalMs = checkIntervalMs - this.isRestoring = false + this.maintenanceFile = path.join(dataDir, 'restore-maintenance.json') + this.isRestoring = fs.existsSync(this.maintenanceFile) + this.isMaintaining = this.isRestoring + this._restoreRunning = false + this._maintenance = null this._running = false this._timer = null this._bootTimer = null @@ -98,10 +62,27 @@ export class BackupService { this._rebindStores = typeof fn === 'function' ? fn : null } + /** The server pauses new writes, drains in-flight work, and resumes only after success. */ + onMaintenance(fn) { + this._maintenance = typeof fn === 'function' ? fn : null + } + + async _enterMaintenance(reason) { + if (this.isMaintaining) return false + this.isMaintaining = true + await this._maintenance?.(true, reason) + return true + } + + async _leaveMaintenance(reason) { + await this._maintenance?.(false, reason) + this.isMaintaining = false + } + // ---------- schedule config ---------- - getSchedule() { - let cfg = { ...DEFAULT_SCHEDULE, ...(this._settings().get(SCHEDULE_KEY, {}) || {}) } + async getSchedule() { + let cfg = { ...DEFAULT_SCHEDULE, ...((await this._settings().get(SCHEDULE_KEY, {})) || {}) } if (process.env.KIN_BACKUP_DISABLED === '1') cfg.enabled = false if (process.env.KIN_BACKUP_INTERVAL_HOURS) { const n = Number(process.env.KIN_BACKUP_INTERVAL_HOURS) @@ -114,8 +95,8 @@ export class BackupService { return cfg } - updateSchedule(patch = {}) { - const cur = { ...DEFAULT_SCHEDULE, ...(this._settings().get(SCHEDULE_KEY, {}) || {}) } + async updateSchedule(patch = {}) { + const cur = { ...DEFAULT_SCHEDULE, ...((await this._settings().get(SCHEDULE_KEY, {})) || {}) } if (patch.enabled != null) cur.enabled = !!patch.enabled if (patch.interval_hours != null) { const n = Number(patch.interval_hours) @@ -131,99 +112,149 @@ export class BackupService { } cur.retention = n } - this._settings().set(SCHEDULE_KEY, cur) + await this._settings().set(SCHEDULE_KEY, cur) return this.getSchedule() } - nextAutoAt() { - const cfg = this.getSchedule() + async nextAutoAt() { + const cfg = await this.getSchedule() if (!cfg.enabled) return null - const last = this._repo().lastSuccessful({}) + const last = await this._repo().lastSuccessful({}) const base = last ? Date.parse(last.created_at) : 0 return new Date(Math.max(Date.now(), base + cfg.interval_hours * 3600_000)).toISOString() } // ---------- create ---------- - /** - * Run one backup off the main thread. - * - * The pipeline (VACUUM INTO + sha256 + tar.gz) is synchronous for minutes on a - * multi-GB database, so it runs in a worker with its own DB connection while - * the gateway keeps serving requests. - * - * @returns {Promise} - */ - async createBackup({ kind = 'manual', note = null } = {}) { + async flush() { + await Promise.all([this._pendingBackup, this._pendingRestore].filter(Boolean)) + } + + async createBackup(options = {}) { + if (this._restoreRunning && options.kind !== 'pre_restore') + throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) + if (this._running) throw Object.assign(new Error('backup already in progress'), { code: 'backup_in_progress' }) + const pending = this._createBackup(options) + this._pendingBackup = pending + try { + return await pending + } finally { + if (this._pendingBackup === pending) this._pendingBackup = null + } + } + + /** Export a PostgreSQL snapshot while files are frozen, then dump in a worker. */ + async _createBackup({ kind = 'manual', note = null } = {}) { if (this._running) throw Object.assign(new Error('backup already in progress'), { code: 'backup_in_progress' }) this._running = true const id = 'bak_' + crypto.randomBytes(6).toString('hex') const createdAt = new Date().toISOString() + let client + let entered = false try { - return await this._runBackupWorker({ - id, - createdAt, - kind, - note, - retention: this.getSchedule().retention, - }) - } catch (e) { - // The worker records its own failure; this covers the cases where it never - // got that far (module error, OOM, kill) so the panel still shows the job. + entered = await this._enterMaintenance(kind === 'pre_restore' ? 'restore' : 'backup') + const db = getDb() + client = await db.pool.connect() + await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') + const snapshot = (await client.query('SELECT pg_export_snapshot() AS snapshot')).rows[0].snapshot + const version = (await client.query('SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1')) + .rows[0] + const rec = await this._runBackupWorker( + { id, createdAt, kind, note }, + { + connectionString: db.connectionString, + schema: db.schema, + snapshot, + schemaVersion: version?.version || null, + }, + async () => { + if (entered) { + await this._leaveMaintenance('backup') + entered = false + } + }, + ) + await client.query('COMMIT') + client.release() + client = null + const stored = await this._repo().insert(rec) + const retention = (await this.getSchedule()).retention + for (const old of [ + ...(await this._repo().beyondRetention(retention)), + ...(await this._repo().preRestoreBeyond(3)), + ]) { + if (old.file_path) fs.rmSync(old.file_path, { force: true }) + await this._repo().remove(old.id) + } + return stored + } catch (error) { try { - if (!this._repo().get(id)) { - this._repo().insert({ + if (!(await this._repo().get(id))) + await this._repo().insert({ id, created_at: createdAt, kind, status: 'failed', - file_path: null, - file_name: null, - size_bytes: null, - sha256: null, - db_bytes: null, - includes: null, - error: String(e.message || e).slice(0, 500), + error: String(error.message || error).slice(0, 500), note, }) - } } catch {} - throw e + throw error } finally { + if (client) { + try { + await client.query('ROLLBACK') + } finally { + client.release() + } + } this._running = false + if (entered) await this._leaveMaintenance('backup') } } - _runBackupWorker(job) { + _runBackupWorker(job, database, onFilesCopied) { return new Promise((resolve, reject) => { const worker = new Worker(new URL('./backup-worker.mjs', import.meta.url), { workerData: { - dataDir: this.dataDir, ctx: { projectRoot: this.projectRoot, configDir: this.configDir, backupDir: this.backupDir, tmpDir: this.tmpDir, + ...database, }, job, }, }) let settled = false + let resumed = Promise.resolve() const finish = (fn, arg) => { if (settled) return settled = true fn(arg) } - worker.once('message', (msg) => { - if (msg && msg.ok) return finish(resolve, msg.rec) - finish( - reject, - Object.assign(new Error((msg && msg.error && msg.error.message) || 'backup failed'), { - code: (msg && msg.error && msg.error.code) || 'backup_failed', - }), - ) + worker.on('message', (msg) => { + if (msg.type === 'files-copied') { + resumed = onFilesCopied() + // Keep the exported snapshot alive until the worker finishes even when resume failed. + resumed.catch(() => {}) + return + } + if (msg.ok) + resumed.then( + () => finish(resolve, msg.rec), + (error) => finish(reject, error), + ) + else + finish( + reject, + Object.assign(new Error(msg.error?.message || 'backup failed'), { + code: msg.error?.code || 'backup_failed', + }), + ) }) - worker.once('error', (e) => finish(reject, e)) + worker.once('error', (error) => finish(reject, error)) worker.once('exit', (code) => { if (code !== 0) finish(reject, new Error(`backup worker exited with code ${code}`)) }) @@ -232,21 +263,19 @@ export class BackupService { // ---------- list / delete ---------- - list({ limit = 100 } = {}) { - return this._repo() - .list({ limit }) - .map((r) => ({ - ...r, - file_exists: !!(r.file_path && fs.existsSync(r.file_path)), - })) + async list({ limit = 100 } = {}) { + return (await this._repo().list({ limit })).map((r) => ({ + ...r, + file_exists: !!(r.file_path && fs.existsSync(r.file_path)), + })) } - get(id) { + async get(id) { return this._repo().get(id) } - remove(id) { - const rec = this._repo().get(id) + async remove(id) { + const rec = await this._repo().get(id) if (!rec) return false try { if (rec.file_path && fs.existsSync(rec.file_path)) fs.rmSync(rec.file_path, { force: true }) @@ -256,139 +285,208 @@ export class BackupService { // ---------- restore ---------- - /** - * Restore from a recorded backup. Automatically snapshots the current - * state first (kind=pre_restore). Re-binds stores via onRestored callback. - */ - async restoreBackup(id) { - const rec = this._repo().get(id) - if (!rec) throw Object.assign(new Error('backup not found'), { code: 'backup_not_found' }) - if (rec.status !== 'ok') - throw Object.assign(new Error('backup record is not restorable'), { code: 'backup_not_ok' }) - if (!rec.file_path || !fs.existsSync(rec.file_path)) { - throw Object.assign(new Error('backup file missing on disk'), { code: 'backup_file_missing' }) - } - if (this.isRestoring) throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) - - // integrity: archive checksum must match the record - const actual = sha256File(rec.file_path) - if (rec.sha256 && actual !== rec.sha256) { - throw Object.assign(new Error('backup archive sha256 mismatch'), { code: 'backup_corrupt' }) - } - - this.isRestoring = true - let tmp = null + async _extractArchive(rec) { + if (!rec.file_name?.endsWith('.tar.gz')) + throw Object.assign(new Error('convert the SQLite backup with the offline import tool before restoring'), { + code: 'legacy_backup_conversion_required', + }) + const tmp = this._mkdtemp('kin-restore-') try { - // 0. safety net snapshot of the current state - const pre = await this.createBackup({ kind: 'pre_restore', note: `before restore of ${rec.id}` }) - - // 1. extract / decompress - tmp = this._mkdtemp('kin-restore-') - let dbFile - let manifest = null - if (rec.file_name?.endsWith('.tar.gz')) { - const r = spawnSync('tar', ['-xzf', rec.file_path, '-C', tmp], { stdio: 'pipe' }) - if (r.status !== 0) throw new Error(`tar extract failed: ${String(r.stderr || '').slice(0, 300)}`) - dbFile = path.join(tmp, 'db', 'kin.db') - try { - manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) - } catch {} - if (manifest?.sha256s?.['db/kin.db']) { - if (sha256File(dbFile) !== manifest.sha256s['db/kin.db']) { - throw Object.assign(new Error('db snapshot sha256 mismatch inside archive'), { code: 'backup_corrupt' }) - } + const entries = (await runProcess('tar', ['-tzf', rec.file_path])).split('\n').filter(Boolean) + const files = new Set() + const members = new Set() + for (const entry of entries) { + const name = entry.replace(/^\.\//, '') + if (!name || name === '.') continue + if (members.has(name)) throw new Error('invalid or duplicate backup archive path') + members.add(name) + if (name === 'db/kin.db') + throw Object.assign(new Error('convert the SQLite backup with the offline import tool before restoring'), { + code: 'legacy_backup_conversion_required', + }) + if (/^(db|vms|config)\/$/.test(name)) { + fs.mkdirSync(path.join(tmp, name), { recursive: true, mode: 0o700 }) + continue } - } else { - dbFile = path.join(tmp, 'kin.db') - fs.writeFileSync(dbFile, zlib.gunzipSync(fs.readFileSync(rec.file_path))) - } - if (!fs.existsSync(dbFile)) throw new Error('no db snapshot inside backup') - - // 2. swap the live database - const dbPath = getDbPath() - closeDatabase() - const keep = `${dbPath}.pre-restore` - try { - fs.rmSync(keep, { force: true }) - } catch {} - try { - fs.renameSync(dbPath, keep) - } catch {} - for (const suffix of ['-wal', '-shm']) { - try { - fs.rmSync(dbPath + suffix, { force: true }) - } catch {} + if (!/^(manifest\.json|db\/postgres\.dump|(vms|config)\/[^/]+\.json)$/.test(name) || files.has(name)) { + throw new Error('invalid or duplicate backup archive path') + } + files.add(name) + await extractArchiveMember(rec.file_path, entry, path.join(tmp, name)) } - fs.copyFileSync(dbFile, dbPath) - try { - fs.chmodSync(dbPath, 0o600) - } catch {} - - // 3. restore vm + config files from the archive (tar flavor only) - const vmsDirSrc = path.join(tmp, 'vms') - if (this.projectRoot && fs.existsSync(vmsDirSrc)) { - const vmsDir = path.join(this.projectRoot, 'vms') - fs.mkdirSync(vmsDir, { recursive: true }) - // replace vm json records with archive state (cli-home dirs untouched) - for (const f of fs.readdirSync(vmsDir)) { - if (isVmRecordFile(f) || f === 'active.json') { - try { - fs.rmSync(path.join(vmsDir, f), { force: true }) - } catch {} - } + const manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) + if (manifest.version !== 2 || manifest.engine !== 'postgresql') + throw Object.assign(new Error('convert the SQLite backup with the offline import tool before restoring'), { + code: 'legacy_backup_conversion_required', + }) + if (manifest.schema !== getDb().schema) throw new Error('backup schema does not match this application') + if (!manifest.sha256s?.['db/postgres.dump']) throw new Error('backup database checksum is missing') + for (const [directory, required] of [ + ['vms', !!this.projectRoot], + ['config', !!this.configDir], + ]) { + const count = [...files].filter((name) => name.startsWith(`${directory}/`)).length + if (manifest.includes?.[directory] !== count || (required && !fs.existsSync(path.join(tmp, directory)))) { + throw new Error('backup file snapshot is incomplete') } - for (const f of fs.readdirSync(vmsDirSrc)) { - fs.copyFileSync(path.join(vmsDirSrc, f), path.join(vmsDir, f)) - try { - fs.chmodSync(path.join(vmsDir, f), 0o600) - } catch {} + } + for (const name of files) { + if (name !== 'manifest.json' && !manifest.sha256s[name]) throw new Error('backup contains an unchecked file') + } + for (const [name, digest] of Object.entries(manifest.sha256s)) { + if ( + path.isAbsolute(name) || + name.split('/').includes('..') || + (await sha256FileAsync(path.join(tmp, name))) !== digest + ) { + throw Object.assign(new Error('backup file checksum mismatch'), { code: 'backup_corrupt' }) } } - const cfgDirSrc = path.join(tmp, 'config') - if (this.configDir && fs.existsSync(cfgDirSrc)) { - fs.mkdirSync(this.configDir, { recursive: true }) - for (const f of fs.readdirSync(cfgDirSrc)) { - fs.copyFileSync(path.join(cfgDirSrc, f), path.join(this.configDir, f)) + const dump = path.join(tmp, 'db', 'postgres.dump') + const listing = await runPgTool('pg_restore', ['--list', dump], getDb().connectionString) + const schema = getDb().schema + const allowed = + /^(TABLE DATA|SEQUENCE OWNED BY|SEQUENCE SET|FK CONSTRAINT|TABLE|SEQUENCE|DEFAULT|CONSTRAINT|INDEX|SCHEMA|COMMENT) (\S+) (.+)$/ + for (const line of listing.split('\n')) { + if (!line || line.startsWith(';')) continue + const entry = line.replace(/^\d+; \d+ \d+ /, '') + if (/^(ENCODING - ENCODING|STDSTRINGS - STDSTRINGS|SEARCHPATH - SEARCHPATH)( |$)/.test(entry)) continue + if (entry.startsWith(`COMMENT - SCHEMA ${schema} `)) continue + if (entry.startsWith(`FUNCTION ${schema} ascii_fold(text) `)) continue + const item = entry.match(allowed) + if ( + !item || + (item[1] === 'SCHEMA' ? item[2] !== '-' || !item[3].startsWith(`${schema} `) : item[2] !== schema) + ) { + throw new Error('backup contains objects outside the application schema allowlist') } } + return { tmp, dump, manifest } + } catch (error) { + fs.rmSync(tmp, { recursive: true, force: true }) + throw error + } + } - // 4. reopen + migrate (old backups upgrade automatically) - const db = openDatabase({ dataDir: this.dataDir, dbPath }) - - // 5. db-only backups: rebuild vm files from the credential mirror - if (this.projectRoot) { - invalidateReadCache() - try { - reconcileVms(this.projectRoot) - try { - repairVmProxyCliEnabled(this.projectRoot) - } catch {} - try { - repairSlotSmbios(this.projectRoot) - } catch {} - } catch {} + _restoreFiles(tmp) { + if (this.projectRoot && fs.existsSync(path.join(tmp, 'vms'))) { + const target = path.join(this.projectRoot, 'vms') + fs.mkdirSync(target, { recursive: true }) + for (const name of fs.readdirSync(target)) { + if (name.endsWith('.json') && !fs.lstatSync(path.join(target, name)).isDirectory()) + fs.rmSync(path.join(target, name), { force: true }) } + for (const name of fs.readdirSync(path.join(tmp, 'vms'))) { + fs.copyFileSync(path.join(tmp, 'vms', name), path.join(target, name)) + fs.chmodSync(path.join(target, name), 0o600) + } + } + if (this.configDir && fs.existsSync(path.join(tmp, 'config'))) { + fs.mkdirSync(this.configDir, { recursive: true }) + for (const name of fs.readdirSync(this.configDir)) { + if (name.endsWith('.json') && !fs.lstatSync(path.join(this.configDir, name)).isDirectory()) + fs.rmSync(path.join(this.configDir, name), { force: true }) + } + for (const name of fs.readdirSync(path.join(tmp, 'config'))) { + fs.copyFileSync(path.join(tmp, 'config', name), path.join(this.configDir, name)) + fs.chmodSync(path.join(this.configDir, name), 0o600) + } + } + invalidateReadCache() + } - // 6. the ledger itself rolled back with the DB — re-register the - // pre_restore snapshot (and any other on-disk archives) so the - // user can still see/undo from the panel. - try { - this._registerOrphanArchives([pre, rec]) - } catch {} - - // 7. re-bind stores to the fresh connection. A failed rebind leaves the - // process partially attached to the closed database, so it must make - // the restore fail instead of reporting a false success. - if (this._rebindStores) this._rebindStores(db) + async _applyArchive(archive) { + await runPgTool( + 'pg_restore', + [ + '--clean', + '--if-exists', + '--single-transaction', + '--no-owner', + '--no-privileges', + `--schema=${getDb().schema}`, + '--dbname', + decodeURIComponent(new URL(getDb().connectionString).pathname.slice(1)), + archive.dump, + ], + getDb().connectionString, + ) + this._restoreFiles(archive.tmp) + await applyMigrations(getDb()) + } - return { ok: true, restored: rec.id, pre_restore: pre.id } + async restoreBackup(id) { + if (this._restoreRunning || this._running) + throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) + const pending = this._restoreBackup(id) + this._pendingRestore = pending + try { + return await pending } finally { + if (this._pendingRestore === pending) this._pendingRestore = null + } + } + + /** Failure keeps maintenance enabled. The restore endpoint may explicitly retry. */ + async _restoreBackup(id) { + if (this._restoreRunning || this._running) + throw Object.assign(new Error('restore already in progress'), { code: 'restore_in_progress' }) + this._restoreRunning = true + let archive + let previous + let pre + let changed = false + let maintenanceStarted = false + try { + const rec = await this._repo().get(id) + if (!rec) throw Object.assign(new Error('backup not found'), { code: 'backup_not_found' }) + if (rec.status !== 'ok') throw Object.assign(new Error('backup is not restorable'), { code: 'backup_not_ok' }) + if (!rec.file_path || !fs.existsSync(rec.file_path)) + throw Object.assign(new Error('backup file missing'), { code: 'backup_file_missing' }) + if (rec.sha256 && (await sha256FileAsync(rec.file_path)) !== rec.sha256) + throw Object.assign(new Error('backup archive sha256 mismatch'), { code: 'backup_corrupt' }) + archive = await this._extractArchive(rec) + fs.mkdirSync(this.dataDir, { recursive: true, mode: 0o700 }) + fs.writeFileSync(this.maintenanceFile, JSON.stringify({ backup_id: id, started_at: new Date().toISOString() }), { + mode: 0o600, + }) + maintenanceStarted = true + this.isRestoring = true + await this._enterMaintenance('restore') + pre = await this.createBackup({ kind: 'pre_restore', note: `before restore of ${id}` }) + previous = await this._extractArchive(pre) + changed = true + await getDb().withMaintenance(async () => { + await this._applyArchive(archive) + await this._registerOrphanArchives([pre, rec]) + await this._rebindStores?.(getDb()) + }) + await this._leaveMaintenance('restore') + fs.rmSync(this.maintenanceFile) this.isRestoring = false - if (tmp) { + return { ok: true, restored: id, pre_restore: pre.id } + } catch (error) { + if (maintenanceStarted) { + this.isRestoring = true + this.isMaintaining = true + } + if (changed && previous) { try { - fs.rmSync(tmp, { recursive: true, force: true }) - } catch {} + await getDb().withMaintenance(async () => { + await this._applyArchive(previous) + await this._registerOrphanArchives([pre]) + await this._rebindStores?.(getDb()) + }) + } catch { + throw Object.assign(new Error('restore and rollback failed; maintenance remains enabled'), { + code: 'restore_rollback_failed', + }) + } } + throw error + } finally { + this._restoreRunning = false + for (const item of [archive, previous]) if (item) fs.rmSync(item.tmp, { recursive: true, force: true }) } } @@ -397,15 +495,13 @@ export class BackupService { * record in the (freshly restored) DB. `known` records are inserted * verbatim; remaining orphans get kind='imported'. */ - _registerOrphanArchives(known = []) { + async _registerOrphanArchives(known = []) { const repo = this._repo() - const existing = new Set(repo.list({ limit: 500 }).map((r) => r.file_name)) + const existing = new Set((await repo.list({ limit: 500 })).map((r) => r.file_name)) for (const rec of known) { if (rec?.file_name && !existing.has(rec.file_name) && rec.file_path && fs.existsSync(rec.file_path)) { - try { - repo.insert(rec) - existing.add(rec.file_name) - } catch {} + await repo.insert(rec) + existing.add(rec.file_name) } } if (!fs.existsSync(this.backupDir)) return @@ -414,7 +510,7 @@ export class BackupService { const fp = path.join(this.backupDir, f) try { const st = fs.statSync(fp) - repo.insert({ + await repo.insert({ id: 'bak_' + crypto.randomBytes(6).toString('hex'), created_at: st.mtime.toISOString(), kind: 'imported', @@ -422,7 +518,7 @@ export class BackupService { file_path: fp, file_name: f, size_bytes: st.size, - sha256: sha256File(fp), + sha256: await sha256FileAsync(fp), db_bytes: null, includes: null, error: null, @@ -440,7 +536,7 @@ export class BackupService { this.stopScheduler() const check = async () => { try { - if (this.shouldRunScheduled()) { + if (await this.shouldRunScheduled()) { const rec = await this.createBackup({ kind: 'scheduled' }) console.log(`[backup] scheduled backup completed (${Math.round((rec.size_bytes || 0) / 1048576)} MiB)`) } @@ -455,10 +551,10 @@ export class BackupService { if (typeof this._bootTimer.unref === 'function') this._bootTimer.unref() } - shouldRunScheduled(now = Date.now()) { - const cfg = this.getSchedule() - if (!cfg.enabled || this._running || this.isRestoring) return false - const last = this._repo().lastSuccessful({}) + async shouldRunScheduled(now = Date.now()) { + const cfg = await this.getSchedule() + if (!cfg.enabled || this._running || this._restoreRunning || this.isMaintaining) return false + const last = await this._repo().lastSuccessful({}) if (!last) return true return now - Date.parse(last.created_at) >= cfg.interval_hours * 3600_000 } diff --git a/src/lib/admin/backup-worker.mjs b/src/lib/admin/backup-worker.mjs index 7b5f5a12f..12e1f8c56 100644 --- a/src/lib/admin/backup-worker.mjs +++ b/src/lib/admin/backup-worker.mjs @@ -1,32 +1,15 @@ -/** - * Worker entry for the backup pipeline. - * - * The pipeline (VACUUM INTO + sha256 + tar.gz) is fully synchronous, so it must - * not run on the gateway's main thread: that is what made auto-backups freeze - * the whole gateway (HTTP/panel/healthz all stop answering → Cloudflare 524) - * for the duration of the backup. - * - * This worker opens its own SQLite connection (WAL allows the live gateway to - * keep serving while the snapshot is taken) and posts the finished record back - * to BackupService. Failures are reported as a message, never thrown across the - * thread boundary. - */ - +/** PostgreSQL archive worker; the parent owns the exported snapshot transaction. */ import { parentPort, workerData } from 'node:worker_threads' -import { openDatabase, closeDatabase } from '../db/database.mjs' import { runBackupJob } from './backup-core.mjs' try { - openDatabase({ dataDir: workerData.dataDir }) - const rec = runBackupJob(workerData.ctx, workerData.job) + const rec = await runBackupJob(workerData.ctx, workerData.job, () => { + parentPort.postMessage({ type: 'files-copied' }) + }) parentPort.postMessage({ ok: true, rec }) -} catch (e) { +} catch (error) { parentPort.postMessage({ ok: false, - error: { message: String((e && e.message) || e), code: (e && e.code) || null }, + error: { message: String(error.message || error), code: error.code || 'backup_failed' }, }) -} finally { - try { - closeDatabase() - } catch {} } diff --git a/test/e2e/backup.e2e.test.mjs b/test/e2e/backup.e2e.test.mjs index d9eac8f4f..eca175bdd 100644 --- a/test/e2e/backup.e2e.test.mjs +++ b/test/e2e/backup.e2e.test.mjs @@ -9,7 +9,6 @@ import os from 'node:os' import path from 'node:path' import zlib from 'node:zlib' import { spawnSync } from 'node:child_process' -import { DatabaseSync } from 'node:sqlite' import { startGateway, api } from '../harness.mjs' test('manual backup → download → mutate → restore rolls back', async () => { @@ -51,10 +50,10 @@ test('manual backup → download → mutate → restore rolls back', async () => fs.writeFileSync(path.join(tmp, 'a.tar.gz'), buf) const tar = spawnSync('tar', ['-xzf', path.join(tmp, 'a.tar.gz'), '-C', tmp]) assert.equal(tar.status, 0) - assert.ok(fs.existsSync(path.join(tmp, 'db', 'kin.db'))) + assert.ok(fs.existsSync(path.join(tmp, 'db', 'postgres.dump'))) assert.ok(fs.existsSync(path.join(tmp, 'manifest.json'))) const manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) - assert.equal(manifest.version, 1) + assert.equal(manifest.version, 2) fs.rmSync(tmp, { recursive: true, force: true }) // 4. mutate state after the backup @@ -87,9 +86,6 @@ test('manual backup → download → mutate → restore rolls back', async () => }) assert.equal(m.status, 200, m.text) - // old db kept on disk as a safety net - assert.ok(fs.existsSync(path.join(gw.project, 'data', 'kin.db.pre-restore'))) - // ledger after restore still shows the pre_restore snapshot (undo path) const after = await api(gw, 'GET', '/api/panel/backups') assert.ok( @@ -129,18 +125,13 @@ test('restore rebinds billing, API scheduler, and routing config without restart assert.equal(keyResponse.status, 201, keyResponse.text) const apiKey = keyResponse.json.item.key - const dbPath = path.join(gw.project, 'data', 'kin.db') - const beforeDb = new DatabaseSync(dbPath) - beforeDb.prepare('UPDATE groups SET rate_multiplier = 0 WHERE id = 1').run() - beforeDb.close() + await gw.db.exec('UPDATE groups SET rate_multiplier = 0 WHERE id = 1') const backup = await api(gw, 'POST', '/api/panel/backups') assert.equal(backup.status, 201, backup.text) const backupId = backup.json.item.id - const afterDb = new DatabaseSync(dbPath) - afterDb.prepare('UPDATE groups SET rate_multiplier = 2 WHERE id = 1').run() - afterDb.close() + await gw.db.exec('UPDATE groups SET rate_multiplier = 2 WHERE id = 1') assert.equal((await api(gw, 'DELETE', `/api/panel/api-endpoints/${endpointBeforeId}`)).status, 200) const endpointAfter = await api(gw, 'POST', '/api/panel/api-endpoints', { body: { diff --git a/test/unit/backup-core.test.mjs b/test/unit/backup-core.test.mjs new file mode 100644 index 000000000..30e9ba626 --- /dev/null +++ b/test/unit/backup-core.test.mjs @@ -0,0 +1,27 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { pgEnvironment, runPgTool } from '../../src/lib/admin/backup-core.mjs' + +test('PostgreSQL child environment preserves encoded credentials and IPv6 without command arguments', () => { + const env = pgEnvironment('postgresql://test:p%40ss%3Aword@[::1]:55432/example_test?sslmode=verify-full') + assert.equal(env.PGPASSWORD, 'p@ss:word') + assert.equal(env.PGHOST, '::1') + assert.equal(env.PGPORT, '55432') + assert.equal(env.PGDATABASE, 'example_test') + assert.equal(env.PGSSLMODE, 'verify-full') +}) + +test('child process errors do not return stderr that can contain row values', async () => { + await assert.rejects( + runPgTool( + process.execPath, + ['-e', "process.stderr.write('synthetic-sensitive-row');process.exit(2)"], + 'postgresql://test:synthetic-password@127.0.0.1/example_test', + ), + (error) => { + assert.equal(error.message.includes('synthetic'), false) + assert.match(error.message, /failed \(exit 2\)/) + return true + }, + ) +}) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index bba99bf36..301ce0986 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -4,7 +4,9 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { spawnSync } from 'node:child_process' -import { openDatabase, closeDatabase, getDb } from '../../src/lib/db/database.mjs' +import { openDatabase, closeDatabase, getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { BackupRepo } from '../../src/lib/db/repos/backup-repo.mjs' +import { sha256FileAsync } from '../../src/lib/admin/backup-core.mjs' import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' import { BackupService, tarAvailable } from '../../src/lib/admin/backup-service.mjs' @@ -33,21 +35,21 @@ function seedVm(id = 'vm-b1') { fs.writeFileSync(path.join(vmsDir, 'active.json'), JSON.stringify({ active_vm: id })) } -beforeEach(() => { +beforeEach(async () => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-bak-')) dataDir = path.join(root, 'data') configDir = path.join(root, 'config') fs.mkdirSync(configDir, { recursive: true }) fs.writeFileSync(path.join(configDir, 'routing.json'), JSON.stringify({ sticky: { enabled: true } })) - openDatabase({ dataDir }) + await openDatabase({ dataDir }) svc = new BackupService({ dataDir, projectRoot: root, configDir }) }) -afterEach(() => { +afterEach(async () => { svc.stopScheduler() stopVmWatch() setVmWriteHook(null) - closeDatabase() + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) delete process.env.KIN_BACKUP_DISABLED delete process.env.KIN_BACKUP_INTERVAL_HOURS @@ -60,8 +62,8 @@ test('tar is available in this environment', () => { test('createBackup produces tar.gz with manifest + db + vms + config, records ok', async () => { seedVm() - initVmDbSync(root, { watch: false }) - new SettingsRepo(getDb()).set('marker', 'before-backup') + await initVmDbSync(root, { watch: false }) + await new SettingsRepo(getDb()).set('marker', 'before-backup') const rec = await svc.createBackup({ kind: 'manual' }) assert.equal(rec.status, 'ok') @@ -77,18 +79,18 @@ test('createBackup produces tar.gz with manifest + db + vms + config, records ok const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-bak-x-')) spawnSync('tar', ['-xzf', rec.file_path, '-C', tmp]) assert.ok(fs.existsSync(path.join(tmp, 'manifest.json'))) - assert.ok(fs.existsSync(path.join(tmp, 'db', 'kin.db'))) + assert.ok(fs.existsSync(path.join(tmp, 'db', 'postgres.dump'))) assert.ok(fs.existsSync(path.join(tmp, 'vms', 'vm-b1.json'))) assert.ok(fs.existsSync(path.join(tmp, 'config', 'routing.json'))) const manifest = JSON.parse(fs.readFileSync(path.join(tmp, 'manifest.json'), 'utf8')) - assert.equal(manifest.version, 1) - assert.ok(manifest.sha256s['db/kin.db']) + assert.equal(manifest.version, 2) + assert.ok(manifest.sha256s['db/postgres.dump']) fs.rmSync(tmp, { recursive: true, force: true }) }) test('createBackup does not block the main thread (gateway keeps ticking)', async () => { seedVm() - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) let ticks = 0 const timer = setInterval(() => { ticks += 1 @@ -106,38 +108,37 @@ test('createBackup does not block the main thread (gateway keeps ticking)', asyn test('restoreBackup rolls data back and creates pre_restore record', async () => { seedVm() - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) const keys = new ApiKeyStore({ db: getDb() }) - const kept = keys.create({ name: 'kept-key' }) + const kept = await keys.create({ name: 'kept-key' }) const rec = await svc.createBackup({ kind: 'manual' }) // mutate after the backup - const lost = keys.create({ name: 'lost-key' }) - new SettingsRepo(getDb()).set('marker', 'after-backup') + const lost = await keys.create({ name: 'lost-key' }) + await new SettingsRepo(getDb()).set('marker', 'after-backup') // wire the rebind like the server does let reboundDb = null - svc.onRestored((db) => { + svc.onRestored(async (db) => { reboundDb = db - keys.rebind(db) + await keys.rebind(db) }) const out = await svc.restoreBackup(rec.id) assert.equal(out.ok, true) + assert.equal(fs.existsSync(svc.maintenanceFile), false) assert.ok(reboundDb, 'rebind callback should fire') // state rolled back - assert.ok(keys.getById(kept.id), 'pre-backup key survives') - assert.equal(keys.getById(lost.id), null, 'post-backup key rolled back') - assert.equal(new SettingsRepo(getDb()).get('marker'), null) + assert.ok(await keys.getById(kept.id), 'pre-backup key survives') + assert.equal(await keys.getById(lost.id), null, 'post-backup key rolled back') + assert.equal(await new SettingsRepo(getDb()).get('marker'), null) - // old db file kept on disk as safety net - assert.ok(fs.existsSync(path.join(dataDir, 'kin.db.pre-restore'))) // pre_restore archive file kept on disk for manual recovery const preFiles = fs.readdirSync(path.join(dataDir, 'backups')) assert.ok(preFiles.length >= 2, 'manual + pre_restore archives on disk') // ledger re-registered after restore: pre_restore (and restored manual) visible - const ledger = svc.list() + const ledger = await svc.list() assert.ok( ledger.some((r) => r.id === out.pre_restore && r.kind === 'pre_restore'), 'pre_restore record re-registered in restored DB', @@ -152,21 +153,24 @@ test('restoreBackup rolls data back and creates pre_restore record', async () => ) }) -test('restore fails when stores cannot rebind to the reopened database', async () => { +test('restore failure keeps maintenance enabled and exposes the recovery archive', async () => { seedVm() const rec = await svc.createBackup({ kind: 'manual' }) svc.onRestored(() => { throw new Error('store rebind failed') }) - await assert.rejects(svc.restoreBackup(rec.id), /store rebind failed/) - assert.equal(svc.isRestoring, false) - assert.doesNotThrow(() => getDb().prepare('SELECT 1').get()) + await assert.rejects(svc.restoreBackup(rec.id), /restore and rollback failed/) + assert.equal(svc.isRestoring, true) + assert.equal(svc.isMaintaining, true) + assert.equal(fs.existsSync(svc.maintenanceFile), true) + await assert.rejects(createDatabase({ dataDir, connectionString: 'invalid' }), /restore maintenance marker/) + assert.equal((await getDb().prepare('SELECT 1 AS ok').get()).ok, 1) }) -test('restore rebuilds vm files from DB mirror for db-only path', async () => { +test('restore restores VM JSON files together with PostgreSQL metadata', async () => { seedVm('vm-b2') - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) const rec = await svc.createBackup({ kind: 'manual' }) // delete vm file + its row is still in backup db; also delete local file post-backup @@ -176,7 +180,7 @@ test('restore rebuilds vm files from DB mirror for db-only path', async () => { const back = JSON.parse(fs.readFileSync(path.join(root, 'vms', 'vm-b2.json'), 'utf8')) // tar snapshot still has the on-disk file from backup time; DB is metadata only. assert.equal(back.claude.access_token, 'sk-ant-oat01-BAK') - const row = new VmsRepo(getDb()).get('vm-b2') + const row = await new VmsRepo(getDb()).get('vm-b2') assert.equal(row.access_token, null) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.access_token, undefined) @@ -194,7 +198,7 @@ test('retention prunes old backups', async () => { seedVm() const recs = [] for (let i = 0; i < 4; i++) recs.push(await svc.createBackup({ kind: 'manual' })) - const remaining = svc.list().filter((r) => r.kind === 'manual') + const remaining = (await svc.list()).filter((r) => r.kind === 'manual') assert.equal(remaining.length, 2) // newest two kept assert.ok(remaining.some((r) => r.id === recs[3].id)) @@ -204,34 +208,144 @@ test('retention prunes old backups', async () => { assert.equal(fs.existsSync(recs[1].file_path), false) }) -test('schedule config defaults on, env override off, updateSchedule persists', () => { - const cfg = svc.getSchedule() +test('schedule config defaults on, env override off, updateSchedule persists', async () => { + const cfg = await svc.getSchedule() assert.equal(cfg.enabled, true) assert.equal(cfg.interval_hours, 24) assert.equal(cfg.retention, 7) - svc.updateSchedule({ interval_hours: 6, retention: 3 }) - assert.equal(svc.getSchedule().interval_hours, 6) - assert.equal(svc.getSchedule().retention, 3) - assert.throws(() => svc.updateSchedule({ interval_hours: 0 }), /interval_hours/) + await svc.updateSchedule({ interval_hours: 6, retention: 3 }) + assert.equal((await svc.getSchedule()).interval_hours, 6) + assert.equal((await svc.getSchedule()).retention, 3) + await assert.rejects(() => svc.updateSchedule({ interval_hours: 0 }), /interval_hours/) process.env.KIN_BACKUP_DISABLED = '1' - assert.equal(svc.getSchedule().enabled, false) + assert.equal((await svc.getSchedule()).enabled, false) }) test('shouldRunScheduled: true when never run, false right after a backup', async () => { seedVm() - assert.equal(svc.shouldRunScheduled(), true) + assert.equal(await svc.shouldRunScheduled(), true) await svc.createBackup({ kind: 'scheduled' }) - assert.equal(svc.shouldRunScheduled(), false) + assert.equal(await svc.shouldRunScheduled(), false) // pretend 25h elapsed - assert.equal(svc.shouldRunScheduled(Date.now() + 25 * 3600_000), true) + assert.equal(await svc.shouldRunScheduled(Date.now() + 25 * 3600_000), true) }) test('remove deletes record + file', async () => { seedVm() const rec = await svc.createBackup({ kind: 'manual' }) - assert.equal(svc.remove(rec.id), true) + assert.equal(await svc.remove(rec.id), true) assert.equal(fs.existsSync(rec.file_path), false) - assert.equal(svc.get(rec.id), null) + assert.equal(await svc.get(rec.id), null) +}) + +test('backup snapshot and copied files precede resumed writes', async () => { + seedVm() + await new SettingsRepo(getDb()).set('snapshot-boundary', 'before') + const transitions = [] + svc.onMaintenance(async (enabled, reason) => { + transitions.push([enabled, reason]) + if (!enabled) { + await new SettingsRepo(getDb()).set('snapshot-boundary', 'after') + fs.writeFileSync(path.join(configDir, 'routing.json'), '{"after":true}') + } + }) + const rec = await svc.createBackup() + svc.onMaintenance(null) + await svc.restoreBackup(rec.id) + assert.equal(await new SettingsRepo(getDb()).get('snapshot-boundary'), 'before') + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(configDir, 'routing.json'))), { sticky: { enabled: true } }) + assert.deepEqual(transitions, [ + [true, 'backup'], + [false, 'backup'], + ]) +}) + +test('restore rejects an archive directory symlink before entering maintenance', async () => { + const source = path.join(root, 'symlink-source') + const outside = path.join(root, 'outside-extraction') + fs.mkdirSync(source) + fs.mkdirSync(outside) + fs.writeFileSync(path.join(outside, 'sentinel.json'), '{"unchanged":true}') + fs.writeFileSync(path.join(source, 'manifest.json'), '{}') + fs.symlinkSync(outside, path.join(source, 'vms')) + const archive = path.join(root, 'symlink.tar.gz') + assert.equal(spawnSync('tar', ['-czf', archive, '-C', source, 'manifest.json', 'vms']).status, 0) + // Reproduce the old extraction behavior using only this test's temporary paths. + const oldExtraction = path.join(root, 'old-extraction') + fs.mkdirSync(oldExtraction) + assert.equal(spawnSync('tar', ['-xzf', archive, '-C', oldExtraction]).status, 0) + assert.equal(fs.lstatSync(path.join(oldExtraction, 'vms')).isSymbolicLink(), true) + const rec = await new BackupRepo(getDb()).insert({ + id: 'bak_symlink', + created_at: new Date().toISOString(), + status: 'ok', + kind: 'manual', + file_name: 'symlink.tar.gz', + file_path: archive, + sha256: await sha256FileAsync(archive), + }) + await assert.rejects(svc.restoreBackup(rec.id), /invalid or duplicate backup archive path/) + assert.equal(fs.readFileSync(path.join(outside, 'sentinel.json'), 'utf8'), '{"unchanged":true}') + assert.equal(svc.isMaintaining, false) +}) + +test('restore rejects file counts that disagree with the manifest', async () => { + seedVm() + const rec = await svc.createBackup() + const source = path.join(root, 'count-mismatch') + fs.mkdirSync(source) + assert.equal(spawnSync('tar', ['-xzf', rec.file_path, '-C', source]).status, 0) + const file = path.join(source, 'manifest.json') + const manifest = JSON.parse(fs.readFileSync(file)) + manifest.includes.vms += 1 + fs.writeFileSync(file, JSON.stringify(manifest)) + assert.equal(spawnSync('tar', ['-czf', rec.file_path, '-C', source, '.']).status, 0) + await getDb() + .prepare('UPDATE backup_records SET sha256 = $1 WHERE id = $2') + .run(await sha256FileAsync(rec.file_path), rec.id) + await assert.rejects(svc.restoreBackup(rec.id), /backup file snapshot is incomplete/) + assert.equal(svc.isMaintaining, false) +}) + +test('restore replaces all top-level JSON while preserving CLI homes and non-JSON files', async () => { + seedVm() + const vmsDir = path.join(root, 'vms') + const cliHome = path.join(vmsDir, 'vm-b1', 'cli-home') + fs.mkdirSync(cliHome, { recursive: true }) + fs.writeFileSync(path.join(cliHome, 'keep.txt'), 'keep') + fs.writeFileSync(path.join(configDir, 'keep.txt'), 'keep') + const rec = await svc.createBackup() + fs.writeFileSync(path.join(vmsDir, 'vm-b1-chat.json'), '{}') + fs.writeFileSync(path.join(configDir, 'after-backup.json'), '{}') + await svc.restoreBackup(rec.id) + assert.equal(fs.existsSync(path.join(vmsDir, 'vm-b1-chat.json')), false) + assert.equal(fs.existsSync(path.join(configDir, 'after-backup.json')), false) + assert.equal(fs.readFileSync(path.join(cliHome, 'keep.txt'), 'utf8'), 'keep') + assert.equal(fs.readFileSync(path.join(configDir, 'keep.txt'), 'utf8'), 'keep') +}) + +test('failed restore rolls back target-only JSON and persists maintenance until an explicit successful retry', async () => { + seedVm() + const targetOnly = path.join(configDir, 'target-only.json') + const currentOnly = path.join(configDir, 'current-only.json') + fs.writeFileSync(targetOnly, '{}') + const rec = await svc.createBackup() + fs.rmSync(targetOnly) + fs.writeFileSync(currentOnly, '{}') + let calls = 0 + svc.onRestored(async () => { + if (++calls === 1) throw new Error('first rebind failed') + }) + await assert.rejects(svc.restoreBackup(rec.id), /first rebind failed/) + assert.equal(fs.existsSync(targetOnly), false) + assert.equal(fs.existsSync(currentOnly), true) + assert.equal(fs.existsSync(svc.maintenanceFile), true) + assert.equal(svc.isMaintaining, true) + await svc.restoreBackup(rec.id) + assert.equal(fs.existsSync(svc.maintenanceFile), false) + assert.equal(svc.isMaintaining, false) + assert.equal(fs.existsSync(targetOnly), true) + assert.equal(fs.existsSync(currentOnly), false) }) From c2e56a2d508534e51f96402f9964f980eeaaa5f5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:53:50 +0800 Subject: [PATCH 016/473] feat(deploy): configure PostgreSQL operations and database overview --- .env.example | 8 + Dockerfile | 11 ++ README.md | 3 + deploy/Dockerfile.arm64-control | 11 ++ deploy/init-arm64-env.py | 7 +- deploy/install.sh | 40 +++- deploy/postgres-entrypoint.sh | 10 + deploy/postgres-init.sh | 8 + docker-compose.yml | 29 +++ docs/OAUTH.md | 2 +- docs/PANEL_API.md | 14 +- docs/deploy/postgresql.md | 57 ++++++ docs/deploy/vm2api.service | 1 + .../postgresql/database-postgresql-1440.png | Bin 0 -> 44768 bytes .../postgresql/database-sqlite-1440.png | Bin 0 -> 49600 bytes scripts/audit-billing-contract.mjs | 77 ++++---- scripts/audit-billing-gaps.mjs | 79 ++++---- scripts/audit-billing-openai.mjs | 41 +++-- scripts/audit-db.mjs | 41 +++++ scripts/audit-fable-week.mjs | 45 ++--- scripts/check-postgres-upgrade.mjs | 49 +++++ scripts/docker-entrypoint.sh | 4 + scripts/panel-upgrade.sh | 3 +- src/lib/db/database-metrics.mjs | 144 ++++----------- test/deploy/smoke_arm64_image.py | 38 +++- test/deploy/smoke_postgres_role.py | 50 +++++ test/deploy/test_postgres_env.py | 36 ++++ test/e2e/panel-api.e2e.test.mjs | 10 +- test/unit/audit-postgres.test.mjs | 89 +++++++++ test/unit/database-metrics.test.mjs | 171 +++++------------- test/unit/panel-upgrade-script.test.mjs | 8 + .../features/database/database-overview.tsx | 67 ++----- web/src/types/panel-database.ts | 19 +- 33 files changed, 732 insertions(+), 440 deletions(-) create mode 100644 deploy/postgres-entrypoint.sh create mode 100644 deploy/postgres-init.sh create mode 100644 docs/deploy/postgresql.md create mode 100644 docs/plans/evidence/postgresql/database-postgresql-1440.png create mode 100644 docs/plans/evidence/postgresql/database-sqlite-1440.png create mode 100644 scripts/audit-db.mjs create mode 100644 scripts/check-postgres-upgrade.mjs create mode 100644 test/deploy/smoke_postgres_role.py create mode 100644 test/deploy/test_postgres_env.py create mode 100644 test/unit/audit-postgres.test.mjs diff --git a/.env.example b/.env.example index bbe7a97a4..5d7f04270 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,11 @@ HOST=0.0.0.0 # PUBLIC_BASE_URL=https://example.com # PUBLIC_HOST=127.0.0.1 # GITHUB_TOKEN= # 可选。面板查 GitHub Release 提高限额；也可用 VM2API_GITHUB_TOKEN + +# PostgreSQL 17. The installer fills a random password for a new local database. +# External PG: set VM2API_DATABASE_URL and clear COMPOSE_PROFILES. +COMPOSE_PROFILES=postgres +VM2API_POSTGRES_PORT=55432 +VM2API_POSTGRES_PASSWORD= +VM2API_DATABASE_URL= +# Preserve VM2API_DB_SECRET when importing an existing SQLite database. diff --git a/Dockerfile b/Dockerfile index 984611584..77d004651 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,17 @@ FROM node:22-bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates iptables iproute2 python3 \ && rm -rf /var/lib/apt/lists/* +# Debian Bookworm defaults to PG15; backups require the PG17 client. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && install -d /usr/share/postgresql-common/pgdg \ + && curl --fail --silent --show-error -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client-17 \ + && pg_dump --version | grep -E ' 17\.' \ + && pg_restore --version | grep -E ' 17\.' \ + && rm -rf /var/lib/apt/lists/* COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker WORKDIR /opt/vm2api COPY package.json package-lock.json ./ diff --git a/README.md b/README.md index a37f0b33c..a6afb4cf9 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,9 @@ chmod 600 .env docker compose pull && docker compose up -d ``` +数据库使用 PostgreSQL 17。新安装由安装脚本生成内置数据库密码；手工 Compose 安装先配置 `.env` 中的 `VM2API_POSTGRES_PASSWORD`。已有 SQLite 安装须先离线导入，保留原 `VM2API_DB_SECRET`。外部数据库、备份恢复和验证步骤见 [PostgreSQL 部署说明](docs/deploy/postgresql.md)。 + + ### 3. 访问与调用 服务启动后，系统暴露统一服务端口（默认 `8787`）： diff --git a/deploy/Dockerfile.arm64-control b/deploy/Dockerfile.arm64-control index 6e92f83f0..4742b6a72 100644 --- a/deploy/Dockerfile.arm64-control +++ b/deploy/Dockerfile.arm64-control @@ -40,6 +40,17 @@ FROM --platform=linux/arm64 ${NODE_IMAGE} RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates iptables iproute2 python3 \ && rm -rf /var/lib/apt/lists/* +# Debian Bookworm defaults to PG15; backups require the PG17 client. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && install -d /usr/share/postgresql-common/pgdg \ + && curl --fail --silent --show-error -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client-17 \ + && pg_dump --version | grep -E ' 17\.' \ + && pg_restore --version | grep -E ' 17\.' \ + && rm -rf /var/lib/apt/lists/* COPY --from=dockercli /usr/local/bin/docker /usr/local/bin/docker WORKDIR /opt/vm2api COPY package.json package-lock.json ./ diff --git a/deploy/init-arm64-env.py b/deploy/init-arm64-env.py index bcdeb8ebb..66993bd1f 100644 --- a/deploy/init-arm64-env.py +++ b/deploy/init-arm64-env.py @@ -57,11 +57,16 @@ def main() -> int: root = Path(__file__).resolve().parents[1] env_path = root / ".env" + postgres_password = secrets.token_hex(32) values = { "VM2API_ADMIN_USER": "admin", "VM2API_ADMIN_PASSWORD": secrets.token_urlsafe(24), "VM2API_API_KEY": secrets.token_hex(32), "VM2API_DB_SECRET": secrets.token_hex(32), + "COMPOSE_PROFILES": "postgres", + "VM2API_POSTGRES_PORT": "55432", + "VM2API_POSTGRES_PASSWORD": postgres_password, + "VM2API_DATABASE_URL": f"postgresql://vm2api:{postgres_password}@127.0.0.1:55432/vm2api", "VM2API_CONTAINER_NAME": args.container_name, "VM2API_BIND_HOST": args.bind_host, "PORT": str(args.port), @@ -72,7 +77,7 @@ def main() -> int: fd = os.open(env_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) except FileExistsError: print("Existing .env preserved. Review it locally before starting this deployment.") - missing = [key for key in ("COMPOSE_FILE", "VM2API_HOST_ROOT") if not env_has(env_path, key)] + missing = [key for key in ("COMPOSE_FILE", "VM2API_HOST_ROOT", "VM2API_DATABASE_URL") if not env_has(env_path, key)] if missing: print(f"Add to .env: COMPOSE_FILE={COMPOSE_FILE} and VM2API_HOST_ROOT={root} " f"(missing: {', '.join(missing)}).") diff --git a/deploy/install.sh b/deploy/install.sh index 571661b45..ae22590a6 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -279,11 +279,19 @@ env_set() { local name="$1" local value="$2" local envf="${INSTALL_DIR}/.env" - if grep -qE "^${name}=" "$envf" 2>/dev/null; then - sed -i "s|^${name}=.*|${name}=${value}|" "$envf" - else - printf '%s=%s\n' "$name" "$value" >>"$envf" - fi + VM2API_ENV_KEY="$name" VM2API_ENV_VALUE="$value" VM2API_ENV_FILE="$envf" python3 - <<'PYENV' +import os +from pathlib import Path +p = Path(os.environ["VM2API_ENV_FILE"]) +key = os.environ["VM2API_ENV_KEY"] + "=" +value = key + os.environ["VM2API_ENV_VALUE"] + "\n" +lines = p.read_text().splitlines(keepends=True) if p.exists() else [] +if any(line.startswith(key) for line in lines): + p.write_text("".join(value if line.startswith(key) else line for line in lines)) +else: + text = "".join(lines) + p.write_text(text + ("\n" if text and not text.endswith("\n") else "") + value) +PYENV } env_set_if_empty() { @@ -380,6 +388,16 @@ ensure_env() { cp "$example" "$envf" fi chmod 600 "$envf" || true + if [ -z "$(env_get VM2API_DATABASE_URL)" ] && [ -z "$(env_get KIN_DATABASE_URL)" ]; then + if [ -f "${INSTALL_DIR}/data/kin.db" ]; then + err "SQLite 数据尚未迁移。保留运行容器，按 docs/deploy/postgresql.md 完成离线导入后再升级。" + return 1 + fi + env_set_if_empty COMPOSE_PROFILES postgres || true + env_set_if_empty VM2API_POSTGRES_PORT 55432 || true + env_set_if_empty VM2API_POSTGRES_PASSWORD "$(gen_secret)" || true + env_set_if_empty VM2API_DATABASE_URL "postgresql://vm2api:$(env_get VM2API_POSTGRES_PASSWORD)@127.0.0.1:$(env_get VM2API_POSTGRES_PORT)/vm2api" || true + fi env_set_if_empty VM2API_ADMIN_USER "$DEFAULT_ADMIN_USER" || true if env_set_if_empty VM2API_ADMIN_PASSWORD "$DEFAULT_ADMIN_PASSWORD"; then WROTE_DEFAULT_PASSWORD=1 @@ -519,9 +537,14 @@ fetch_file() { # 镜像安装：只落部署文件，不 clone 仓库，不在本机构建。 fetch_release_files() { local tag="$1" - mkdir -p "${INSTALL_DIR}/deploy" + mkdir -p "${INSTALL_DIR}/deploy" "${INSTALL_DIR}/docs/deploy" info "下载部署文件 ${tag} → ${INSTALL_DIR}" fetch_file "$tag" docker-compose.yml "${INSTALL_DIR}/docker-compose.yml" + if grep -q 'vm2api-postgres-entrypoint' "${INSTALL_DIR}/docker-compose.yml"; then + fetch_file "$tag" deploy/postgres-entrypoint.sh "${INSTALL_DIR}/deploy/postgres-entrypoint.sh" + fetch_file "$tag" deploy/postgres-init.sh "${INSTALL_DIR}/deploy/postgres-init.sh" + fetch_file "$tag" docs/deploy/postgresql.md "${INSTALL_DIR}/docs/deploy/postgresql.md" + fi fetch_file "$tag" .env.example "${INSTALL_DIR}/.env.example" fetch_file "$tag" VERSION "${INSTALL_DIR}/VERSION" fetch_file "$tag" CHANGELOG.md "${INSTALL_DIR}/CHANGELOG.md" 1 @@ -631,6 +654,11 @@ start_stack() { fi cd "${INSTALL_DIR}" auto_sync_wrap="${KIN_AUTO_SYNC_WRAP:-0}" + # An existing SQLite installation must pass the read-only import check before replacement. + if [ -f data/kin.db ] || [ -f .local/arm64/data/kin.db ]; then + if [ "$mode" = image ]; then compose pull vm2api; else compose build vm2api; fi + compose run --rm --no-deps -T --entrypoint node vm2api scripts/check-postgres-upgrade.mjs || return 1 + fi if [ "$mode" = image ]; then info "docker compose pull && up -d（拉预构建镜像，本机不构建）" if ! compose pull; then diff --git a/deploy/postgres-entrypoint.sh b/deploy/postgres-entrypoint.sh new file mode 100644 index 000000000..a177fd862 --- /dev/null +++ b/deploy/postgres-entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +# Bootstrap postgres never shares the application's password. No password is logged or passed in argv. +if [ -z "${VM2API_POSTGRES_PASSWORD:-}" ]; then + echo 'VM2API_POSTGRES_PASSWORD is required for the application role.' >&2 + exit 1 +fi +POSTGRES_PASSWORD="$(od -An -N32 -tx1 /dev/urandom | tr -d ' \n')" +export POSTGRES_PASSWORD +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/deploy/postgres-init.sh b/deploy/postgres-init.sh new file mode 100644 index 000000000..2ec02d8d9 --- /dev/null +++ b/deploy/postgres-init.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +# Official entrypoint sources this only when initializing an empty data directory. +psql --no-password --set=ON_ERROR_STOP=1 --username=postgres --dbname=postgres <<'SQL' +\getenv app_password VM2API_POSTGRES_PASSWORD +CREATE ROLE vm2api LOGIN PASSWORD :'app_password' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; +CREATE DATABASE vm2api OWNER vm2api; +SQL diff --git a/docker-compose.yml b/docker-compose.yml index efaba6264..042339942 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,10 @@ services: container_name: ${VM2API_CONTAINER_NAME:-vm2api} restart: unless-stopped init: true + depends_on: + postgres: + condition: service_healthy + required: false network_mode: host cap_add: - NET_ADMIN @@ -17,6 +21,7 @@ services: environment: KIN_PROJECT_ROOT: /opt/vm2api KIN_DATA_DIR: /opt/vm2api/data + VM2API_DATABASE_URL: ${VM2API_DATABASE_URL:-${KIN_DATABASE_URL:-postgresql://vm2api:${VM2API_POSTGRES_PASSWORD:-}@127.0.0.1:${VM2API_POSTGRES_PORT:-55432}/vm2api}} KIN_KERNEL_BIN: /opt/vm2api/bin/kin-kernel KIN_EGRESS_BIN: /opt/vm2api/bin/kin-egress KIN_WORKER_BIN: /opt/vm2api/bin/kin-worker @@ -36,3 +41,27 @@ services: - ./bin:/opt/vm2api/bin - ./share:/opt/vm2api/share - ./src/config:/opt/vm2api/src/config + + postgres: + image: postgres:17-bookworm + profiles: [postgres] + restart: unless-stopped + entrypoint: [/bin/bash, /usr/local/bin/vm2api-postgres-entrypoint] + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 + VM2API_POSTGRES_PASSWORD: ${VM2API_POSTGRES_PASSWORD:-} + ports: + - "127.0.0.1:${VM2API_POSTGRES_PORT:-55432}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + - ./deploy/postgres-entrypoint.sh:/usr/local/bin/vm2api-postgres-entrypoint:ro + - ./deploy/postgres-init.sh:/docker-entrypoint-initdb.d/10-vm2api.sh:ro + healthcheck: + test: [CMD-SHELL, "pg_isready -U vm2api -d vm2api && psql -U vm2api -d vm2api -Atqc 'SELECT 1' >/dev/null"] + interval: 5s + timeout: 3s + retries: 12 +volumes: + postgres-data: diff --git a/docs/OAUTH.md b/docs/OAUTH.md index d8ddb0c92..402961f4b 100644 --- a/docs/OAUTH.md +++ b/docs/OAUTH.md @@ -1,6 +1,6 @@ # OAuth 凭证生命周期 -活票只写在槽位 `credentials.json`。换票由控制面 host `RefreshIfNeeded` 经槽 SOCKS5 完成；`vm.json`、SQLite 只存脱敏元数据。Go `kin-worker` 不再做 hop / refresh。 +活票只写在槽位 `credentials.json`。换票由控制面 host `RefreshIfNeeded` 经槽 SOCKS5 完成；`vm.json`、PostgreSQL 只存脱敏元数据。Go `kin-worker` 不再做 hop / refresh。 ```text sessionKey（默认 Setup Token）或授权码 diff --git a/docs/PANEL_API.md b/docs/PANEL_API.md index 78030f277..b5ce2faad 100644 --- a/docs/PANEL_API.md +++ b/docs/PANEL_API.md @@ -16,13 +16,13 @@ | `super` | 总览 / 集群 / 用量 / 日志 + 虚拟机 | 读 VM + 拨调度 / 清冷却；集群只读节点列表 | | `admin` | 全部 | `*`。可管理用户 VM，但 admin/master **未 pin** 的 `/v1` 只打未分配平台池 | -环境变量 `VM2API_ADMIN_USER` / `VM2API_ADMIN_PASSWORD` 只在库里还没有同名用户时灌进第一个 admin；之后以 SQLite `users` 为准，面板「用户」页可增删改、改密码。密码 scrypt。不能删/停用最后一个 admin。 +环境变量 `VM2API_ADMIN_USER` / `VM2API_ADMIN_PASSWORD` 只在库里还没有同名用户时灌进第一个 admin；之后以 PostgreSQL `users` 为准，面板「用户」页可增删改、改密码。密码 scrypt。不能删/停用最后一个 admin。 `vms/*.json` 的 `owner_user_id` / `origin`（`platform` \| `admin_assigned` \| `user_created`）是属主 SSOT。`PATCH /vms/:id/owner` 仅 admin。自建 VM 不能收回进平台池。 ## 引导 -所有已登录角色与 master API key 可调，不是公开接口。状态按用户名存在 SQLite `settings` 键 `onboarding_state`；master key（`apiKeyKind=master`）键为 `__master__`。 +所有已登录角色与 master API key 可调，不是公开接口。状态按用户名存在 PostgreSQL `settings` 键 `onboarding_state`；master key（`apiKeyKind=master`）键为 `__master__`。 | 方法 | 路径 | 说明 | |------|------|------| @@ -52,7 +52,7 @@ | PATCH | `/vms/:id/owner` | admin：`{ user_id }` 分配，`{ user_id: null }` 收回（仅 `admin_assigned`） | | GET | `/billing` | 用户计费汇总。`from`/`until`/`group_by=vm|key`。user 隐式只看自己；admin 可 `user_id=` | -启动时会修复旧版本遗留的已删除用户 VM 归属：仅收回 `admin_assigned`（旧记录未写 `origin` 时按已有归属规则推断），不收回租户自建 VM、停用用户或无用户记录的 VM。修改经 `vms/*.json` 写入并同步到 SQLite，不重启槽位。 +启动时会修复旧版本遗留的已删除用户 VM 归属：仅收回 `admin_assigned`（旧记录未写 `origin` 时按已有归属规则推断），不收回租户自建 VM、停用用户或无用户记录的 VM。修改经 `vms/*.json` 写入并同步到 PostgreSQL，不重启槽位。 用户名 `^[a-zA-Z][a-zA-Z0-9._-]{1,31}$`，密码 8–128。`vm_create_quota` 整数 0–100，默认 0。 @@ -155,9 +155,9 @@ Claude PATCH `max_sessions` 返回 400 `claude_max_sessions_forbidden`；OpenAI | 方法 | 路径 | 说明 | |------|------|------| -| GET | `/database/metrics` | `{ sampled_at, database, usage_cache }`；SQLite/WAL 只读快照与官方用量缓存实例统计 | +| GET | `/database/metrics` | `{ sampled_at, database, usage_cache }`；PostgreSQL 与应用连接池只读快照与官方用量缓存实例统计 | -`database` 只执行 `SELECT 1`、只读 PRAGMA、migration 摘要和文件 stat；不返回数据库路径。单项失败为 `null`，文件明确不存在时大小为 `0`，探针失败只令 `database.ok = false`。当前 `node:sqlite` 不提供 SQLite 页缓存 hit/miss，不得从这些字段推算。 +`database` 只执行 `SELECT 1`、数据库大小及 migration 摘要查询；连接池展示当前进程的 total/idle/waiting。失败字段为 `null`，探针失败令 `database.ok = false`。不返回连接地址、用户名或密码。 `usage_cache.hit_rate = (success_hits + error_hits) / requests`；`reuse_rate` 再加 `singleflight_joins`。零请求时均为 `null`。`error_hits` 是负缓存命中，不代表业务成功；累计计数随进程或缓存实例重建归零。 @@ -307,7 +307,7 @@ Claude `routing.quota` 的 5h/7d 闸分三段（`critical_band` 默认 0.05，`g | DELETE | `/refusal-guards` | 须 `{ "confirm": true }` 清空 | | DELETE | `/refusal-device-blocks` | `{ "device_id" }` 解封一条；`{ "confirm": true }` 清空。空 device、短于 8 字符的值不会入库 | -`PUT { enabled }` 写入 SQLite `settings.refusal_guard_enabled`。环境变量 `REFUSAL_GUARD=0` 仍强制关闭。与蒸馏拦截独立：0 注入跳过普通蒸馏针，本缓存仍生效。`count_tokens` 同样在 peek / worker hop 之前拦截。device 取入站 `metadata.user_id.device_id`，否则显式 `device_id` / `x-kin-device-id`，不用 IP、UA 或 API key。仅 admin。 +`PUT { enabled }` 写入 PostgreSQL `settings.refusal_guard_enabled`。环境变量 `REFUSAL_GUARD=0` 仍强制关闭。与蒸馏拦截独立：0 注入跳过普通蒸馏针，本缓存仍生效。`count_tokens` 同样在 peek / worker hop 之前拦截。device 取入站 `metadata.user_id.device_id`，否则显式 `device_id` / `x-kin-device-id`，不用 IP、UA 或 API key。仅 admin。 ## 决策模型与硬正则 @@ -315,7 +315,7 @@ Claude `routing.quota` 的 5h/7d 闸分三段（`critical_band` 默认 0.05，`g |------|------|------| | GET/PUT | `/jev-intercept` | 协议入口。顺序：蒸馏硬规则 → 去掉 reminder、按需展开 base64 后的硬正则 → 拒答缓存 → 决策模型。没配地址则跳过模型；模型失败默认放行。`fail_open: false` 时模型不可用则 403，不封设备 | -`PUT` 写入 SQLite `settings.jev_intercept`。字段：`enabled`（默认关）、`hard_regex_enabled`（默认开）、`openai_platform_enabled`（默认关）、`provider`（`jev` 默认、`laya`、`modernbert`）、`base_url`、`model`、`timeout_ms`（200–8000，默认 2000）、`patterns[]`、`rules`（`null` 与 `[]` 都用内置规则；关掉这一层用 `hard_regex_enabled: false`）、`api_key` 或 `api_keys`（最多 999 把）、`question_ids`（题库 id，缺省为全部）。省略密钥保持原值；`api_key: ""` 或 `api_keys: []` 清除。一行可以是裸 key（用 `base_url`），或 `https://host/v1/systemone ` 打到另一个 systemone。`401` / `429` / `529` 退役该 key，之后不再调度，再次保存密钥列表会重新启用。网络错误和其它 HTTP 状态只跳过这一跳的同一主机，不退役 key；失败放行的原因写在放行 keyword（`网络异常` 或 `上游 `）。另有 `safety_instruction`（空则不加自定义题）、`safety_threshold`（0–1，默认 0.5）、`block_if_below`（默认开）、`fail_open`（默认开）、`dedup_sec`（0–3600，默认 60）、`max_state_chars`（256–64000，默认 16000）、`expand_base64`、`strip_reminders`（后两个默认开）。GET 不回密钥，回 `api_key_set`、`api_key_count`、`question_bank`、`providers`、`builtin_patterns`、`rules`、`rules_customized`、`openai_platform_enabled`。 +`PUT` 写入 PostgreSQL `settings.jev_intercept`。字段：`enabled`（默认关）、`hard_regex_enabled`（默认开）、`openai_platform_enabled`（默认关）、`provider`（`jev` 默认、`laya`、`modernbert`）、`base_url`、`model`、`timeout_ms`（200–8000，默认 2000）、`patterns[]`、`rules`（`null` 与 `[]` 都用内置规则；关掉这一层用 `hard_regex_enabled: false`）、`api_key` 或 `api_keys`（最多 999 把）、`question_ids`（题库 id，缺省为全部）。省略密钥保持原值；`api_key: ""` 或 `api_keys: []` 清除。一行可以是裸 key（用 `base_url`），或 `https://host/v1/systemone ` 打到另一个 systemone。`401` / `429` / `529` 退役该 key，之后不再调度，再次保存密钥列表会重新启用。网络错误和其它 HTTP 状态只跳过这一跳的同一主机，不退役 key；失败放行的原因写在放行 keyword（`网络异常` 或 `上游 `）。另有 `safety_instruction`（空则不加自定义题）、`safety_threshold`（0–1，默认 0.5）、`block_if_below`（默认开）、`fail_open`（默认开）、`dedup_sec`（0–3600，默认 60）、`max_state_chars`（256–64000，默认 16000）、`expand_base64`、`strip_reminders`（后两个默认开）。GET 不回密钥，回 `api_key_set`、`api_key_count`、`question_bank`、`providers`、`builtin_patterns`、`rules`、`rules_customized`、`openai_platform_enabled`。 三个后端走同一套 `/v1/systemone`。题库存在 `questions`：可增删改，缺省是内置六题（综合、色情、破限、逆向、渗透、网络攻击）。每题都是「是否安全」，高分表示安全，启用的题目一起问，任一题低于阈值就拦。`safety_instruction` 非空时再多问一题自定义要求。`POST /jev-intercept/models` 用地址和 Key（留空则用已存的第一把）请求 `{base}/v1/models`，失败回 502，不会把上游 401 当成面板登出。 diff --git a/docs/deploy/postgresql.md b/docs/deploy/postgresql.md new file mode 100644 index 000000000..26248ffde --- /dev/null +++ b/docs/deploy/postgresql.md @@ -0,0 +1,57 @@ +# PostgreSQL 17 部署与 SQLite 导入 + +控制面只使用 PostgreSQL 17。`VM2API_DATABASE_URL` 优先于 `KIN_DATABASE_URL`。连接失败会阻止 HTTP 启动。`VM2API_DB_SECRET` / `KIN_DB_SECRET` 是已有业务字段的加密密钥，导入时必须保留。 + +## 新安装 + +运行 `deploy/install.sh` 会为内置 PostgreSQL 生成随机密码。`docker-compose.yml` 中的 `postgres` profile 默认由 `.env.example` 的 `COMPOSE_PROFILES=postgres` 启用。数据存放在 Compose 的 `postgres-data` 卷。初始化脚本创建 `vm2api` 业务角色，该角色拥有应用数据库，且没有超级用户、创建数据库、创建角色或复制权限。bootstrap `postgres` 密码仅在容器入口随机生成，不与业务密码共用，不写入用户配置或日志。新集群的 TCP 认证使用 SCRAM-SHA-256。端口默认仅映射到宿主 `127.0.0.1:55432`，可通过 `VM2API_POSTGRES_PORT` 修改。Compose 需要支持可选 `depends_on.required` 的版本（2.20 及以上）。 + +手工使用 Compose 时，先在权限为 `0600` 的 `.env` 中填写 `VM2API_POSTGRES_PASSWORD`。控制面的连接 URL 由 Compose 构造；不要把密码放进命令参数或复制到报告。启动命令仍为 `docker compose up -d`。保留 `vms`、配置、数据目录和数据库卷。 + +外部 PostgreSQL：在 `.env` 中填写 `VM2API_DATABASE_URL`，清空 `COMPOSE_PROFILES`。控制面使用 host 网络，数据库地址应为宿主可访问地址，不能使用 Compose 服务名 `postgres`。业务数据库应由本应用专用；应用登录角色需要管理应用 schema 的表、索引、函数和 sequence，无需超级用户或创建/删除数据库权限。应用默认 schema 为 `public`。 + +使用 systemd 直接运行 Node.js 时，在 `/etc/vm2api.env` 中配置连接 URL，并安装 PostgreSQL 17 的 `pg_dump` 与 `pg_restore`。`docs/deploy/vm2api.service` 是启动示例。两种控制面镜像均已包含本机架构的 PG 17 客户端。 + +## 已有 SQLite 安装 + +现有部署的数据迁移和停服应单独安排操作窗口。先在隔离环境演练，保留 SQLite、文件及原业务加密密钥。 + +1. 停止接收新请求，停止后台任务并排空在途请求、事务和 VM 文件镜像写入。保持源应用停止，直至切换完成。 +2. 使用 SQLite 一致性备份或现有备份归档取得源快照。不要只复制仍在写入的 `kin.db` 主文件而遗漏 WAL。保留整个 `vms` 目录（包括 CLI-home）、`src/config`、`data/backups`，以及既有运行槽位的数据和凭据存储。 +3. 准备空的专用 PostgreSQL 17 数据库。在受控进程环境中设置目标连接 URL和原业务加密密钥，执行 `npm run db:import:sqlite -- --source `。工具将检查结构、只读源文件、导入事务、内容摘要、外键和 sequence，完成后记录 `settings.sqlite_import_done`。 +4. 核对导入清单中的逐表结果与业务合计。原样带回对应的 VM、配置和备份目录。旧备份记录中的绝对文件路径需要与实际保留位置一致。 +5. 在新镜像中运行 `scripts/check-postgres-upgrade.mjs` 做只读检查，或通过安装/升级脚本执行该检查。源 `kin.db` 存在时必须查到导入完成标记；同时检查 PG 17 及已应用迁移的版本/校验和。已应用集合须为新镜像迁移序列的非空连续前缀，允许后续新迁移尚未应用。缺表、空记录、未知版本、缺口或校验不一致时不替换运行容器。 +6. 切换连接配置并启动新版本。通过登录、正常 mock 或授权业务请求、请求日志、账单、重启与备份恢复验证数据。记录运行镜像 digest 及验证的提交 SHA。 + +目标接受新写入前，可以停用目标并回到仍保留的源环境。目标接受写入后，原 SQLite 已过时。此时直接切回会丢失目标新增数据，应先设计反向数据处理方案。 + +旧 SQLite `.tar.gz` / `.db.gz` 归档使用 `npm run db:convert:sqlite-backup -- --source --output ` 转换。该命令会先把源数据导入环境连接指向的数据库，所以目标必须是预建的专用空转换库；不得指向运行库。转换库 schema 必须与最后恢复目标一致，通常为 `public`。旧 `.db.gz` 不包含文件，转换时必须通过 `--vm-dir <目录>` 和 `--config-dir <目录>` 提供对应 VM 与配置快照；旧 tar 归档缺少目录时也必须补充。仅凭脱敏数据库镜像不能恢复活凭据。生成的 v2 归档保留 VM/配置文件及摘要，随后才可在线恢复。在线恢复会返回 `legacy_backup_conversion_required`，避免将 SQLite 文件作为 PG dump 执行。 + +## PostgreSQL 备份与恢复 + +新归档仍是 `kin-backup-*.tar.gz`。清单 `version=2`，数据库文件为 `db/postgres.dump`，同时包含顶层 VM JSON、顶层配置 JSON、schema 版本和 SHA-256。文件权限为 `0600`。CLI-home、VM 磁盘和槽位凭据目录继续使用原有存储，迁移时单独保留。归档恢复替换顶层 JSON 范围，保留这些子目录和其他非 JSON 文件。 + +普通备份先短时暂停写入并排空在途任务，在专用事务中导出 PostgreSQL 快照，同时复制 VM 和配置文件。文件复制完成后恢复写入。worker 使用同一个导出快照执行 `pg_dump`，导出事务一直保留到 dump 完成。 + +恢复校验外层与内部文件摘要，再检查 `pg_restore --list` 对象目录。恢复只允许本应用 schema 中的已支持对象。数据库使用 `pg_restore --single-transaction --clean --if-exists --no-owner --no-privileges`，不创建或删除整个数据库，也不恢复其他角色的所有权或授权。 + +恢复期间拒绝新增业务写入，后台任务暂停，所有在途操作排空。恢复前自动创建 `pre_restore` 备份。数据库和文件都恢复后，等待内存 store、路由与调度状态重新加载，再退出维护状态。恢复失败会尝试恢复前快照；失败状态持续保持维护。`data/restore-maintenance.json` 在修改数据库/文件前写入，只有恢复和重新加载成功才删除。进程重启发现该文件时会在迁移或 VM 初始化前拒绝普通启动。进程仍运行时，管理员可以通过原恢复接口显式重试。进程退出后，应在停服状态下使用恢复前归档恢复对应 PostgreSQL、VM 与配置快照，验证一致性后才移除此文件并重新启动。不要在原因未查明时直接删除标记。 + +`pg_restore --list` 是对象目录预检。恢复行为还需在隔离数据库实际演练。PG dump 不含全局角色。新部署使用目标角色拥有恢复对象。 + +## 本地与 CI 验证 + +设置专用 `VM2API_TEST_DATABASE_URL`，数据库名必须以 `_test` 结束，且账号不得访问部署数据。Node 测试通过 `test/pg-setup.mjs` 分配隔离 schema。备份恢复和派生网关有各自的隔离目标，测试结束后清理。 + +```sh +npm run test:unit +npm run test:e2e +npm run test:go +pnpm -C web test +pnpm -C web build +python3 -m unittest discover -s test/deploy -v +``` + +测试缺少 PostgreSQL 或 PG 17 客户端时应报告失败，不能切回 SQLite。ARM64 镜像冒烟使用临时 Docker 网络及独立 PG 17 容器，不挂载部署凭据或宿主 Docker socket。 + +只读审计命令使用环境中的连接和应用 schema，强制只读会话。数据检查示例：`node scripts/audit-billing-contract.mjs --database --json`。另外三个 `audit-billing-gaps.mjs`、`audit-billing-openai.mjs`、`audit-fable-week.mjs` 直接查询环境指向的 PG，不再接受 SQLite 路径。 diff --git a/docs/deploy/vm2api.service b/docs/deploy/vm2api.service index 6dd51961f..89d489324 100644 --- a/docs/deploy/vm2api.service +++ b/docs/deploy/vm2api.service @@ -6,6 +6,7 @@ Wants=network-online.target [Service] Type=simple WorkingDirectory=/opt/vm2api +# /etc/vm2api.env must contain VM2API_DATABASE_URL. See postgresql.md. EnvironmentFile=/etc/vm2api.env ExecStart=/usr/bin/node src/server.mjs Restart=on-failure diff --git a/docs/plans/evidence/postgresql/database-postgresql-1440.png b/docs/plans/evidence/postgresql/database-postgresql-1440.png new file mode 100644 index 0000000000000000000000000000000000000000..771f10974e15622354bebb202a05a3ff10d64044 GIT binary patch literal 44768 zcmeFZWmuGN^fhW>A)us4sg$&IgGzVj5YiOI%70YO}9*#?34;*&_( z^;w5P$v6aEj+1JC>^;Z`iDN|~AQ2ITnM!r|sOfoH8J0^J=Hj|#UxHzdLjVF+sPXSP zm~d$|*54sw|q_m({MLPDE$TppCp*Z?xp|2T z!KupaQ_sg`KK;}dDo+o}t-6(eT>WzkFVv4)uZLgU!Y?*zSlNbf#5cO4L*nCCNJd;~ zExKY{R{9dFUYnGrOAQCcu|q14Bg79=XDV#WWNx+Nl5uoZfQ^G`@Hp8ioNzIJ%F3#U zwd_o=_#JVbsIfScV*gq=OwmLA-ru#7O=$G|F~5AI?J+G`5xbK%H}~4BqosvTM8qK3 zWqAYPsluU_ewxS+&FTM|mFTj4hF8Os@JsY==j>{KN`aS*NPmi$4&l)EmjU#lWNSO# z=cmMM+FZ=c`SyvU+}SGsJ@l(*3s+^{UBd~&C#NHN_wJ=a9NSp0N~&0c`Q5fQY3QtX zkzRqIEv-Uqk5VF!b_Hj9XLOFaX~!Im0-V`xdlDfzR!FV8zc)Zz%|<`#K{)*dA_@Vj`h>Vn!{J2`9!9-$}LwAzU(W9>q6LUjrYHHBux5g(X z#60(2?5uRBatt8Wi__DUW1gwKK@ZLj3y#>0Hu{_xU3k06L_FG03mc$3d645Pc609U zi5{#(*`ax*L`W*&MWSF)YN;YKGQ|q{I zTH(HRPflX^h2$y*jq5ywL_D-lyGc3!-~md9%xDW)r4pv&eYnQVaR#=M+q0o!Lm{Gn zi~d>{#B`)MtkIZ3rBEZM(l1|AA_zYsB!tC!-<+SM<7li$gH}VWmJdC==uf5NH z%7aqC@prq73$e7V&w2A?XlCxh!UD)cWW*1j4dp1OMuI$~Hd?5+R2-d=5gJ-vtJ};e zAldereE7~$S(&!Bc8Fzt#8>nI8Pl!|pM84M&svSwf7jz}+9`N4`6C4clmZnXh3$R1 zedNse^5xI7oG-e1daqfA~Q2HD1I$w2s)8TBfH-lU&iI{fnVPS z2UaV6b6lay**wkG0Bt1Vch7vaY$}wx0xbZ+o=flA_Ya_O}q{0Ouf|3 zN>5~lb!2lbFEN|`>F;@3wgmn<$NA#+Ex9Am z1!tq>zp%2OYLvhe@IKlyv9c=oyHHnt?YCaXFNIzHJU^l7Xql>lD#S4vml!ofGpckV zEtL{CVryL2QA@_z`9o2>JkXg!(?HaQAy~Z+oAuroA2CXaeD;=PaXKYNI)%DvlKZA- zxJiE=_zXgmTEz34@-)n`$!Tb%D`>e>=kZKZjf-WaHMTV8@5$d@j*i_HND=c%cq(=G zTrZZkbgk@BE|#Ho*XiT;pU#bmG5KID$5k5atI`tUk;G%S8&dD+6&w$^t7T3lTG>tn>> z-rgHz==(KL>*8;cPyK{Kay z4w$uDq=P3mQEovYWUksArs#SPUu#Ch>mNY;@`1dy#dqQt5w(b~=kZpJMvh9VsE<*D zs$IUOJ1Hq?iE&eKRb~r0-o{w;ojYR3JIE4ig6glLKH27!J}2AJbyFT&zQdw0JaZMscAPyxodHoJ@7f8{%i;M;mzoJ5 z4~I_D|2t383V>|e7Jjbhp@RxcZI9sO<0I#^jY%;5bdP+nKY_DBJu74bDOhg#F({Bm z)P1^r>?AA4DzZ6nwb%IJ{;Fdd$U8Ge3Al}wc2jMr{(>a)&ZvobSP5ohWhafo)^+N5 z8#UI}mhE1o-<9+A>qC~mVesSIpA7v9?Tkh(BT6g5<97yGh0DraZI$EPz7y15KIjBk z4k5p@is9qq+8jES6yK4q*&uuha&q#4(u|cZrY!kb&%HghKFSFeo&50vo&FtU?cv(U zYLLI*a+-)2>g(69e}Y$Uh~ccH&f(%CvhKvZJJv8eM;T4e!=p%eUxEGnZ%+DAwHBJ1 zj&cIKn2(R|a%-T8UPJ`bo5;`3&i*fHovvWouipB1FG~-VHsv-X8%QP1v$u=@$m_X z@d-9x6B52YBqwLOq{{gE^^#Tf_Px7aWRI9rKh+$UKj-Iwqhn=l z7K4Z7t-~A}J7D)6Z{RfH;^8SIPmWIaDE>S8Y)D7PGwAGF;?5%VY}sO8Lu28&J=8w9 zi1KoT_P^z?>JWeYjEhT1$yRuoTh-~WBOlBJl^Gd;~Ju#}8K_-4JOWfl_R;rYz{4^y5h%ig|^V&@SOl14K( zs{fb=cO!%-`qO4!xzb2sv$aikU$rp=bOJYS2GW(6EhRAh{gbawPj7EwA?y415-}XR zp|>cVoSdHQ&gemMRR8|{hew-`1?r232_gP7<8d+q0q{`3I(`oO{0~*`C3Et{%Kmu9 zuo4>40&;nHZmu;qx5mGYz&Ea^SCKX&9nP^8DnkI!(&xs%w|eS-qy0l*>&Kv0>5uKw z((Qg(cAqd!+`L?+JbYDLm0uw8L@dk3f1aP?i_$X9#!L=sM_F03WUiXO_w%EQor`^v zyIGi(?-i#nVef~Q51mTc^ox$)Wz8H83Ak_E5qST8mdSX0OhBp?lbVw5jBRcmE1AXA zq~x|oq+-sa-A&ERK&so>Ud+G<`}<>4o51lqY7WSJef|DM*$@6MaiLSa@op$B@{`O= zOg-=A8qAZEI2b3r5yQmHu-F!+(U|$;+kIikX44Td?DDN#G|b7>F&EMl86CYy11p4G zY#`#}^_KH_%dGlig>~4GQAS2adS%N|f@R}b-8v#7B8$j3_HS5OSwZ@J*7wvd{C~KB z`ArNuU??zIE)938{Gc+}_s)~_5Z4h{&%qRoV4mGu5-AP$#sl#N&{^$7eu|h*KXs7sS zaaW{h;%c`}t)NwZN^}!+wo%jr``*E_?H>7`q%OuPKu0IUe73yFUt-kcb+A%Y*RBxH zZR!XU7KXUJ<0Q&fq!i{75;_~x!-_jf~Jpto{OuHG8r))m>I+24$+y;~t{%B$n5<=d$3t?ZHiFyu zN|=`WX0(eEL$UcoHI(IVL`pA%r0v}zibZR446x0?Vrny%cZmvMzQtW+aN??UbSRlr z2(g&ghO2==Qp+vSMuUrxP|vjGxy65$xybOnTg3jwp&362@9)n zW4hz~tEnpc@G31P4eRu5yD_Ni#xxrbkLyU+vb(#xsE;djIM<|}9-c{iPxc-ZP+|}b1<$)1KT;?cX1o)H}dM4+Haq3q+Mhi zdac;>hCJE)Rj<4&kpt8y%(%gE>eP$$(r4nnS0V>(JvYQsH7X_Zp<_>qD0HE$FR?#R~aZCh*vpr$Zu)_O-M4)}^{)nCP27efk72 z6uQo1_kF6&r9XB7#zwUZK(Z zNcz>6&*X#r4yrm1879)*>+| z42#a04>$&G{w`J`IB_tubayh%9<0)RwTl%yy%BaD zJt^Y%IujuNISCn3qN}6hsesH2E=yPb>Tm0xg~RQF>yN55u;Sx%DuM+k z-z8}_4mZ~%SA%S4cf|$NmK792uC@aVf4nhbxbShcpZD^@Mm;BX+GJp17Nn+l!NoDq zMh41AetslqFp#1@m<5S2k+`+mkzzQA7ynEve+h#VI@9RIZ8La7??oKLCx7vKsu8F6 z%U64%6L->ttBQH;2gj#->UC26Jsf8JPS@1ZDH!s=%13A7UGB|ifF8O~AAt%8A|PUq z6+-y+Q3@s{Cy&$t1}(0MF`UZbckcSfu*Bn?8JnX#g+))Vb4p<+@4-_wF2j23NH*bP z9y`t1o}NNz1iKlda(6PfPwCUZGRsa8zvK50ByM(a;cbriAB@^vWZO4!q-0?HaA6RW z)GlRldxHvBi-=;XGKjcb_cR9l6?qn;;kifNrjj(_RIP{ zcNo4eM?+`Z8Ox^d`t=uV!mhYy%o9H$&`!rECqsB_=$wY2O(n4gFUkUXr#viVQ!A~; z@T2Tl4NIlp-!Y@jI4*(+2$>)HqBmBJv*Vs=>x$Xxi5brrC_NgzIECEC!-dqDl$&LL z-d|z#g)8V3i~7v-d7nm@n9MISC`H2=Rbmwl$7jV~4~-@hwTFXVc%slN6gtxg$$il% zDSkKEG!R$*HVJA6Ss&OqXS$5T%A7%%D3*7INYIU zw#MNEO;lTVq@c4iTfH4bT5OnJKV94NFk=Ou5o#zG$r|rgwj?b-UAOl<^+gj|iRhGO zQI+}biZTDCgQcnVQ=LYBf3YVfCY+_5{x7i``(!9WI4+4!MUdH{{kV`_TLj)MbQh>o z(|7c^F^HZ}{uGT`d`+>l*gFDZ3MDfa?pxC%)oK!*w6(4~ORKAl>mPeOJWifzet%m~ z00|}S9r-D+wXM-~i)Cm*XEUO_HfpB}HSQs4K`5x^8Mb`4#9*)od`Y0(a zEiXIJ;G^Tl3E95GakM$6)gP`l#%COTB-aLo5?x)o)i z02(?tvzlfyV72*FRC?FwKsBQl({<@nCMC1#lzIRDyTAtaqwUIXu{(cs~>H`Op*tmDHv$Z)u-$l{p7GTlE=Q zt5(H|xNh_Wrc2$W5_DJpo3q01hM{$TI34fcUARhY)i|ch9*ZoaNU*;(>aSk}%;2c# zpFm;{6Q9Dfsc4p(`gcQ zRKo7rl~vHuLb=!;cEF7tZH$m}q8RVs0$xQPU=sGcly_WqeFjJAMH)FuV?~l-)V5>! zHMYyBZ{Hqo#xUWMF$2%7WLQ&JY~;emml zooYk|M>Kn8hpg?opMBN?Y_R=Ax?aXdyeLMMlr=`x zOx3Fwb=^7_oGxAGikZ?Q@*6E-kSgv>6%Wi3Ic=-;Vl0)~fZEJVV{~?3Syk00fQjDt zp$>R}gd4K#8%qAJQ6LRBmBK!HFS$T?$H~>>`n78snV#a6MMXtnsh8)t3-Z?ni`}>{ z@alYzygL_mqj4b=6w9#n@YCPD14$Mxi_Cz`W;i-cNW? z1h+-ccdv7Hf{3_@GRw%5l(nB6gf!+IU)WWODW^QJM_2W~CNiEeU!83J)>m3udAxtb zVpQRXU?z!XRAA@~cThE^ps-`oq&a)Ga0eg1#_QZcYFm)ksnjWIy(g(o)-N%x*nmRh z8^K>LVGj>C^kQ6XiU3JOetjOzazcPZSwOHm>9U{6giTFdq2VSfU*u=^qZ|GmC+L?g8K)^T@nws4&QMj>I#(j$&{=0NI-+o65sGe# zBEB9zPb&@7C+*~7!<zJpkeKiYDriWr^HP%vbc+L0!pFp zRjO(4#qXUC^!6GyECkC>X5r)G1N2|#x?VPIKTsAXWQ_N6xI3c<=tP^9R3UsuDg!qSsfgkC*g1q&RKC*!&6T(@U5YS%2nkMF-Vx9E-;ZIA~QR6VOqWAz@5 zp!Y~VZ;kVcL5-h zBJziJQc@l~h-^3~fM+)f$t>Kt<2TcQ%gfDOq*oMD9{o#&_GVsA?!Ap!h@mtJ_u}?q zw3*`?Hzz0hfFkn&MS-A4Yxfl6v(u6Q)|S#}s}xZWs_~N5pivi)*g!h#njEf+a$wbK zWcDbg6fCn+I%VeMEZz7}=yFsWCUNr+z7CMo7<9&cdszEbQo}8L=q#ejzQA(dsP)&u zau&;&48f%uEW6$zeV1O^g|+{Za)pS9h_-vUNXyVrmowy2wK?Kds_$(85&IVw3~J3w zLzcU9xk2Z^w%#+e%JGyo17t){CFB1zNOMyp6em?dWMQAf&wfjQf@iB!b@98jCp%E%6!U~*1Wf#wO zbUj)HeG7_akp<|fx>UT^`1{&va5Rg*F*6Z0HFcCjrb=p~Q^rS+-I>kd(uS-o207cY zY`4Xacy&i>Behxz1S%?b1Z+q0Xiiphl9YYUe$&|LXq9M>%+4C82XR@*6P!7{$Wy0R zQUV!i%5h_OJxhU%1!hE@1-tYIF&F#rDWGprx4r(*&Ko1V67hXWVnyok4ZzDzmwEll zydyp}K8ZnfGv546Dd?M<6&P6fb81S)IT}a?K_5P_8u85b^^K0HxU7F$Z&J2L=C=l= z7=g5M^f(+hu9#ZLTjcpOf%0;Vth*MtOWhglj_4~}|HB1*cj^xBBI&lDs%d|s@v(Er zvYQh5fKvYbbslW7rhDbfm%F!2@NVC(K$OM?4_R@TUtY(nL! zU0qImFI8WEDJrs0rwMG#hj)2cPpQMg^{;iqnY@mtTlYSF0wwk@@6>3c%|QBu(}c#G zCn%uXs!vnSGA(gOt0a(z(_deZ<2hZ2kXP46tNa+dkr5hpbec7_m&MJMGgk3!gjQg9 zNLQrZ_yjs#>5DZD7E_ohi#mMBA{UX7@e9c2fHDKplIvnyZ@w0D9Gk}E__+0K;{|FB zh*86>`I>nK=^s|gV=s4EbV@AywoF0qTcxtQJ;m2;AJ12>3Q3IOdPqS5R2qaqP1(k4 zQm2GlGcr$pK6w&BBi66Wv%2FU{0(Ao@H$JOtTUQ}YmM!7|A zq4H&-t3;-3q=7(V&9jz=D`98F4tW7wxp zF*pvOlg5#-q!#eq%5|S3(CW4f)8>GDn<=yE<#O7RmE)&FIUejO9%?PQ)VVI#Dh7Q7 zB=_ZWmEt5$PR^*YjwD|%7g1jf9onsdlEPI5>*P z?|>EHUN-(xVWGWi;%sMJWRfR>GrUeHcmP{sqJec^!ZST@$mr_o9#|?}yLQdrxKw@M zGo39BIl16o`;#%~w5z+jn9mL-YBu(0W5TeizAEA*O+sd}%C632F}qNYyMQsc-WM(J zcQ{%!1+-iS28KJgZjAKW%+#;H_j0MPuW$ZETWru!i8`@}k{O@MDXYX5v>Wj~_Njc+ zWflE7+c_)5e{zwRl8Q2x)#!0e=dU0FnQ-c8TAyRBf-%?K(PfE#xa;d*vD+ye#$k_y zRH+WsqvGEA`LG()E{rDo5E5l7MeC!5Gh5m346Jf+aO|IN97#)u&|RaNtZ;R6sZx$t zKS>XbE4oWSusZ_# z`9n(2Ry&7^qV$e7v&ipR^qv8}LuBXTIUnei>~%}rFNAp8l@b^P!!xO=rfkCD)jpRd zN6e`LH#U1}LOTl!lc=Z&8$;;uBdEoxxJ^Q`0xd$b!M+ygG z$X)2GJCnI(CI;yV{fWC)BX#=YMGJr$O(WR+hv)fGH6LB%*NMAR?}^!h2Ps&0kdHDSea<}BZ<3SrGLJrsUJYnibV^X;5ciao9K0v++0~53 z_eo00jF4dFB}AphbAK?U;T(uQ4815ta&9r-!;a2@0pza?<>bkOm7YR%M7H83kGJ`@ zJ}-^9|8$LA#6oL`JhRinX{3e}~NqTGU zz4O>b>Qn@LUg?PUt8q_8kv`&csB~^EyxSf=U?2);&85ENI$#$_5V9Ej6e*rZ z`;qSKPIZlsLA`I0NrV1Ir$B&AxW+AVIKk47c{B9Eij(Va0|G7mbJdIB0G@ zT!;?M$Vk`7DmzcIi+uQDU=bEC;4azBvht^${Q-Nb9qdDW%>0yH9(_4?wz8Z+>oYi! zwV_wi8hFmHV8{IlbQnC$keU~+Wfic_G^m|*$W*P{#0?6~Yp{OqDbu&$QO6oVi|bY?fHQo1k=}<Q9)-c4LVba9UWZ#|f=;n0Y+W)uARXC3T_od#cKg+hI%Ph=M2nxrqTljDK~M zx9_4|7lGW6DX_OPJ5NJNOnKbLWdAXOl{QaWyFf9@=^1RuqRo+co_KhblOmDclO>$(Cl2 zCzSrDVE(uy;HK3e>pqNhQrk6cUM77+K}|xI@sNR=Hrl)`wo{3R|DZ{Ht!?Z-#q-A{ zWAODJrQR(?4d&C3>QRQjC;H!_e#SE!r+|3;r*h4Oq90uiik9%kd49tm`!a8PY&E8kFx&pfD+{Z zT34=8lkQ_V4XgFp*kaubc3L6;0}fM1U>&)hn}W_;OQ72Twus?6Al}JDq(0(xaF*?; zBWSATvR|o>D=}jEu`1|!dRq5el}V?xv=sJcn%PCW(CBh$!sWsO0LOZ}-K+7sRW^|N zr+(*s4W2BAsAF<|o5^6`Grm1z_{^&~u_=QzgjM7zXUQk48 zk8!=oJO;3OP-LkLgJr)lA#~S!ufr++@(>+*KvuFAYm9LJmC>$3RBYAqHTcZqRs?0v z71&Vl(5m~+M2T{HYin5Ip9y+n0J03vd;o(lz`^D|-)ZoUQPzxmO z1zPq(Fe^oIJH=c6N4wX4ob+hQO^O&dQgeOpuL;%3*JSnB>}vqrlhRi%i4}3b?7R|B zFLZ}%KrJusQa{CRa9M`94>k%D0w8B6N07*VPYtdWhXK1rndhY#637qi-Gv5qFPV2h zp`8{LCF@DBba2SN0!wZ1%a6|KXG$Pm42hOtkb*pb&Q!ci$Mx}U+!{dq@fiV@!28Bc zobC@Tlt8>1WLudBTxjQzlA_{p9a5-DA(g!d>-`z%l<`PUyOdM}dNWtcft$HIqf4&=WJ8qD!S1MkkNWN8_J|sV4v_Qjb zy2S?*v*9l<&Se)Y&FqV=z=Cd=S2;9Ay#3Yu z%OWQ`RzfS4IvG@(=%;LVPynjPw#%E@BrqSI%t#dlqb96Zw z-ZspVe{G+ve!{x0?RDIv#c&PFdH_r=HWP)B*L*`a9IS)X#D7MAS*_=syJCne>3 zGu&ThhbXIH)qM;FkcNH_i8z`2^yDHd<-1!Q!&je+mEJGw!rgya&^0lT%;YM;m*%cIU z(Sf=8dG#Xv@MJ?kCF@sNrvn-XDmNoiLnj8qEUo_%uXC;U^(CMY)~kN)f3*BI0>JHQ zpE*@m*Uvx$bSeb!76FEek|8_zn7Aqe1u)?)9L5R1&qUXzukcB+&F-eq+ zh|do5gG9x96o2#^xEMPqn z_y;6??D*f@xpy~8+X9#dD(GOw7eSGxbhFnGtOH5WrZtkZ~9iUN`wZ&*p`LUJi&rA!t1?tgfRrn>Ys2gMrCkuKE zjPy46BNI@Ph9nC!7eNHnqIdA`Ow^jGY_2F}zEOCm4a?m}js^H)?ny$48Id%&f( zkPmuk7kI;8(zf*^3C;if)$VLJRF=p3xB)-0YWoqd>o?*WQ>nY%%DMy8KRMTejLHZ` zi)7rph23|fc`{Y8_2pz3KL112h`5rnYJq|Xc^nC564Z!*=TbFmJOiS^7sp%+*bBO~ zi|LU+tfJgU_fi=&S&g$5f|~rp$ym{bT-8!8FnEmpF@|29$<5ta6Z^m;B94KfEuPC# zwL%RMIJ6mQ`#)Sjpj22soOrGP@E|gXE|L&Z_Uglv5#UCPs{=W-GnT%5N9_(3*wexY zJH(q-eYd{vFE0QZS}daDVO30b$}{Oc3=GvdJKnIhvUAy4SWc|ky@3-Xds%ZRw86oV zya(KIIyunU1`aB*eW9Lrl7hlQpJ_z5PDVCoyuE;PvR12H+N$5!*~vX-vMth#F^kf& zJ3-H3@aNk0A`8i~4fiS=#y&7UFAq}zth>5jA+kZA9qbE69rk?DXGK+2hvSrj?i&{h z0>YJ0q`|AIZk^%+mvpmq`uhXBoviBBZ-G0<6ciU)EJq_x%#z;il6BbD(k5S(rr3cj)Z-HOJ*=>vzc$#LtH9qrVCJz6M0AW4(TxnxCEiJ9AyGS$b4Pdsp4kp7A`;_tbui=}dGQ(K` zSys0@7y0hOTYh{Vje@&+=0;=>P^&ZY-atJ1!YxzTtpk<);RLuq{36J+AFJ!XE3-b} zGb$&;*M&WH0VCc2wL#TS_F5=lX(q>YRVeuf0WXJ@kP#mYdJ<_6{qcyWf3mzuA0G1Z z=g)%6@xud^U&Atheqg{k!v2y`=Z{?&K~s!(|AB8a7YEs(-aBQVCBWU%U7g)0{QW=gLoS-C)OTpR^rq ziooVaYYYp3kcWhw>PA7=seR{Hj4CyQJ-b0Emh*~z)VW|$U^@mZ$Xl5{vhLXA$*R~C zn!KTV=cUH=jPj@=+$I7l(NdeuzCrrXPN;N!>&^^JcMldJ!mL-=@u`%kK&OZfk~Md} zvNB)ej^=$zMKN!TK^b@MCJ5P%X?R9xB0@UHQ%xSsRnr)Ovd z>vPnIom{)35-%_5`kFRuEh9a{O{uY6B}J~iTCv{$TsumbTch)flI=)tzSvoJiA%ko zUm|}o2TuQ3KNQ_1p4keFZRo8UyKF^rp(-ewAGI@Zlf`997}jL=s9VX#KW$CVN+=}+ z9+i?tp*<%=zD{JK5E9n$?C2q$U*v;Gh1inV+qz>?PXgV}k^3`Zpfb?Ci; zXjb+a|||qsN;QUB+}E;wbu3Ylmz!bls!NhNw+Uw zuoAZX1m~2Pn|ny4#>J@K8SzR_spP99p@l-6fa7WP*&2qvn_i|5Fi^wyM;*o~$#P_m zpWrmrotFP1HydgG%jk5 z!2QLGIS5B8D6d2Hasjo7g6^cWjJEznLHdEfEPOWpWf)8$bL+tRMXBsZ&_J>vznp9HU zY{&GgGJ$F6&SH9o87Z5g$s2Qk0mll)O4xeX01^Jh0E|*dA7jhL`x-XZIb6fKAu z5k8^rNvIO&67pXA;jjLn$?N>$I}Q-RVwCZvd~U1_;*@9J^^)_r68OOYg-| zs*3=Lcd*BRVgLxC0q~f?!A`xla1a@V`0U%eG~ckw!2P?`PUGDfw|zumLu32YzLU)2 zk{U`s!mRV9STGrx$kuk5%f zoK_|mwY)Btm#Ml+#}&W{esn9_d5C z;44d&DB+=#f43}C@*yfi+zz{-CP5^K*EPts&&9DlIAq1%A&D-_qVGE(xUKs6GbuY7 zuvU_1Z9`c~zvQ$|DWe{-+>!M|Xa<{Z*bERBj1(H72Ntck8uqG4YE5dMW~2`X#ANX+ zn_44Xh>!}#bxwAjC@n&?97nC|id=t9YU*WoTnPcS;FHa`oOZ~ay$7hoM&BbL4k(Q{ z4Cdpro>LxH^;Xgb+EAabJ?NSvx01-VX>64*k5pIU2O?JHPVLx;Vbt>8p5ugz|Jw47 zxaV-Kltx~*{MBN$BGiuGt&z7Gj%n9ykt3Zye~yi*T*W2>(-@D9Zi#a%N&#lPy|5+l z4H{8hEwB#d;pUINBS$(yY8JsxmC2Wpl8#C{l4buYW~h&52MpEp@NmoEs{>)RQeSet zqGAr&R9}R*+ivFOyBlIb!`~D6Z63ovH;MucR_b^--Y_Y%unWJ|Cc}G!3vXkjD}}AR zDx!pkn7$&`fA>$596pz;l@a2as*n`P=%I|x`+_fx_yU%KePE#hXh^k)d^yRd$O!dv zG%#mH-DFNM@@N`bW$!?KP?1ZXG>ZC3R=iVYfA<^tCcEk+bfaR2m?R!_u`07hh5&>e z(+KGU`&4rvDkAl8^K>AAi_f%2a(zhKPCTkm&f?jUVeT~C)$C`~tdH9!9dgTs6f+9E zV|g*%?B2kNwkSgYe-8|R$3Q#kE)OAQcEm1_+b6 zEd#W+@(SwSw{=RUV!jioGGE_7WVz<#KAZfOsf}P}OXT}QL}2S{mia(c0ivxOU4!lR zO*!2zI)S14vcn)2L7l!2&E9&QwfX9m&=H{~emoa2To04e>DlR4xtE!11EFy(_qrg< zq*@g?GBJz3tas~be?skA6-ik)Jf*&O%6%fItU$k}8R(-sGd|;>(BPbksO!O-NV?%- z?x>34S1hqC(QBsC(rvb)8ecbfzp=8h4F@YhJOHS6-5!^IwIrjG;=e=!9PC7fa4>aK zN*wk%%oG8^!BEIuB&plr73I~ zA+lmp96;etN*DEbktO_I z!u8Eq*T9pEbWx_q&!6SWXQ8;*TF-~$4_wHweBYod=$}AnlA291q?eF$rOi72CAg$o zL6Y$4(`^b03PWGgU1E~FZyzud7tY&fMQdNbf`-7NbBmSVaJCQF6|?47x3_6)zKdwM zcc&#B{P>l1vk7=Yi>am>L1Y|Jl)&xy$0v2;Sc+_DKGyqqY_*W)|CCLH&NVTa zMi3*}*g}3g1LNd8*bNzxzHofdje+OudGM>5rxjt+Fu6hEOPG?LPJ4a;@z@Fc7%11~ zu{v`>zaKX!l2eL!yc zXZ$|JePcWJ!3Ekf=?qFPYcmzs+9>(l|S^kit&;2zE741lZ^-okdO}MBknED z$SN!>^iSp$XJ(cMv!Z9-=+mBrCOVO)Pv>1|MExhR{oZnM`1W+jFL&gmfn}XosQ^`h zVb4@Z%$A8eMkPMszo<<0-F!H6GqpscZWOQ#kzXa;FVsiIYt15-@HH62j*j<($KqWM zMv?V3&I|?;l9HL4c_aG|T=^jo3S>nlg?d2b+g)HA1ulURk}+ySVOfNz=1>P3xhB4V0z~g)kbJ@AIK!>M*H0(t3$PBi&<+Y3K zjyI^7F1wJGP>MR~M+wB6fe8Vngy95vPzW_xA0#i=c1@efpbXuWH#d-;!`>(+&KZk^o)XjuwHWG=3?6%LFVKA7iGO;(m3C_$S z7lh|mX=1wi&Y+@F`Q74|T%yJ)21lEtFf+)(VovOR>)XMTeL zuNgQ$-XKeT)*>!qv5nHyTzrEctkvJeIs+SCH|XF-x|`QqD9jJ%<`+Q1BC>P5%(*{?8F15puU`PJa93v(Hdweeu!H}^U}%~p(j zlsF9{swMfh6&eTQz^QYI-kpzNd1wK>j`~^)xPIoiZYYLR|oqt7(N5# zC?RMw=}j2!k9i`|9IXW|rwD77kzQn?aL9mrxH+O^dv_4QZUNaKSy@?8#*ZKF3OGz& z>VunItk?q!1j0@eu2WH3*%!;}0^n9wQMK0;JDF8GAQGOP_F)1R@`)%YSXvcP3)h3f zEnw47J6K5*vUv;Q;GiS#!ia3aSEx@qkIoL%>q^`LNX3xVV9lv<}GD>j54>l z?b|;_M@t0pv8}M_GYlbxtBk*!*YL-YKn1gKeyLA(fngQ3UH>FNLZ1Osc()rIguLXG90+=V!0&iOY?>ZRG)jG-HT&!ypi_59V=TejR0EtWA9lh>3_(BL_rFQ(u? zD-$MHy#g51Ey067%<7O)ve(h18?t<9B}vd{y^JJ*7fQd3Ss${xyQfCOR1Aa=yU}91 znRK^s1GGGxX|~Gva+BYI`3G!iFsmE*{R<4Tyk~p^Cq33MQ-jN83Yu@}*1?#eL{Qi> zV%DeO!m>a0o{H(Y^Xldssg!1{W4?a?EKT29p9RFy+l z97oA}=vW*}anvR7@x6k1;o^p(4=nlLs2YRFf@z!PC?|^f^woQEms|Eet*bspp$edgRhoe)tLa_IyUSY6dBz>l(b> zFTF6jIH%7&`uXw6hnS4j2}R%2a&b9tRi_HOZarQ9n3c6`B~}22J)4v-q$d4`V95!h zK78@q+hgRY*63iM`{u8=jdNzCh}3$#zD(h<8T9n@1S22W*D!WK%`eoeOW`3SVk=g^ z4MK)t{UexyUsa|8W_#3DX8@RKxJSm9txruEsAtL2k!_OvBRpqrL*j-N26%jgZEaHHnXbPjd+YL!N|J_ z2<5w9T6{tDIA7)9D2Xka>`UU68N}05yc+e9?vdk`xZ_?`R=35g^bl`1JD9JD4NqSZ zr~Pt>1%wZ}HeT9r2=%a@t&})-52 zAQ*hz9D|yt+nuDWa|smd)hUaT7MQQ2NSL+ZcRUCP+~13kbxI1@ad+_W;EpB>ANKE%jhB1OwuZ); zzmi`cW{L(rV){;xgyFp815fG4)R*)_g>5^_xDQW2j6#)UMp`;ZUU`VL} zhO+3A?#>xXh92r%?0w(o|2vP*dHv3Q=r)-6e&Skdy_ab0(dCF}nZNV4rJS&%MUXyM z)lYnA+gloxIq+g-*ctxk$$PBip2F)nj^#0DbP{uTRafK9dQRpxsH|PtFV5^;=Fr1v zrv-oQ6zJ>!^(#YZMK?$+`mW?jD40#Fpv7sZ*wkqQsjPY=B`poUGf95O_F?f1W3`t= zT3TAz@#vsTMLnf`XQ>6wv7P0ig{7s1`FSQDVb_%cPOd4Lbf)&WOQT!9U zsL|KgMgEnQ@74m;tAK~V_t{tc5;?IbkJkvA8ea~qfHJGTgWa5rRZd?d1!_X0)@sMF zQZGx1V{YS9f~AsOB%<)IOYou$t6U6@*!QgTBfSbVeT1A7tltgCU647nOxx z8${kY_FC#+0YXCa)1teQq_ixRjyowBH4e4RQd_g3;sgtnD)%ZSB3k+8KqpHlz4X`HJG&gT%{v|8XO>N=?!SHK1tMfNQ52e%JQ;E9hAH}liNny%hID@QGT{ZhH zY{YNea25AnBq#ACDlg*hkSgG^+aJm{hH;4%gdkvvS!c|*IXu$Q0XPO3ZPDYqxx;+l zj?P@j&{E@Wttz)$9xK^sD1%&Fl1}xj+Uo3gYqEtYW745ioR1UV>tCX&ao!I*$k(nt z!1Y-rmcL5o^;|8UV^d5JVVruTSn0Un;NY~mu_5k`E47{HvFrVj-xM)|K zSVWm|AUnoTY8Qe&lF_n6irsgf_O(o;7$4x9s5LzfVwjEkXO%nqTd&-@Es;Mw4a*az z@+M8+mnZvKE9|gRxj;gDz^#M?i>OxgTg*rssP91Hoj#cs7JnmL9MK?X|i+eNv3gi z`F^9_(!CYdp@Tg245eBhN_lioC9UH0f%oIb3DjoQ>wh41t^UwcUa9uyfM$u**0tdm zG0I5-f=3HM#m7`))=i}>QDn41DogM0nzqz!r?MO`uTNhjVqi&SFNik3RbY32@{_1} zibrUgkGE&;!WV6A2A>}vgw!8Defr)utgOr#I7)pM(ibi`D>VM_zrtKntZHu7jF3$J zvvd8~Lwcs(;^OX)-2-3kc`7*!wuJ`@Bt$SI^>(8bmG~WG_?^*8x1*1wO7V_=e!nn? z_!1l%ny*zJrm#HXN~_>m-4I4yHEZ`X;9ACYwLK9;ZL*m6yoANYKlU^f%nH7Cm%OMU zRuHEz(d9yB)}FI>>ByPT#-iq{xVcL|DQ;qGyB#V0bIA8D%btlnQsn1rZ>v*{Z~X!@ zl@d8g@^#U>7Mn88WQdV;OxMdBH-vGi=FQ#?K)k!tDxMeuAJ8B2lG7@m$UXm&1gL_o?wu*7_LXsD%D-Dd*UjDlC4 zl5+yQ3@q3q>%TFT*gig^iX0NoO9hGN4+6z>^jSLZ>rW_sv;XsZiqpXJskC%B<$l!W zpT+KL8dz@R>{<*MJ{>s?YV(vx*$Bt?i9Kt*++n45IT)ndT3m$@ChMEnq9S_~g3;uO z@96Ju-yUgcDW2hWG>dz$j51Q@x!MRT8^29IRc%+E4$0iwv%p@mNRPOer^M7`ZS1{w z|EN5A^ae8Iet~a?l@jktUS|8IMf#b;O(~2B$AZjp!ak(XfBpJTmFt(w5hf;`DT+2E zhE+ZW{2O@F_WCR;kT6=JRZby3D&N#<;kg z>zXR!lV(W%Q^iwO@{vZlt&ODclW}F^hRVt+dB-C2H6P2XDjJ>!gKi95t&#Me@MK3j zO7i?Aec0(9GdMU1QJ_}&uQ> z6fd~Ai>!9;#P`6>|D256e?HYzVe1#Fj*EX(;22XmR^uvO64HA8%=mTwuO3^vl~}H! zG|7@ZM{}(L^&%Ko1rpJKnUK5^bv-^IA?(%zg+1~VF>gM{{vFGPqWX=t65APxUQ80?@$ypv zxZl6*K+?&zlCm;tF{eMBB+T-vnE|FbG#7}zc2P!L+`T+pYEfp>sD$OA5;6HS>a<}> z&i~?v+c}w+80VGZ<8R*=I*m$S4N38&l=;jR~-3rc6xd=UAjErnwZ-wsY8Nu z-ylfMqzI6vjkFa zUzO|ivDfDxb+0#1%ee=<mN(qo3+3HU%*z5zWo0B6?`}+e3yu`4E1m5g@io1dv$0q zaJp#F^8xGHp)UO^*{v8(b+NX63dwdF2jC~8L)xv=Qv}kzJI4PW>yG=&$a>$z9!otwESKo z(H*f>7=9`3?-Dq)i$$imXP6FelaS;pNlQtE#^0Ab?t;NQH4NK)*qt1uLKE{Qt4MF& z<($kTAJxvJs%#}s_x-k% zI)?Gnst&Wvyt8A%TLsG&G-5t!o)7Qee+APnOch&yo|bDDA)^9!+#xqr4Rgk7(|PuT zmHfDJAyLzYQCO1?rRNwocNX+qmn*>li1=jQX#g*fQ~XrV0(*3M`sDt`=1_=+75}|A zj~{3tQU(VrJ~Vv@GL{C7D(5R)&;d9Gx?@lK)r%LS(Xp$m-+V;DZCCKu?o}4Gj0!Ie zwUQyvKQdiE&LPhmTPEzRnMh0u;s~gsMF3(aNzj+j2;k8AWN@wni2ahv&)4cJU zl^*<#W(#PJL{=F&_BIa9g0cG+qcY$Q8EY#c=LBya3>@Ek3=DPA-4ob__$ruuBB~(l z!KHwB!8$K6P-Lsjqx|{X>aC^T=;1%VyJo1l&7uYE#yy6Hb@g<$SrbNo z84vv!;#Mhs+T}0rvo=`JhZEoI zYmH?yC{{pCC!$w|(T;Zg?3~;s+L2E#*Gf8)bH+@Q+GXmRB_&f`*jeK+HQvh+S5Q>Q zYEPoEE4mH*kb9a>nWR3(f4WR$;yPZTvcR2q^(RH4r`@7_X*nV!Kb$ zuk7q?8-2OpuE@4(N@O;lIq&Su7A@m?{5_p}ozR|WWX^S;kK2s=g14%$CtY7qz~`AmiQmyagyUF&5=J63e_ zwX0q{pSS~4z{|DFJ9k9nayjJRoRS4MmPj`b3D`{`XsN5>9sUTPhyj9>scb z$rTwh5QRRsv9Qux5YyY`d368jz)p9by*DbXH3@)F43ITXW^u3F>|E63cA5PfB>_*{ z>bR>6;Wgk%*yXKme^ z?UM{vQ}1~aF2h>QoTn;tslyefji6s;a7T05t@d*yWT2I|I3GGKtRAZBajSj=!Kx*W z!=Sh#(}zpa0~=gccOW7(emkY^wW!B*v;tUF-@mVh=xs#6G%zpwg0%>n7HrWRHSQw? zfM{tw$@Cx<;<=~HcCqJR%b|(l(rvhkC1cP)97D)ubGMlXz#pm$G6OsCph#V#%zjlx z6$fMELiJxWp&$ z*=O$}ibdVGMQHDTHC#!^*L@hv^)645Z<3HNoI&}!+79!G3(^zb-1DuWF?uYaWE zoh_fx8bIkNUd~{r5az-E@e)Z=Hfst-yYKe%=2ya_{A>d%=no=293xGTP((HcQNQJ8%1{(9FilX!DPzf zukT$6Y{nt7rPGTic9Y&ZS5Qqa*IuHr%o(lOdA^WxhFc}aq$nh=P_73{2(1env>C4O zSxp)`?zc2$EZ&b1vIHY1(mI!iF^H0nu#g&Xeb{$5ds$^>I{Kl-a+S*~wzmms90fe{ z?0Gu*#XhDIIrk7A;kUYtJft&5B>E%R%w5X3Y01X^A2bp+wpLiGKpbP`q_-ZGP>v*6DJb5Bku09u{{Pxn+Um}%~JUg6|*C>S6Q#f?7)*?%mJT50$*zb z6Vj)hC51iSpdtLuiFf{z07HG$Q4)%0zAsM`X}f|GKNLYqB;G7ejgcFC&%!fXSb9P? zaG9DWvzTZZkn?zoEHNl&ElbzHBxbzKSYo&At1cphykn5`Ks%eG?j_i+Ob+wrWGZ|? zsmk;yiD&oiELY}wRJ@nVsaI93^V-5Jul6QSbw*{iI?gzg`F!ZbldZTbhY?XQI_hw{ zZ?&kX1U)N8FLTByBrZ%e?PvVItRO)J1e)zu)1FV#|imcIO1R?7gmZ zBvpNC>EHyO;&*lx3?jGRfiFk*npSoDB{1vlyE1B$nD$vFk{uc`w6ZC` z$#WyrsLSeaDa(th8GV-*778^juOy;Zqib;h1(QvebxE<5o&>{n#(B1DrEMMH?0blq zaFji>LnFl6?t}G-vK6#=U9YzXx@t=-C!P4Asa#A1H&@Ill-p>nW8tHtYH(hk0XB80_v5Enn1uG3BnEfC%@ZV3`X1A9uMrKI{fAC;9xWd2FHXQsx-FsT2olz<3 zmR_w$F#5y2<7oBIcB2?!;}*BFYk$_)&=E9$?Ws0j{|O)uQZIfJ<>Utv9xM^WuVsFL ztKWv@Ew{9bm#S%{{YD3zU%kB#O)}L5GLD}`JyuB8fD8k1{pI5#``qOvq(vN9XRyZ0 zqqSx1w4OyXSsij0(tp`tc$`heIW-OzKi$ImL~5y*x+fjP5J~AgAOC37tC=cN{$flQ zVd%>+8KJ~5G|)<B zx*{i?>$E@H8K;-eN7sx4B4- zDeU^nd%dNM->y1Got10iG*)*MVhkpCFW1torf^ALy+ivY=V|r^$<=|oWQt&1S4AKr zGZV23YlU?mORx8JBZ0lQuzE()GdOrtp-H0dKuB6>!e?>J08@*cv2WQFRIOC6BM0p@ z?l3DF$SG^WfzUEmds5LhQQl&uiAKR@i7-YVAB z)s`94aCzBXelR&@>^whjQw7Nz2Zz_MM!&AYhy~y(s+sEc%>#zAg8F4jK0`C4l7!9> zG7fcc{Dc1$t^nPJ82hC#k=ta{hd(DH!{Q$zjKXTKeCFuxqNF6>UqA2iqfg9jQ$VYO%CdzCsf25=%stwy44Er4 zuft&o4)aa(*gZKW7{pK=_uN0M!C*wYt7&*%e(hE6eX4AEuDc@S!^h~`AJp^b79N=u z-cF$W?<}BYz#P}+u3FM(dcoasLBwUcG)=Tfo|W0#!%0(2uGR?T^~l@xA0CkO2@(_W zfh`8jI+fc*?!G&D#4TN$NW9uKu+bG|j`$i55#(Z}GG-|;5_c=e7RDJRJrxgjbIQLY z8^@qt`t`mqVcCD!=pMCTeK2XP>zB!N#3lFTKK{s+oh1=>p8zrNEq zdS|z)-25cqoSB)B!WJ0KFm@nzCICg^cVUwyf=l@avz!gecxPvSolJhdnVi=f^UxBr z9uIUxQZMt2l$H5amTUQ73Wf8qRoe$TRhHC}$#OXHL9)_hj)pQ*VOJI-4ly;8wf-D; zV++}wz;$^8X^&p!j}&y&ObD!8x{ARR#6LWoEK(UuLYB|Y>430}(3cSM)o+bP+_kZN zA?7&#a>!l_rEO@~)YncgLDGu^?sXKe2Rq?cRt%MfQY(Rne|?Z2S@vN|(yzTl*Axy#R+{QTFQg~pqy(-d2o@FT3$&h?kcqkVM8ycF~J zj8k-<&RqcawiOsYG6XkA4&qfAPZBUkH>iTmny&g{wc>nkxCfWIWTfQzey&=+giM;i zcyqCfh=AV&A*P&K^7{AnnupG3Y`)PwMn63PD8(?PEw}Y0DKXaT**pc4vs25*%Hrd= z3GYh@jM0%y6K`tA#C?`|u68sgvUYRZ^;f(VdiU|^8AwMUe)r`XNcd8;uRZ{2|3&M% zCL7kzxn<|tJ>cBv#H4X-cgu5V=#RZrz0KHU^-Rk#`$-zAX!XPOF{(?X4EamFwS6Py z<19zk=|jWAzgjDhotY-~iUIg#|6KJhe!Gdesda9>eshociKO7kpr72$U@*AhH8Y3I#pA9Ae-Uyxh#sD}Mc`E5A?eO{ z6h1V05R<&J**vp4>Pahz$ruyGI7J&>nOjRuBc>$D6|Ola^9r1prd-g`fL=8imcJC; z;9JR!WN?fL%+WwqIPUnwM;Q5^3yE83P1GA($3}H*IGD=GS$=*MVA00BmmpN<)I8mN zJ345@b+y@N1wfdjE9-k)`eP;+Sg*+p>7)v{yWk5;&TpIu87 zMBySCsIxi^BLorJ=Isa_p=ezTHU$KQd=~N`hm7-+QL#B{le8(7e;fj+XQCho4>?Ua zQ`_-0Q&rSuA|fA@Xogqs*SFn1&z`?RTTghyeSPa<*!yrPNy#DYtmxp1=xdj5XC$@S z{Ym!Ru^MnQIv1BvlUUUl7&iGkD^+Jv#-&EqJrG2)&O@lM~?s{*0PmQ(NJWGixU1_3n=c{}mAKuSr z^A|QGt{{g8DX&k!XM8|gjVX}l%F1(h#oZEqk+V!}kVF+D(4OrGY1@oL7LtP1JKSph zpBIzU9N=^clC{Ny0{yBYVA?fY%DKpIHx}chUHdqg@5qyI;LTOk>xNYBl6l|(ryEO8 z#c+8rh&s=_nG{~1?23W~zQwKpEOS5y-TW#0d;YVizRUt-5Cg+fe#iWIYC*Kp%jSXO z^=4>_(tW?0tsF!g%Wf1StQAHzRO0hWG^^8~S3F->fWf38NfKe(RG`6(@zk}Pm?(kEl_6`BDm9{v;uGAOsPOkHB=I68OqVg~m%X)Uo|Z%}4~~6NywXk~&(HG^iIcb}6Op46(5tKNkuL)-8XgvH*^o%cv$&NvYe4A;+k& zfRJS^^TQHv{?l;Vn`zj1o{=@yl;PQ}2QFzxvL38G;yZZ4g~7zccod0BSC z@R3@^Wdw)3S!KR%)nkl3 z=fkHLC^Tff(DtoF&bqF_!O_TIvX-_~UT1jtAi=?SILi%bIK4e&s-T^HPr#N_VX^x3;lI%7v&`#T63hy)jOfH-gosVbVj%Z<(#D zo`H^-?v#6y=Jzott#lwuE6X`6dvIiAYkAndfolfI%5JJH2k{8cjDEUnP2Ly6o0zaW zxo2QO1jx(l>iBLOHc9!KN3ti^X2)F9!h1IYHh;;{1i&63grjQ<-kw#1bEu?e%r_d} z#W`>_*9`+w!PVX#ab9_CH5syLtw+!9sFpUm{u+7X=nl(yHobp@X7m-cH8rYuKAUALm=gFvTNqZ`(a~8t6gsNDO7EL zo~GdVeZBhNn=Q-J2kT`+L&KQLI`Qd8w{6krgeC1{QD0eVb$LdOb9-UHx;m6#i!5xe zxo};$+zvS&gISHnIxh9*FzLKWTj?aa?BQF(fl>m8a&Av?r})2orCvizO3J0S4kgQ2 z4rhZ)c_g*JD?{QK{}yzcJ)G*KuG%?4stT=Iq!r#>x_b3;?8G@q&xe7@+pGqxdX$t{ zTCd{!VHT0%3cuI2PlFtyg z#M395d?Ra#iKKnRl#!hIYNx{m&1WElrT}P5VQOk7t-o}3>q1lP)^MmEf?UQx--CSX z`nR6^*Dy&jmAe@in6cb^q?9vbITYImUgc&VwbL`i^&xpWbtuS5;%L)7uTx=%i`pCc zx4<%ATpmMp-_Pt9cNEBV;Fq;j4QEz%oNFo&Px`dk+*8d;u)q84pm&YZfNf z41yTUC-}Zxl0N%X^>e@9R=u#~+>G>VxBcDhx;ccO46lr(2t?JmjL@aFrz~CzdApf7 zNbZMFQ&2D(_+x|)13Ml@+Xa)B0LhA+OP7J0TQ$zMdJBJIE^lKY$=H`krlJt~2l?k|EP4G-~GQOcdG38>qN%&c+{o-3~$+6hWl5 z;B14iBqJCSI+S8+UH1%S^i*8lP*uHk%V=icm;KIA`P;x*PxL3})DM{4B|dQN`c*_f z-LA#8F)-ITvHCtMyW-H@iGXY!V5p4nv9qw^JzD%6bV9B*dQG{slP}ipY~p>$z+YOf zY?|>)JWs;sa3F7FdG?dVugN3Jrn?$(hGW9*I)C&EZK;pj-CXXI;(xOGo3wrSuF6pj z4u1uCh55Oa8i$qhTZ8V&fUEPu9~{DX)@Bq#56Gpb)?g~-(5cQod#am1GxvGkL@7sn z3{q&YF|diNwinq~@)f2PR)EvWai*8eI^jvR%qB=g6q|_ot_KTkC!Gz0Lkg9sb%cjV zSwv2;Q!db|TCTT*LQxDgZ`?J)%1)mKo5t_Z4~0Z4C>+o8Pab3^T-AlC$mdP~al+q0O+vl0j*n5# znx?KhyQqFoPd8`GPwHc&g5tLHiL*DJzCj5gRH{yt5w2?{7faW-8l+|p-dZI@2yhyF zlWS;&SaqMAvS?z|{}R~P`a=AW-Tmvlmig=?o0eUkVRkyZRGH@LB-2zxdUj6L6L_0 zbGXn4RN6{ZT%q=lD<(3hIh+gO4vV==CH>m#%x3sx1UHpkk_bz%<@aaloGcN5>Z9Rx zdPB&LzF@JB@@r*aWo5M-&f8Os*DiJPUw+%i`pzR3iZS*H`%9Oz(x_EEaK9daWGC*m z{2mpAntg){Ssiu!?6Q@kYIOtA(L}{N7V!ue*5q*YC-f=CkP42;rW2c-5DSTeM=KpZ z-Z>62DTCW_dzNdTHp=5BhyEh{(nP7V`BRf$Fc@ozW#!yg9cndgiFlmTRZQcvG#AAb zT65HVT0ES>=b%>ybyvPCs=yA_>p)-aw}&RkofU z_WjGN!gZA1?)P+N)lbJsKJ3YVcN@FT4J%@Gx{}lG*0=`GGkn*B6fKJpIy!B~AnK+8 zoN6+1L^c~ioiw>udrFl{qg3z*CR7h}B>op4=ReP|lC{dO*bW7kB@34x=@}cy7FqvG zx|(AacI}{oTx`U2{+*57S6tN|i`iyxMo56!F`S;F#U7vdnh{YVV;3ysy*nt2m5xm8 zy*pd*u#w^H+IZ#RTJAtAE91+>-%>Rnc3Fe2cy9R~U%~hMc?^&Xlu_1vg=~+>K17b4 zt+RBN8}fyz%NA;Uer+3g=07;TwD{jy06Xu(D`qdUAddgZ`0CT>l6nuSRs=>4j*TaS z^yGxto3R`Te5#%S)84@4YKBvCZ*s34qgY_%^1~Re=8Zl*=D0#CyXfWsGO8T-;qEmfphk$~Y^#m&&Dz8cN!F#MMwBu$E5xhh$$WwNyO`I3(@w2lv2<+8-FDtiyYbjxx*qj& zJM=aQy{d(X0Kk~Qt>Rg@!DaUyr5vN~9Ix$BjXKAmH#50==Jgt`=WM-;?2GUtg21Aa z^duuQXTE@a4bIe}gCzOu&7ifKDkp+`IQt_Z z2eFrHAi=qY@#4xz&3c@i;Jz`7c$03MZwUeBqf>7|;OaBD%?BM(C>t4ud|K?@j<7a% zM8I)1?cpq|#yvLi&29q0l#t|_z?6n`0Sj`SCyNLeHrJ*vv&37NE1=U5# zbwwo^fkdF{;eBuraGgM>Xt^&buO;`MS-8|MFkHS~eQ#;hrv2{>Rp^rP&MGf2T#iES zN=izbySN~HZtMwJ52MviS123W{(T-dL8`@~q-th6it#@FZhS>PIz% zxU|)P zamXY-x}wKF;2hkf}Vh9uF*a>y3$@acP zfYU_V3k2RD5hl9z?(weWsOW@%i9(i_EU;I8hvAkp!}4#1KSXj@2|Tc-tBp8joxFJU zDuNKVytPw!r9GH| zDp3Vuk7y@IkH_AaFRzr(AK4sTJ-2(cyY5gB8<1AxrkA5WpQ;jJcQ}7D?5Bgs0tAkKZX^bM41og*bdlacvH~~J# zfB^p(HiI97gA4)!6&`(Z|9MqM3|()G7r)iwe$~Tqtm(tftB&QN>S{0iT>HxGC+t|& zZzwiU{>1%ruxa-InF~+T$}q|XNcaD#ycv^8JJrZMR^iNS9I$yOg==h76l6fq`aGRi z7HR(8?QczV`lnm%&6*2p6x1-{Srn{x7AgmPhvCWQF6}Ta%kCYMC;dDi;D3!0QmaEkI<>+z9g;D3>&A z`2;cK&aoL&yh(FgTigHm0Rzw8q>wtt}l z?xtM)T}d-?=Ks|7@IyvuistWI6gLDdaP09!mx(xGc`j+#BH=_kH{Nl62|+N>FMyxY zf*3+dMg{!m4ebEW($ALemz#fRfJaT+1EU5RCklYdx;efyo`Hh38-M?;je4T>yM4&o91Ni}^2Nuvz=xuloJh|KBxd!H2onTd9Y+8s^X% zORo&^49z@!GIH`P&rPU&qZW2<*ZTZn6yvmHzdp@)xB$r=l;NS2PRT-nHflJ#%h;&S zhX=wx&NhVdQTHp^ttVh%-O`N5>6v(4M!DFPQRF_D8mDan@)U1Iw0{T<)u2Jd^&2I! zqdMxGunr?G)Q)q_T)DWhg+Le)WpB&{`R0xMuCTb`9>YDyN<=9J2?E9P_K-OCn3!Dx z;oo+s$u%AgL=L+ScjqTDMb$UJ+76@Q1rs3Jvz#-S@5-&?jbHuwx=M$ph*8nke?BwN zs=;T5bY!lqve<}yD^6tbtj-W>O%1EviW~>-#1Em218?)@wEk+*$W=}esqyGrJU zdDj=Zn8x9EeE$4g$NiXwQ0u+F4JDUQ?Xk``)0aE2zr7OI4jm!5zLF}!vt%JSDzD@z zjSLMnPcSQ|psmOFTnqF`m=CB3t7DPrjiX~@Avamru^$<@5v`j`BUQ?4DFCCV;0^n@<`lC6`;yYJq8*15=I z3kT(tvYXFxOqx&92coXy&0Y3G-SxP1@BK|pW@zt-th+!&^^k`r#yN+F(e>597d7xS z1sMmfKxC0;1Z|Q0n#Ux~y@aW3saM=>@hj(~OdMantws)Dhy(onxpW1Lyeq?~M{+mH zt%qE8o4V%Y<9Z8?w32)BETV>;DFI5p)KDhRG@SkKQ*>Y)@Zdb4gCyOazE2Z0oOuR( zEsw7OVIDi+uf+3%vUGIn6hyg>lKKoQ9ku{|d`Q^tM^e(5zqGu4^X7KsIfwZ+7GDS# z-LyNZJ-h&}U)h`KiN?_ElIh$E<-#ZNkAEM7VuR{n%(3smV_w|YtOCaf-sjalrubZ&ZMef zUy{7CP-Z*kaP&^nX2f~97X`@f8AcM>O_tc3H-onsNVir2D}L_$IS%cL!Jj`Re0dn^ zo@M#o5l71}gkb_V8z*1eyJ#{KHnAGzhRIkSvrsBtGkAb=H93`S>}V({cfPdGAN;!( z9o1xbo|TSek1w|y@mfEKzp!H)PQ%|Ys9l1h;4%9U9PDv);D3YwmUN^Oy;`OMMi7wp zW7YO!{T!59g@#pTD>O77m(0x--{2>|OKjPKGrTpH1-|m$KiWSDn`fp8s!8EV9LFSZ zx2SNCviazn-`{NUy;^XgBV`khzaxlej*o&~9f*5ijV?-ceUu?QJu+)YNdDjN)ir_M zE>VEob%~dxtJYt}yY}ID3Gz9vxtc+~hO%9E{egpi=8T`%CNb1ZHAe{#;L3cV@59H7 zN4LzHt~@9zv)boSYELALi0Fhcc)rC6`gBD^jaDaopY3GBW&2tpgJ^TAcZ}^ek&-d6|JNUetU%vD^+awKs226oJ)wV}YxwWVXxJK-g}yw9<8ghA_5fkRkH| z;&Im~`NiG-ydE3YAaL2w;rgOk)N*~NQDN6e71{bj? zip+I)!$54b6OPgB1T0HTLD{EY+k#I#Q&^?wG^u%l{(pA&zW+r8YgoAZD()sF<-aAjZ!+3VMl0-F;z&sk zIvRxPCp_dJw+yZTs8D-ho!4vxKBwK?=AJwq{mr1trosx@dU)=@AjmK~|4mCfn@to;TmXKB^WsOj-Sz29p zfF?1?%Vf{ifa5R&M^1N#h2EN90AzVNGo9I?%fa1UQPVfpr!moMWCJKYXb6JpK+GJV8M zIOY~Ve)IeHZ*WxN#?y~RM$wLIz}VUnUjY$8$3py3@`P4?)N%0%Y;%$90ui-Au2kQ3 zyGU-=scvAGGL0&A#ud(yfXc7EZLO2X&Ost(4_Y%mw$Fi$NikXDi$3LWr+EaOWvi_xwB>JqpF8ibopsW4i=n3>{^L)4=(^mZ3;>dme}5bQD5c z%`HY00qLbUpklBBz%gXveQsASUL7slf%@!lW8)d%?&(%MJ94{n@ViRyhxs0Xpn*~^ zP(TqAcubAwxpWGYfWxsiekhGA%}uN-iAt~Ljqgb1as?@f%;q8OwGJ^+%wpkVbcS2k z4@^}ITP&rpeL^*4Es7G(0VRiNJXYq)YtS0GZ{SxAo<98r^CAoesvs^RQH)cL+C73axRwzGaIV!8Nei>^65x|24wS&wTC>}$=NwSLm)w^vx2 zn*tH!%^nzM+pN}nAiLe5AY&xaL$CR4?xKWv?%!M79^1E2t?MM}?W7x*XQkO%_P44Q zypvL>Szy2wx{{r}cw(ryAZCGE!%1j1=bhhK%Qu)uBK_+@N@7gKjn;0xm1q<(tVMi& zEQa%U%xpe=l2=f*ObZpy_$B9lV(c}Lg?1iB)l1jI-8TB=SBa>3#~X%qcX0Ty+x*$U z1uhCrbZsil5Obx$vWgKS85Y|O*iQf4S>B}vq`8RQLQ(-_9F&^z%aq0o8ldpdR1cJq6N@Qs_C zybm4(0YG>UUxrD6+1~lH=WNC?@-C<6&CFEqFpPE`=V6?zqrjzivUhyjVFujV6QTD| z!pf$S_d3j4XP#46$YJpj7Z(k4qXI6y+bgEw3$VSwLk^aM2gV&g9-r}$zj*g>L3|UJ zxy>QJ?*g}ZxEP0gXp)JCGnSCo@+^QXd~S;^PRCf0a~XJV^y}#=b~{C9`T*~Rfg~~v zswTttiyeHAqp~_`d^Y9>hOWRcPVbBL^o{0%nhW8nBDvzsOnm)q%ZFL4iq8z#)D!Z6 zw93!V-(w^Iz7-I=VjD^rxN%glfOvTwW|SConOz7BPl>+Nu9qk#ZSd%}^NH6gNS8M! zI|Rj*aMu*?QD&)+@eoxD*mM}^Z_w3bv;h#MAr~N12})!(8_L~h-Rp29$TF1hvnL6e z6Xz>KOseiU^r=Jy>P$L&$c<8R;UTQz-NF>)Ez^`D~wFlz)~$-|E% z2C7dOuQ)o+*&&W;ExG}#wgm4UzHWYXog3zsyz#dmm`(5VJK*ye)KnvgXoP`{Dyr47 zTrxhC6x?aS=aIwe28xWTBP&^6dSt#1#sew0(Yc z>^lcRCTR%Xb!EJ$xo(reaEOAlTbJl4Dked}a8P8B64s=0kx9$$*bYtiG|wjfQqXZ*{G- zS^3hnZ&MFZYOM|Bc0D2?Atkog7(>Xfffjr5s(?K%sj{;2=@Xonk?*l)`)zq4dX_}5 z5a5>ff3*c5KK`pu4_awOZ>gWQ@(TksUZpdkyaLRDh)*WqPf;^(DFL5$IHXfHg63a<0OVUZ->+UnXP;+q0lkVitSk2#U&f*258) zRH9nzJPx;Es7xFpldg68w7V_feP{W=6N_Ek#%~uD7x(&~JE?-JXN7UseWP(7KdGh= zX!t##r?fCOrZ3#u&L3z621E1b&y(o<1eR1q-2g%(3DgCkUv)oA)-3URuM7tZgpW^< zr<3CJr@>DO6x={5%{tupdQR37ivJ9N=4rOY3X+$`ghd)9?Rp?AnQ5X5v`~FCFkx$F zmvu$BcSbQ|Oc=oLcZNHBn{W5h^D>BcGd;F5RjT&W5mSo^p1VTcB<|huB%V_b^L~{2 zp;ue>lz66c;`YGMhSjU^+t5w357^);7fWH^bq9Ak9yOjIU2YQ@b88d!<1aXm*%dCM zy}-@wsw&7zdFAUJ1D_iPdYN}fKfn5U`uNRXhd(c8p!6Skb_YHsBWbwGf6Q}q@sc~g z@#+ugf0EnZJE&(P4)q6@#hp7xKu-NpBID)drJN!dC#p9O(o0JtQi|imx_NPCXoH-< zTn+W0>hGAj?Ciwr-azY4lQi>j44%|I3pcWyCi+DBS>)j8B2j{bTrA zVjqW{dYwz{S#y6-rdt;1FMT+W-lVwS+eH_3keaC&&z0O6C}G&7GQrmO$v-WM_NWbF?syaqR5xpC8cKsB*ZKPj@z}yBT3ST!yxd>-=U`sT4-@hT&O;DJb#y;nUmV!jMU?>SCrc@HV|^=1 zN`1H#<>`c_Jr$UoZv#<}ZpuRR&NjGv>;r(29Jpfk%o8cW0*a^7B(gJjYQ6Y;$W7t{z7}$ee{9va6S0!{6);IOOVC zfYYx$&v;{Jq4&~!_x;%O=xP?;)h^y3otU|h&V&u~P@RpuK z5B2v*e&r6g5!$_uPVs9awN|N{la*B zzVf))M2Z2ji&I6uZhWlQo=P3KUKufBc3s8&eu*ZXv69C+#ORC zFg;qRCcAWdR)S|hxI*FwH>K4Qs7^A5&%nA|DJyZp$262AE``!Itho37syDQ9f%g|t zzJcpYygVn@dI4}C%r^oK()E_1O97vHYfW(sl8cusZB4GRXu%R6omuA2f1! z24tg|9wd<->1^C|wH{F2p&_Vxz7KDiL?D_2ztAe`(*4j=xQgG5A~{UmYC z4lgY9_xG=sQ&3o5T#wok#y~By0(8egrxw3*0c57GcWM>$!Pf3z(yE(f#(P-u*F(x> z8ro0=gC%*# zh2{FqULiUj3m>jH2)WfB22ea`Rk_h{3;Bt0IL^k+BO;j(5JFq#Me9J{t6!}k| zJ_ETDIB_aRNjcltSla@pz8qWkQF)isDKeBar-OR9>9|UPM%4ZU@oX7@B+{iz{i2Wx z9xpM@!(+u2y<9;sZRrb^Bqkh%Ccp8}ZGjXNX}ddH;2d^=Q(w$=U3KoV>By^2)yQ%Z zGBS-U%`&*yHW}=t#&G$kQFltdO9Am1(bBAleq1cLcREu(5iR=@1BuZ{)4Mh}eHg~n z)YY4hc1}~UHj`%147QCb3^(A4)?twgiYqjNkdV}{Znr5%h$yJD3AMQFP$7VO!x;~p znuArxU!C^k64erv;g0@=vb=k&$MJifMYmGehYw#%%2s+I$-?8keR`{xF`eSSvw*+P zV^v%s6mDoG>G<~3b0}4&8Fpyqa>}Z?M{~<>#_<5CZXRa?g~Z;~!gjO|4%)=89x=fn zn+tgyM2|X!3kkjoV8=qqm_)-z%fPzARHO1^>euQ<00@VAK4R~OM&qNz{8x`Het=u@ z;LK_863Vo0jMley|Be}{XB!}F_kJ9sZ6D|qe>nLITLR^4K2!6xCVQ5Zi9*gfkTu@l z+w*jH&&^_KW&+Qp?#oQzx=++P`hviORJ^kWN?Hknz+jQ5@AGG0=aUKh^5b-=0TP*4 z3C8?QvEn|2YUOawVty zYSs|pRHf4Sx19HHV~YFAi5yz|r|{8_|u!;jI?){*^&{3@Jd ziAn!QCqaVvv zSvSDt0&bodap~nsyta&=+D62+vT%iH~g9j?&3bI49 zOmwx0zqho+1FS9@3f4)xl`bF<@3JuF1-n{4Fe(I>f2lldg9vFQ=NUH+19hidr!rhEiLEM93gte!xeG@bi99kU#-cAwMWnfIzirB4@IN!Pj-E2IyYYU0en%Ym}GEo;@O@lM~8!GZC=v)>PPJOTt zkg{$N?NWq}7aS-dz{0S?DCpc`wiTkRQqLU@iViBm-JPI%?ZhIsoSsLB3AY^Pz1(fP-M0~)`CsQRd&guMD?N5?v269GX| zm2`H#HP;n{m*!~B-MJWoL(*5K7iF(Zac-frZikO>J3ABQ@HhQ|ZR`|2-#7-mKu?>2 zVv^&=6KAvHg{Zk7@w(uxn_JuYvZlrY! znuv0;p;2GDV^cr8vfo@%tjH4rg}&dn<1Q}1r6tvfiTMavgn4X>jZBFnjiq}XlzZ<~ zRFpCtJ7!pZ+f5T^wS6R*X(7vj5AamTVoW><3RyDUak8Q(5WdS56@sq+eaYKUWhmY1 zqG|aNE*|Aaev2C&WI62%Ac$Y?Rq2V$*38(l9ajNkh+?yvOiKb%D(aV9|A2A=L`d|r zS}3clr%JJfB_}6uMSfrY_yc6m3@ms9t>?xBwbDoRgm+dUc6WMbi)~(w;N)#%*jp8+ zFdm4N7Vg_>LNp72|5hA2t;!c>=H>4wPEaYP1|>OTZ7)Q7{-UQZJzom8e%%t{d<9Cf zHV@xL`D1`;X%39oM*YaXHLSX5jE}fgjhWHgY6bh|xH(|a$5k0u82QeuYIX~fEQUoH z&OrsR%kfvB&|FQeZCaPwfLpdbkq$i?UT);&l=j>T%7Vh+zJV1o|6`5tD4k}pS{9keP70Nj; zeR545y?n{RsOv&Z*Jmo@pdB%WD6g|^1@yjiPKjJB+BXIv;fJW|_x;Pl!k6c+O;%FH zor`jV+}pRN2FRv>(wY0sq%ech(7c!?pg5m;GOjyISRcS2a(`|#^!wvE^md_Zl=_%X zP-g2O`Q;hRkb{7wI>Tt`>O+~;O@jB8HXMT<*vNT%JAq#XS$M0<(7n1 z4!*#^vV=;RZoqsV2j^@2igDH*AtwU5#><$?=gr}7u@}>1$#|EuAI!zkUeFf1moDQ9 zBqjAAZ8g4fIAO8YL)aRr6nHhg{K+!P-ybvpfrn(Xxbr$@p^C-z^A14T2p-G+4n`$k zrKZl*x4oK`iiikJ2-orS8VC9vtmd4f{ z(Cz=8X&b8U@>&$jBAtu*^Pcz8uvA8*-@tE0Jj0%gMwl3xRuXsBav!7~+?_*!WwV0M zdQv(%U|U8m-&}9?bC6Sz-Ssl%P_JorPj2KbQF8*`-wf)5OAr2W%qgc;oe}|yrl`jl z5C_$Pz&66gD(|3Hn+AWb8d%PdN9)|EcL3MN>*n+J3o!an*EQ!YRd|N4%)CMN#~j{j zIWOp8jyxfw{HU)R5NQw_5hR31_dEZJfp{Fn+C6%DZhgbW_U^Br{pUAIB9A;i_X z@W~%6crGy}w$V00Ys0>1h3dKw_JS_s0c^h(Hu409AHs8}PngRV_IIlGy!E+hhEgyZ9vo?h> z#W#}}Lg?tgAC!le(t&Ar!E1h_h12B+FP zuy(()r0tuq4pi`2^4la3SW9G^G?XBs#R2)E94K?p2o5tdGmFLY7Dl9J+WOdUEF8+n zxLFoN{=-P6%cfdfHLM_SM{n_3Q0Dx-6`A=sk#DSxF~uXGP~sZMH?4EL+3`bR2FTL@ z<^Ax9-r=GB6R%Z)8h2Vo2O5H3J4MKMbHnPzbys_*45Vp_ArgZ*ei}Q80xG?L&BlgC z?oR2ZyHVZTEEpI_$}X*ffv^;>8)OOU=z+ufm7lX+QhL+#sG*_XajfXzue7%uD7BXM zc5%wV#J_hhrz6#YoqMtLwQ=*K3fHp)vUaJBbpP0dhJ{{~-=y3M5Je~DhG^Zjwk{aA z|4BA#@)o#g7JF=~&^dDJ0{RiVv0xTEu`n?3${x!r6>{XXqLpf*w-0GFhQUWV3SYjt zSwKLLGh~l$_f+k`BwO@=NURuoeDJ8k6O1VLJrdpC5L~_v^J8)GZhY}4u|qsLIXNOR zR@5W(dqk(p6X1m|;45SNFDaFczsT?#0`{TP(a}Bp{JXwf@T8V=D7fxfXhg!f->8;h zV1yrE+&cQcYiz8&bHH+MQ3O!5Gg|V!L!%<-o@xVEu)K?eyuXdA=L=PB&9!YD!lIFY6xOT|$2bXw@F8^1~Ne^QuIr6QV!!UPR|1Y1Y*E zew}0YGyT^YscddPx6_|{8LxCqy#ImG+Rl6JzZ2achV+4sj^(tLnyShEa_sMj|J|G7 zAMC==0S36M1D?~0N7=qG4Ikfmc4$hQfLFRT`XlgKBCG(f0m`p;&gcb(9oobV{oP~b zVc<^L7}~#Nf5rAU4BQF(_X{k-{&4||uzy4 literal 0 HcmV?d00001 diff --git a/docs/plans/evidence/postgresql/database-sqlite-1440.png b/docs/plans/evidence/postgresql/database-sqlite-1440.png new file mode 100644 index 0000000000000000000000000000000000000000..ea8011f06b9cf33d8029e3c8b7dfa9ab2482cd86 GIT binary patch literal 49600 zcmeFZWmFXE8$GI`aE>TQH%LfGiF8XhNOzZXcZo=sbc1wv!!U?+OAXD?-QC>xoTI;W zKi*IOb=SIgKDow$Vcyu!e)ita1b>tjd-)vy`J+dVUP_1yD?EDi?E9lf2w(qt0-l*- zeC+?|(W^%i!U9UJsk`&f95JL2o-LNvD;2zcR@BgpCcOK{{I*MA6X5EE~f1q#BFE#1V@cpu}mx*zkc{pdC8CI0Mdl4EWxyIWrk z7Og54gVyHelHai;4$dUam(j6qeH%|TsqIusb>bS$GOeEjFglVopcJ{$w;oTS|wV0AZ4|0rClm7ib(ZRywU$^I7$=v#iX{Uv+!yQ@0b;@a35>crPdE@ z84?n%TCQGgyrVJ^$Zj<=1?xKR8cb31zSt-7=Jz@_y8q&@2G-=qLCJI#3LX#j(Fxi7 zM`VtBrmGy>-`7tvO@U|e@aQPMTc+b}uZ!39>EA9}o14O-q9Lp<`wNO5uD#K3AP~sL zKpJCtqszl%Z;DRa$e)*c+ski{QCJylv~qJ;EPv^(h2$Q~=<>H3bV_x0bktalx$mc{ z|5+)s@0h3@&#~!PJf~jKJw*yE(WzF^v9YoWmln{}?4O-AJ;ClQ2rp45N?|jjqcMyK z#bMOfpZe97E0q9sInasrz(%0efp`XIl=FHlSK|Er_uqjiB+hnMgg!hT_gbk$7_VM^ zGy13i)oMh46C(BdsKtily8e)_5!b&3Pt;k#1)N$E5+<)saN-#?AshW2gm!jE%SJK! z?P8LBy{rK!cwzok#;J&>2UIkS7#>SMJ0pm=2U7X4*ts2d!{A$E`NJ{X2~+KNNJE*K zndf?@w6N8IsE#>2u}C6*_tS=%J`5_kC90z?QBlzs9KVmELZdEI?`CKMW8e$C`ybQO z0|RuFl=%8$SN6tfBz!-Uxz6x6KkWn!&dtvwAo$VI4P@~9M5J6Hugclm94jkz2=+#I zPL}Dsfhm=%-}J`hpY2Q#ahr@!>Vz50iG4Y}Uz|5o2K)>*-Zk z?`@-zi?w*|s8kyGOT7fvPzM;E^cpR+&s-j3zV6e{M|#pSLYE|D7= z7)W9=szB73=HjU2_Uo)PY-XW`b5kQXsDyL%4?lu2sP1ktv)_%Xt7FOQP>6_&i@P4L znu*)Ki)Yk^ej%Z!FSVSmTFy%=E%ozjnW8%)B>cwdKw0^kf!4yvh=OMQCP=f&sPWXz zWQ?+iQH}3ez#ljg2|*$xV7KD3TOO}8>?Y67>R6K%6yz&R7c?+3QY%uDupzN|O30|& zV()c>O-sOPUQk}n$fj2B-JV>#Ge1AS7WjIeTD_`6dyUmtv(}u+_1A|=oq@KI3W2+k zY*7mDH}Y@EIj+wME^qw}<^qd%G(Ea^i&EBlRX^&;X7JsSZ+QH5 zMceo}xR>(U7z6~Gr5dcls8LZ-%UPv=ajbgiLR<@zt(8hO+!j)&ySlnK?bkX7!|<5J z;oXyyIIZ;bgGby_@pSMr+=%tp|G~)UQ}BzGN-%)B$5O`w90i<={pQYYYd-RyGEHmJqvfK96*ydYF*AR?Q_H=?1CZeAB_ z$F#n_zH%9Sit`KF5_F-#B5m#MigZR*Do+X5`r_Lnx3kap_+LMX(Bbt^Q%n2JDbkT!d|H60cRpY1lwoGB z?zn+?0PHpG`PMelw$vUNiAAeIt)A7ejW}5jZ^*N!kJ}i|2=zA|&S15k(YkJeTp<^g zs@K@+gs%7Ue*JwC?=?Zea$Py@>ged`8y)3;{bO?2roAgX`gr>-Ymq7jfvc5`y{^ z6^mn9SvN#+_xFMa?G+Cy;w!Y40u=)uVbr@zfTdvVKHw06$vH^fkC70%X1AE!eYN4dt>WzSH zJ=cJTgOlGh-QsbP!#L?F^2z>u&W$dXQI}8be78ODrNk{+OsbzyPXc?!A7m;_RMFDv zLy>d6U+$bJo@CI5dY)}N*=4FoNg)IHjU;IquMh2_xh3pV0sRrZKYltj}alj zoA$vwB$TFx27okIpvVrV^RhEBx!oRmH=h$JiO1^-Fe8R@=WAfxXfumE$zUW ziJr5E-y_^z(=xE%oE`~51ph1yi&#kH;^Em@%DW@RE6QX4R?6^67z~fqBut#?wIb!t zmBWTok&^tJ7Ke|Hk?`uY!^Yk19o)kmn_j`_DB4(FHxHTh{rlrHd9A}*?OC<}-7@=V z*5d3zc=_%2{g6<)xg-4k?lO(dYG$U^Tmqhzo&ASqM~VIekAheBS0@`%7A7jSl4}~Z z=A2wyL1z{!r4ZMn(S6<Mc(6{`;K&o`ZJPUQQvLK|pp79d<)T!Sg>7OFSLi#{+m{ z?d{~>KI0x&mH*z9)@)3EkvDE_ZEa>_d3UB4_uqPY#2FbGk&ux7ER{F+gWdl+fYTEW z_|yN7wE+LZw;9*}t>1CXeNYk<6tvXRl9slVmX@wrcCqyIg#7QIr_|NerKguc)63Go zl$Aw9#K*^p|YH4XXN?STQN{1c0 zXn8^kCGeRQF}C=R;ZF&fDl6|F3z&ty7#xtUc^LZ~_#0Ndbvy+H#l=4f3&+M8YiQ3u zEP6&gyk7k43KN@{IEo>y5t^R<=M7pzH#c#0A6{b^k}F~uZcScKJPwT1{p<@(_is%4 z-;1(;eE+t&Id!^4m!rFxfJg{fZ`j!Do~Yk%xOtJw;YLnJ=EWtorF!L*5d88A)8u=3 zx1-;GFW9U%HBS}Q)jd%Blz3WvoAsHTl8udr=Zk6H@0Rk8^wpEE0sh@xT@tMwjL$i4 z5x$J(GAI1`^^s8veNTKppGw_^d?x3hUREl(2s@2@)*8=0JBe*YePy0^IyKx@&_ z(FNOo4h%?B{ASeG?TJbFfdL|~3BM_*fzn@TX9w=XjE2UaDtxD3Q;YWZr_RpYOKCz- zHvt-NZ*58b{`99}qmyId5y5fbhs98tn3z(%et)cc!i;4yvSo7e>!CNMI}c{*;MM>oW*Y+ksYc@+{N|uevO-rSy@?b!)%O9O`Dx|&Lc_q zz`7`xX^@LW?7G=+#Mhr59rY&jvcvSMHR}&tU0sXr67yv&^z;sIJCVPV8w^&P4yV*x zI-J0)JUqDFstbCk4OdvwxLleYE_49pjuM+|e)wReD&Q-C$83T*e6!lC2*Ksm)<#AS z1r1q|m6?I1fzfIG2)ApK$)KiksRCZP;(xoE1(*G>n;$uH{{V{Vuv)v zovNR97P2pajRU(jiaH8|p*+7gGD)tQRFw%I^C7d~a9Gz;iX}U>ck*g>nvIYr4}>@S zkg%Vu55}wtd3t)KuvkU8-YxmvfMU#Px_@ifEn865>U9&beO{_ro672-kvGtjpMn^t%*1QfTMi_VH*ega9w!&8 zmM_)U*I(4`u&>4?F&o=EI(~!hD}MT9DZP6WsYAl&?&jv!dg2<%njL^rQSlCkAS{E& zu~8&nch67K0{57XZm?j5)JRjqvU(H8uqZ;YsQk_hf{(n+qE z{UCR#bmEI1v{%<0?>>{%P{H=a>@5zL(50j$!vljc;k*vO*~*>uo}@>Ftk3*S>yz+JPi0 z8gO;IHeY}686aI`WO+u$vM?&Lg9!P*CA=?apL5Z9ECTw?e6?m0hMb zJv}`IxnlmU{~EYH#Jo2zh~8J&F8w@aHWANQ`0?Yr%#I4+U~0|IUQuNJ7YDPY=S9*o znDS#SH>g}r`vCc%cI_JAV5Rdmv73*7wCd++bSPJAGzZ1oeCOtr=(~HhX{2?`!}azd zUI#Wt-hh2YwMc5zP+yf&u)##JO4GruxGcL-Q&Uq!gtN;&?+>>8A)SJcFo_a=+smmj zQ^wA!lpAgjZcX~NPIu(h;js!GtKX}A{|4OhzeihH-B)bfVHWoT^R*fyqs@%z=JJKG zC<2^{= z@eL(C+vSb1(4i?b^4N)GfFhT#kZDn^LhqU(NMnS zb2umT9!_Uw1d@WgF|RSDF|eW%*}i8xEznVrIN5FE58z znPu-z6WXa78LdWRS-gDt^5$Zn@&LQstz)XSE1b~&_H;UbSm(91WX3{~MPOZC9%W{x z{;c!f$<;P`BA{+&>7y|jd;%~exZ z+k0HgzZBj-YgUc3*t=`_&?>WR+9oA}9Msc;=wz`ps5Jlg!d_@Xxxatl#&Al$ zakXeK1k3hzd^*N-NK;*djFnb3PpM^wwW7-OL4TTXPwd4OqYazakl?LiJVPSQzr<_g zfSQbNJLSY0;<|DDbEPx~mtiS!mcin^W_p9y+CU2d)c5``{I0q^ePa{oQirdoaq+eP z)1?Ki(LWDAH!rAr& zk)2<0@iIJvPn&BMrC+rKU{NBw%Xt`HHCc96R~Ru3jU4^fXlZxQl90oCkFO&FG7yY3 zsvH~Z6DG$CV=8@9DL=H{BXkC#s#VxBd9OAT^Ftb=3BmCj}MB3=rUwrS%>lu7vZ&eqo1-iYzfSamo1HENw2+q#kn2W&b3jj!v> z)XM7;>W9y@Z(yZr)OQWW#W1k!1$p89sy3S=wL<455#6$>oUW^`JJ>Q5#$S)F!!05i z_DJ}>_>A;y6ft5PaLQlvC32YHjv@)uPgk41L_v|CKLN=y)J7(%9qBq zvP?C(U&b1WOvhwM^u7$+k z`cY`!ZI5mE-RlZ9Y}Q8Rt6j>+#ztt9xLZ{}>jL_z<*D^S{5!6XFW@c>Uvmf}M(Xu< zER$kVP>%gtZ4N|Ys*SH7*lb#OOdW|R-rP_zFJ3aVxLZgZG z?M#MLVT4R{TwM61A{YDGohIvj7&_aarVK@(-KaO%R-Uwr5x~j?+ClMnL`y#~2)Ir< zpBJpzrS%2g`_tV8`rcsvWL7JP<<`cj(N+}}v&oRNt!+h3P2zjE5`*+A+gRudhT=Fd?xeJ3Y}$Bx9eTwLUfU%=)>eA z17d!+cxI0}8s-g|B#!H=epR3_vM|Y~E~qpa9o-B??(SK}c+!q1;u381r`&|^Xg(#B z9~p_GjcvBy6ZLDVwVlkRWe>rk_XJ@6`(!CD+4VjnRL$V|D8s(k6OJt6uu7lM+A~SK zOhAZ>er~p1r>tID4LS4Xuag<>)l@OK3u)&*YBs%Fznm}D31vYHh|e?IiM#ha{~+$F z2JcjCWg`?)=)gW$JNa0IO|M)ZyX>neI{4v-ucOeJZ}#Nd>ml}tL`ttaxjIQNJ3;do z9HK8sW-^7E5s*;ws;kUlB@!xPNV>Xk_p|LVd|niNU%*3R6lp4RIS`ERe~JL@joEZC z%7rW|5H#%8a}{h96deQOHR4NxG5OL`DQs3iB6HcBao8BrMzc*!PTbttY4tqo8y;?T zJu;}PV{@(xSv3Fgqiv!{MIdLi#;y4JZ2Np?mVg%cC<|4}nvehuR*P6_c`c`+h>`4W z_1tQFHb>hgM-?=*D4-#!wd$_`ks%a|etCUgr`q{$f;M4EE`{0UKD%2^qwW)|Kp(-- zaD@Wyo8^yOw4l{&w*$e;V2;99rVaK-LM1V-8UnHk7QKYUMmhphB@+~SuV^|{Usbg^ z-%`yHD6${~5TnE`bPdDVe4;=BviXqzT0rZ|D={b@CZ?sqv75D~!%59!-wei}(5E@t zJ0?YpyXW&axssl;3feQJ zX*lIoMng3o52UD(7t__oo$&P4MVt<#r1!=HDcl?Peh)qfFHg^6JmX&*>%X!XO-{PH zLW-qU$-A(%$OTUfx8Pf5!6IB5tX(%+fkwlma{ ziE#67@n7BZI$WfKirxICvzcQ3gYDhnzQyD2ys^NDIY`jDb&CQPvT^^L`juqW67_c$ z43)P7^TvBeUJC-=?p*F0By*CK;~&>PrM>B;@1!H>;rsGaP_Vt-YE*NDax})sIqo*& zR#zhYad(frA%kw?+jxt~LPqTMp`-N1xzi$BEoF-t%0wK<{aCJ_u%TFOf%iEUs7=7IRAn=n3$54F-Eg z41rLa3OucsJD}KV?||Nqh|76v(&yKfVmECpBfIH?*UtSRGuuj&RnY?*b@-xwe~yIP zzG7RF$0piGD%F5)7#?ud(b=h=(o|a%8vcg5!5$&m_Ur3@3Q5VajeswX9dF0OnzNkg(pM=^tKrJ!CR z`5ZHT1N0~uql;d>>Gsw?bZ4@(>q}rD(&$SIiyg3dd~U}zirwb{0U;EH+9|Mb0=8XP zw(5=wJueS0n(cUzBL7l@C^)DDbL9uEv(Bp$PGo|tJN&S%G8MGx;Rb6K5h=6{t z)8g)bx^_&BfOu*#md9!k1Vr6cG*=yfKIoy2idxWP$ zXL5K`Q%3ev2~OcBKB`=65~8G}866Y)b921$Wm=($l%>ZKM+82u_~e~y5zlc3B3s#y zs}*-KRw_2(Ub%dsIZ}&vazta~3pP1NhZ!9)lmv;zS`+%= zh=RnQUtQ9xCU?9W23!BxDTR1_O(mP^e)yx&JgyHIeP$LGMBL7V7doxpLY6AIl55d# zunt)7;lSMm?B(6Ncbl{M7x!MxN7bTuIzcc>1w`QY&ARNz)2gaSn?!wYSaHBSreQe+ zig}~`dPg-VJo^3nyzbf%R~q2itENABBN=Zt(WHHDIbEjJQfKqvTck{j!>Gc-pd+wn z+)V@XUZ9%ky1u?HRw?Io+&paa3sfmlOC}Wx0RQs3-87DQgKUsoP(aPwViT4><+0ck zwUQrzvgwyFUaXm~Xv#>)f99qp7ALvZDU5sR08aTAlA7&mQwPj`{ut}e!NHYR~S!yhaTrDeB*0?IE99UbRnV=^NP0p=)=e#PE(e> zC9Sc9eHu&7U8WMND^a8j6K(SCX`gRr?lo<@ao!p7q6-WZD;Y=UJbX1mkWlacd^P4@iEgvrq@XlopattWHsfNV zqLU0?;I0~tSK#H>OU`C?+)X^pf#hbX(k`U3bdSyBdJBW6U!M-nRh}m`hn`}5s~b4r zeZs$2%u1J5|6PqCsrbWtREc}J`|_MFG{D`_22)u7e*Rpe-rFxHQDXcp7nl35Mg0pB zj@j8+-4wHyTQ^W6*22mOIV0fQka3P!){bG176yY_B5 zGz{P&%3h;=sw5}9zTZ2-++nkt9F6|=m*c0g(ZRd4q{71699o!VSXkIDt6pLQr}gSL z!s!Vrt`_%|c*R0=15YI42uDO@90uK(Sko^hx{U?$cU;9Vn`?_1!fY1NJ9ipNp=pR$ z{jyy8r5k<j!1S9;$vkkdB)q<&pb(G4F!%5Xt%jTsGjf`w8VlLIc zaX>uJOm0{40uTe}li7Zf;J$j*V0Zz{rROQQ1oh^yPhE2&e_0=|_Nl`Pjh5>LulZch zwK%KP)io%YfBOHs<=&`p%RO>N*W2^Bek4>n(|*h1co$EZ#r$s)DGA0^()@W=7hAKi z#9XHInZk!($_`vAHfp7bS<^e$BV$*CC$g*rYdr=DmrIiwoChtAGi%x15MF{8XD`XK z$$9TQS0zL<)jRQxkG*SMcm$Ys)>H$9bA~<(Nu}Ss_ILF@tgEiP&kX!0-2>`*^Tn=E zHILN#0HY4Hrn=T^V@W<=cCQn8L%C1`qkp9a;NL8zg(x~{^7cG78vIclQT&>X0Ikk(hNSQodsr6f#)w?)YR4% zs@BW)qQ80Zz?Yi~^p47`5Hme|b}L`RhrI9|iq6ItZF1>+F|jdW>jQCyrS=(@82D_~ zb^3S7gPL{vff!x1WyFN|_}7<5F`Nz?!>LS8EGu_+_ zHi(vokW$HJ(0*eqFAZOIFZh%zm-J49N7xDlg_MiSJdiXo%(Nh`> z(Kk>7l>fvIA2JcORTwsaEDs+uT4Atqj*4|f^8@Xd#Uo1FKGVWYT z+axEM&CB;MR9_Cn{j_Dw+*k7EPR(YYTaGbh053N-7L@zAVwiLiR|}vz&?h3VboAWXZaJ+q>VN(OoD&CZx61{J5p8yN zSZMH-kAMxlY_Q|+m2LJ<@7<0&p0D-)PJ!hteY0*5hRB6u*xU#GkR#0?`?>qxNYaEL z2Y))O>h;Pu?_VVNwojiu>+I^xt*(lm1itjl%v2A16wT(=mcPG0ulF%aakgQeG;9eC z&ERccM^;vrmKGf75Bxj5^YcWUL0bvPLCZA6{NCt9C-qNdd2q08 z6C%pWAj4vj#N?)xLq=(<7C$UJP7mc%?S~>9w@U2SZp+&KV{yVc0=I@jkSv58cv>wq zH?C|tl7C%%*l@Z}lbYA4Dcye7%ngo1QYf|`H%o`|;_*8hmQ;wq?*iCtq zT%rfrw?CK3bK#+tb(!G;?%MQ3G+6%&H=Z8+8XI+bR}&!{^mk}zhn%aYk+fiHPSR{2glVQyM* z2q#1d-xj2mnp#i3{qt!QkRCoi`bS1kU`hk2AW0gURYf%oW{fSQXMvtGHBP3D?{l4Z zr+qeif`Mw+!w-r(gG$b;I`tOsYJdMm|Hj7Kg2(M$<7M*7PW+}r+KwI@QLq8vBBEJ? z>wC>^C#xD2Mr)183CgrDz%`n#=JXyO zvXds}GWRUqR_++Q3RMNQcVYCWmQxC7hH>b$-B#A;rBus8?+iCU8`!T}KAcD@G*e@O zekZDM`iR@Oi$}>=L|K3Ql4hbvDK^Mk@NEg|D=!&a@q(l+)IbD&VO7a_9=H!7TfuN` zic66a-NoV2*w1Gta_LMsJMacbi}wleXGr2f&{DnP179tYyW5Rci}w!ngxt!CUPcC?YiD|VNG{)%MItAm~QRtNC@*kmTArl-9s)pE+POXFg^`=qftWxL`M8ukV4#4;?yc? zPE?fq0S{bC7{1YSq=-twi`9=76(!Kt21Jd7gT1lr>fGFCWRW}UHo7f#7Pb#MMUhTG zPaZMsG`t|{>zOT?QRx@m~-iXPIg$-W~y^1##KXimlB8`PJ`%&ZdAO6m91 zpU6Te;l~p@w6U}?{5IB@8uIT)D1`~$zkPe%Rm#oVvC`~*5XiCf?_aOaHI%&kgPg9Vl}9x8QlwFCRwP<=4LFBgNzSmO|@u5idLm( z5(Cjp4{bu3mdnP1-&&{%LB51vwsA~=)83wCuFq;(w+9iCmAuslB`ZC>kJ;vLf;SVy z!yk*)_F;u!(bTxl=K;%Uas5h4waWDF;g#Z76b+4;w3%9sYVS}rW&`oo4%4SVsNbFU zdFsWg2DLkph3jYoeO;&7ZT4zp<(pjQrWs=*fBE*gib}kjy%!((;z*3?G$Bz5v&pw0 zs8Cw+1o(&-BwQ*5de9Chd|R)bII1HwrKfFabXY)QDP~TET{G$aNWe4En;YkX^6)HX~U=_0m z8K05?2^0bh&8LLouE^ApAt2DTS;)rn?sDBt;c&aQf8Q%`HD!ShDy3RF2D|2I#+dPK zX>$7cBz?{I#1O6IuYKZlZkGcbB)6>cLe=H~(0Bh81aE5D8OinA)&|^LI)gzQq}3*N z>VGjYybtw2O$Ez5{sd*J$>CsyYFPL>V5LHBYw7B~{D?f}stoivx)4vNLiEnV_p)g% zAVX-74s^fiiHVm-l696S=p*o@h9W)kD*3XZ|pZ}T;SZhIw1Wlr*rs-6b zVPtc&K^3Rl(?H+MwnG80H(9!GVw^-zhife+v>flUT=q&+nmuoY75nFSN>Amqo7{O_ zhIYvo3p5(Pu*ntX=~CBbWZCa_By|s`UmVQ8q=(_4WpSJn)ayJg+*eXKNY5qejFS8EK{Z z>1SOowx&Bfrlu}C%qVtVaBS-4W32R7^xr zDkR*LQ75acyh&6{3`sjVFfvVvyVsvuBI4XxUw`SbuRuKZIw~=r^#p_|!gp`B=36WV zVU5)^Ch`z!Mq*MjSKM3grO^o+L_$snqnJ>J>ZnOuNT z((x>htmFsFy`zY}LVCJC=dgGe80qEiI5MF;t_BR^4jeCO%U#_S(V)TA?|a$x>@>jVJw*-EbhzqVuYPx$TbcW<>=y9mtObFaoutmnL0YnqP23FnV|5Z zQ`K?syK`08$Qkt^MW%iWLZ{Js-TIeRoO?s(}iGAFEp&*P3z^=tz0!w8*Y60PzI$a%VlQ8LN*=qE9@lM6nk`SGHQC%AVfd zND1K^qs??BH>rT&xi2E70WzVK-EE0R;ZLZa*x=wGWf(Cqu&pUZ3ToGT@;LUc&$T

U_k*!{LSqKh&*e(YjAD?1Z z*yZIXi<7q*r-}+@KhQ8SclTE6&F9zGW$=T_1D6ab>C@TCd&8qbLs3ou1L;xTtHxn~ zL!4BJ+k9p1Jhq`Y^mP0z7;^?`tiq)RDp23rS4ck~-cdY%1xb=>`F9Su!E`SNlXNmm#X*^z8S( z+pOLsT<)V2gG4U{S6IyG;$r;LTOD&jwFW!(q4Nz74;jFYgSp0&Y$}j&=s02{BU5k^ zM|OM^k-e+F+k2qR~f ztd)tE1|=MQL^YMmMPOXdxToj(Y#-nrbd>}*;}skvC|FqEq-P1z(D3nP8gS+#h(&~c zUOAp1AP+?OfSXx2y?q8ILFiNgUK2j*mQD@%EL76!d~&{c&ue8>rH6+#^I~gEUw=1e z3(&jv;aY+aa;sni5hWL=#Y3i3Q8{A%=jvU+NRZ2DrWXBhb9os-#DuFHD=}u0eIcSH z^JSAn3Ml@-E3=tzEYHr?uGUxsH`(VJGbGN}SAeXYjZK!h!H3aviqHMZ^K#Qat_MfH zQGQkeRE;8)w~=a3Yp+1rdHO8ywd9MxuWFpSi5YE3PH&}gd6o7^CyJGLoYr}gzoU;t zg1~>g+`A*x7vr$eXwy!D@$&p_y{-0c*^q_AAB%cVi9elXp7R)p`%JWu?segOX>8== zvQ%HwrkrFtRT!)%dB~Lv7~*th z)Bd+`{~nxr`Q$?Q;~?g`S=sfH}GzO{T-Tb=D)8Ncu)F)*@xmQnjaqBF}L! z#Z0&f2z-2W!c)9efH#=TQli>ua5b^R#yK^MvO9}#M=`cRH$FDm{v=vNera7*q)l%(veT4v!R%j_mxOQ&Y-|^7rOhaSslTsnDq+vJY!FgcW2B{}?8J@VdKg zWnTPo#0H!Y;QPMZf#EZoXlJ&Kk>kr0iu>GeE>SkecpWG^YgakJ&dGi`HuuEp=S*vM<)o%o zQ$j!=8yyubuU3O8n|%mQF3sWid*?$>{;x8kSzLU4sOKGJlBqC`cy#mocb7Z~S@Cr$ zJmqNO{PeZMZD=GUt#1G?KukmRiM}D7SKH#r=#${q!j1rE^iK`%6h4^hV~pZ zjUUeN+pY*E>wEA=9EIaFUL>PC*luoZn_JL+5rU~wzk!^b*7T-cO$9T9>WOW!ibkmB z>{x^2K=M6;&!@Lfc26VtQJGXAjCq|wlrWu9j$w{YBj;PT*QkT~#$eh`=`45fCiizg zL@|h}jTU#Em4C{oA|QewAqB4P#UneVg#r(5d$L+Q^GWpaCfBEq22jS=2Gf|Yy!bqC zPd9_dFuGrKzt@QD;Q*mV9+M6|rdj!R_gY}NB(5UZ5n@xJqu;%7bY-znB6)`OI53Cj zrgv~&sP+?9PeS+4Il}p{&3Bec<>!D>7pn^Rh?!v+o^6R&hDVl{ZznsY)YV$fh=+Az z$iP~VP_hrukgYUWWO%wc?|OybKK3{#5-e|DKoF}kb?8WN#b04D*UY-`0DUSzZH5gc zNhjWjcH(rB_a=)hwx^*=)T$;wW0Qgi#wqQw!^UzXGKRLdP7DNx}4jAI(~@*o()FcWk9?Ay1n?{IWw zOyRgoPm_UfP<#I>lyhhE$jbGO!0EvGCW^a0GqL3w<-q2bPiLv zR~Rq7{*(H=1}pwVIlb0wCSGf`;d^^l^6xv@I5fy+INYvpfBCj_Ue$lDgKN4( z+WE-XOcpf)AF^mm^hsZ7Uxk$?14ULwK&tFn9*{F(Jht9+8R5}`BTGNmifJboL{h5t zd_9xitCz!T1_uUe9DA3e-}g7)Eb6-oQ@q99d@qn3R>Y|i(%sb1gx4#yQBz}kdhYp9 z>2fg?y-awo!tl1K2x1%r48w^Z%>UDf4T4fT#GSf z1nKI`@#$$Jf#6O%-0%;L2{$%V)iN|cMt{yNV9(noMH4Qd@HRX1fGA^>Ar;tZLmSf= zSHEi~^L@LEFwd2@97cSb*=*qRW(`yRLuCgw;JbJ>_$kay6rFF zSc`WE0paRAE=}8ItHlLDDaU6j@6}C}TJ$n44||S4dJJ-4g}{UO#A?dE;qm{wP`1+rN*B`^b}Alm2_H8n4^lpFWbneL{au^K7Gf1j!Qtm z{ppe;XA2OLH&B&lck=fFc6XDFjjgj#3m*{b8lz!+N7`PA?>~2!CYC{*A+-fS1UaFk zszQ86spzYxy76`Ptsi=``<)4Zqw@LL5{Wyyr1x+5)Uk*bQ|k6(V_A0%M4o3xhl?6Q zA&nN31<5+O-xnG;!X$rI2zaHE@VcGf=BA~|L|9IDU@=tO1uJHl&d`42w*j}GVog~U z@ocZrem)Q2p_=;Jp&$Q*451FXUt-@?$f0(;0jdu8gIhe{ZXnUInfigHBza+3yuso? zfMu`a>oaO`pcBsm*dF=$>{qex5CpIXQpbp(u+;bZr!XRqUs|!%)otWm^ohJ!6JbAo zyw>KOuOvs55)Qtty8bce-w!<9Zix?DrT`wRdQ;^kB|xI|^>ukp05W7+z+P93wx880SO!cf@NX zf_6!*IATg^=M6BX(Wn}r5(;a*8-Ls3X%BphO_d4gT)us)>QBz*Ldg20PT{jy=gj3@u4kgduU%+O7IAAoix!;;c=k0ETIE zhqOH~8Mp4u@;m-$0kP~pnGtDC%O8?&&+Dz{iMcF=*WON?tTE0(g*KFnCzoHng?p@R z*KV`!e5d?;e11GZJAu^H1g&-GiJ%Gz3IZx3N<^yD4r|BJ%kCx{Ylvs(TY}~i(AM@U z7VUu1&9JyDWNc;at(6tUDkb$2M#VCNj~{$c+Y8DK)2N8AOU*^eye zf@Hgdb&{vsy++&XYq?Fy!l}YB(`!`RRL?<8g**bkz!&gJhXly_(F)aNTe1l9My>g@ zph6w$Y~7CS!97A{8yMF+bT}U0veIN<>q}B3yEX#`**>f#LBmNs;H`K*!@SL0Lt)c# zdv|1TJ&)&Q&p1ttqobpeYF0_W*RSHef}~=5g!~22a`k5KhXrjgw-pl<9qnnS%&fnx zOD%7Jhh(&B2Zru-t6T-z5zjCV&bP))LF64Q%g%<&*67-5#*;HWx#5e;Mz5QprZWQU zfsC;{C)!}~=m3xWU5~H^t2GJd5p?e6jN~hR z`Ki}8Xzk7~ygMIk>~&2xt9R_|4>R$m!)X`i=WML3R4_9Z#pBmSd|QtbjL;OTcQqzHz{4opm0R5t_hYGhMUL(IITL zQXaSA^YLAfg2r2-HZ&rl@V8EZ5!xS<*jR2C+(rG5*Gz^}p$<3koYki5JyAp!CWGmr zsKnUkBplSE;eW4mL%gp}JC4Y~nD@ouVJfd_y71}PFBkc0XI>VZ7`ig*uSDD~LQR>u zxf6|w-F`P04G{19)gwQBA;dz^;sx9MGdNLL%I~-t)WF_rstfr9y%0miw0!EEMeK(< z1Vuo4la)iEt2Oa@2z(C#NSl)ILtJiPwa06HS!f*Aw-k=3Mbo!1mCiju$zNXI{{~j%@JUn(N<)^f=p%NTF3J6OD~` z1(O$iA&%Kn8?r73bGBY52Hkao)Ef*8WnddS!FV`3`8+PG{biighR6A4z+Sx-oQBwA z@A~Cenl~Xh77LI<;ZujyG?J`|1WK9YRd$cXU9D14q8tLa`eY|nmK^;4ugm`iHRI?t zVMc5Vbyi5;ljW-HNeO89_%rqJ59H+JH0nUBpx3NTWA`CNSqE0)sLPK)$KvB>iz_Ov z-rB(m8UCRO--m*IHutq>?P`En`nL)QrxF15FyIVNr~tnefWiSZ;NFz3Ob0>|U}$7Ue%+lvKU(t6MEsGS^54@wsztkuo-; z#`paEz?3qEG9Wg?>B`Jeh`H!!P^+n%@ZKANZ3yQ7kD7S#m` z_3@6&oQp?<$QLTS34BSU1@+{d^E|(86wcw9?f_0FvSD{vUmPg= zVzPQ;$S_mAnzGk2pkU^1zFC%NR;o9-RqtqxRyFEpDyP$2U!TG?wvxu7!kbR2%8^9@ z1PV#7U-G0}!%dF+YL$X+g!D(l<}<=05_M|_8!T@W(|v06UftmrvMhcYuXxO7XrWU6 zK?x1*YEMyrWJ@*Qxs>AIJrh$zXlS#;219&*ktg1F4D?ojNH*|B^71)|F)bGxnnOcl zW2OfNGMG6}W%aP$V3;4NTsa{5d_Y>VJ6GF0MZ6@QO*uGOHoLfg&!D@&<4Wc^l)#_~ z)F;Lf5`>9OFkt_J@E7Wmppil+7Vp$$xK100=Tf%Pj&~jr?1@f5kAwmvmrc^RYI6>< zi;FiA#Cj8g(63DFlpW+aDX=mtcwXT<*^T^!IPFAi$G?$^7w!ufP<0AG5$u`k#3MKk(LIfyE|1xS{g)-?6@IB6-?xQ9M&=(h6LZ_g6;FU9WqA>w5kqC=VWE2~Ak z+gj82?DvMlv1>T<#B=Om1{TaGPh)B6?$?{Z9&~q_q1fy4RA#o`ojHmBYPb2TjV`}j z_kFMH_Y(JYZ02e|rVMAvy`lD4Y;!m3P)K^tO~cmnG@KN2hP%xTQ&l!Pny{KA!&T}7 z3nD|waPh&95e$0`aDUgn6TIAGJ6WFP9x^uJutcZbI6Pv`{%Bw+=Ba`W$6fg7v*h(^ zlJor_;y>+vC%(VA+IgVzm;dh{Sf4G-7ynTF+IWwhTOr>=)NSQOaIr0m z?M~}d2DU@j!d)P|Hn^>}*QYTr|tToc^;wpRBn9 zwi4;;wj>w7n8gJl?{liL{9|(>Lc-wahS8!dHs^gRBjT~Ku?CMLOv~N2ve(Tzp8Wim zp+xMm1zra$VVQ`t2hljOnQtLw7JS5$x>zSgaZUk3lpZ;I!jWX}3#m{-$FpuVC;UeD z#ZsfS{h}F;~tD3CFs2m5v=Z|tXOs&HY3mI>DPuC1{%Us zguIcLI+LxB3kbYDPGSNZy}G_6To(>JcX~p8|K)y!Mw#Rh$yC+r`JetF(br$zFK%^W zwV3pMrrE|_eZDnQcLVgSU^m`t*nNOu=xn}4>W;7L&g{l$9T|EPWUmpwb2my1pHiih zy?eMCk=CNrqEl`s^uE&9a#CXHuAt`;))VPE7whthCMC}%pT%#W_7ErPdO^z$2MA%` z8&C#F1#Uob|5IS26x)?;9A?^nqmla2+va7p7o3eA9shZo?I; zA)eMMuAll=D%EB|{I1*SQ4#FwpR#)5GSm$Re_WTDFszVBRk15`$?(oKpM*M4M1>J^ zj{Q0iWaDHVy=uBENS)&nT>l#0_;dAngcJfY?#z{63$3GwtY%BMdM6wa5f`w%@fX(b zZF}1&c{b@E^KfnBsjE|%s!vn4qC1VOd6V`J`-P?4gtC>_X=*)q5R=1WU^nery< z7+WR2q4WVOFFV?(bGOR*Nx1}2QgR#>G(ZP5uM|~S% zzsTBlg0qkY!h1H@Z!&vxc*IBFkVSjm6Y{L5+tBNR3#g%X}TS3-ok%l`&sIp`w46L03(JpZNlv^R7ysad@|X!rU0noA@Vo6veN7n=r`30(q4&D{ zRefJLk&oWN{uQv#I(Qp%*SQpaA(^b=WMvgmjbCc34~iEZ_%7>`wAeXTu|VjP|BP%vv&@6I(vy}t4BF4C=up^N79xIjh) zKCRiNNwu5LP_l%)-P}j-+QA7P{^fr?{Drqa{W9%$*-1m-OUXIF3v1sz;M4%Kq>3j| z-Ol;SE9?P3Kfilcvdts=Z25tm(`Su>h^f9zsqkn-@$iOarv z^~zw^f0Eo+POgn3SVW{bh-|o82}x!`-0O6AD3P1_*u`beu`%e=C-n2KI+wN3;JVEE z(>88?_rOLv!+R1aDngp-ZfE)1D7A6w;m<}W}?LEKRnsRM#YXj(mQRmsf&!5B{dLzjHVj?f1 z+p4(=ti~&k@?v|m%FR|TcZZT2E>Lhf=W5kZShGVvOJE!!Z|W<*q$S7lPVq+();Ipe zSN4@>%Vze{KdMv;tw$~2cjv`?P@3hP8ttG*Z5PK%BM!hGMfVc3N^`^F{^7LPry-N2 zzt(Hn;sZHQIA{Bh@rujbLk844aar9{=P+)#r@ zYp@n{S=!7rGT~jv()~JE9pLsD2|QSR88km#Wjhxe7^8TXMK_NMPjwYd!h5Dqe#ih0 z(bAWgN_4zF3rj^a!Sh62`io4gtXa%nj+4*QZRxWZ8(`~;s@&g0OII|1F6xx_m`RIj z?|8WdxNHFj1RhfwTkt)!A*#1q-35{TH-*Xo*I9*oK0e zR@SNm)>w28zIee^xy4jCQFb*g<2vg7z_u4;4$av9)2BPl97=5IhRT;YoEA%~(RV-6 zn+{4l5ZU`Yq<>^SoGIhr_ZfjxOr)BtTw_fnVOHug^U-qLRyyW9l{_eParWwBzk8|* z_pwx22?*KbR`eBDdrY^WEvIkQ^Djw@0< zS|3wGoouq3U#Q;0o|$V9gH>tAC;f@Kn!BzBXJ?PYN-q{JuD=5_UGeih!GZT4H*ia` zi({bGFu?-A)=X;*Dwdb(Ym_oYh&7hiyPEW=RKc@-Ih zr19oQ&8t4b1->sJ+bJwyoSB|!`T3KAl5(!mS?r0cw8iH~N>Mg?O%yMyqkX{%M@1JN zry=CJlBs2Xg?xB)6wj#lwuNyxOvLXa&Fr7A0Ufx@6*-fY)>jo@89xh4Z8>jGT%8>_ zY{HFJD{>kvYcPdtv0Y0!w;XoU)+Ul+#W;WMUg|3=7;&3h{W!&Xy8LW-nSp0}Ys&Xe zgu}FPbbDCl4(-4rC5x=wTOBZGf{m#Xr~CdPL0%irose8_(EJT&nxl-SQ1`?cdN0f3 zT8=5|G}-OPtnnC)=joDhI_v7rC}l`W3(xlv#mo50=PDO|!>=37k#XceiU@kO=&vd* z0pWfL67RmrBWk(WLY*wWpXk%XJgH=*p;TlP+$a#1k6|$FUpiqaN{^xo@%9f^+rx?4 z-!PpQ)$>Ze4%*RoY2u6p}M*;Uokn9#Jv-%W75w zw783-Ex)5JekYruJf<@SR_xG|pND5@AYR>?=7uV5q12_Xo2yXUcpzDR=;`90!D^)zm=(n^ zXiJ_yKqDO47;3PD^>n#te|D<-$rY>-YU8LR zg7&kaO;qW^GOrbP*PRp3Pqve3!iL5xc^xie(%y-{tNGC!r}yzs$n-$Xesknrce8MxnJzYbZmm%Td7?1#QJ0o;AR#e;_NjO+eG^AR?!sXcvQ11)*7e0uR0lj**XM^0&p7FRX@!Uoa3Y0!L0~$)FPNtCnlzm#Gs(Zlz`(G+-h0gEH~m;^YrD_s>Co)htJ!aA2knq32C0$n zckUuvd3oQ0NtwfttoFBJeWzXQBR|ncL{U-EK4PpVx*3j`DAGmR!siS?^|QA8akCy= z`iNqcx>s6ST7nlCA)%o-L>!hnO+e*nu=|b~LAIz>mixpN@?~+ZaNG3VN|OYR>!6%D zLUfgd>&?1UYIeudHN5FTAIUbA~K3@UjLHF$J2_UN&?lW{zYcCB~`g@K=c&#PEw zrFy3>1IT7coVC6|L*}-B{3(llh0$U>FF}BjmW|$GuV~xcuZUpYXb{`99sqAr(^ISB~~5q3KK0I9Ow-9Tm% zt*J?)<|mKmntJehhlhr+rZ$D0P^VIDBB&i{rZ*NhOie6;eFDKc$ziQA(AlX-AklD^}?`R9bD^kml-5*G=)E)^? z+7$Nxn8nGyHBphF)2UuCp;2#^hCqAp98LE6eH5ClDm` zKHroOeIFy#pf-+fjn({z({`!?Bf#EE(`jd#l&lESK5zwlxPRQ1;jr30)adM35^c$C zGuP-zrjH46w}Q(fi(P>Ga?1a74Wg=gn~o&k<^OB$DO)Y3prC|^OV(I;Am8t{p7EbQ zTMkzHewAuw7|lL>&WT$101}*mf%jF1Hb#ry#deWBr&5+}xP^Ru1sLnk+Yi%Ead7Dl zTb$&6IrzTzOKJ}(4ouhnEFiasju?hg7H)p{Qf>AW4U$M5IZ z*o!$iW*^Yu8cd#WOzv{`FAuF!q&TU>e`E6F9e7k)+48P=YDaT5<(>Vo#LhGNcwH9C zhgy)IC`ZPy&UNlZNB&By%ydg9s!DG1nyFCXFW0>|-i(4wxzb{Gj|c7gS`= zDF$`h?pw%&AM)llUYossOX)cN$=cpY&~bZV2s^vZ*&@3zYqZfBMLV%Kc}Mt?)B9!D zf`sDC<0;CE#3r-5jhQGb10gI&hik(@r2+lNZwzS)wM!H!l;_QC131vyHX&!4qqm5N zf3YTOYCNjy{@MF-Nwa@*LiL1lef#>>t*Y%=M|w4#g4Sk5ra%9bGd+EoK&_4~akQ9G zRdqT75wdur8QalXdC0Z>O>?Zr5wK@Ad7r$~!GXYDJVeX9Y=21}i7(WQq`Z#Af-&q&5;*lr8iSs&$t{e;Y{Vn{QonH1F#j#1TcLW}k$u7%j{& z>o=rHjiXPL@saY|ul$KzM36+roc-D0E)uLQ_q{wki+I-{FaH9QgYvHj=J&YX6g z#qD|+_wnPu5Pf%8E7klHZI{i;z9>y=Z!iaSOe_~({Eqb7)00o)Lm~66=?cn2U!O%( zN(Ep}hIcOPWyLLNKVi_zR0PUdtmP*ji?&~;7%@kb@Y{%(FG%QKCgSg(`Rt?C964{T zSYNgayPc*s(o&q_|5?#Kam_(`(>y!-s_^Sap1z;cI!S7lwx)1qq}70oR06+Y6wl*- zW<e-ls+J0O@y_R zE=AIuV1~y>i_!Tynz{^l5i6af`aK|PK*pv0&FMkKS8}0Osycz!%)UYr?k&xjxFU|m zLGs_6=4k`A?XAuZ&w99+5)IJk3)~p-fJ%N^^=wGR6;>TaH=k4T{g8Bthj~JD=rxc% zjfhclE?f_`=~~Vo2P7q`diyaZ=4pRRnL>EKh4zBnS@%(5y7&IVs3ndGBt9#w?K&k; zzp^OU1koC&Ce9E2GIh3m#qT&DrQzqJyen&s=ZvyJN2uZ2dqmy+utjr+CEjyGZ4Ce$w3EX#jiD@<+UC-QPn5nTh>(77%SZI%|izr6HYagg*-L$#FaKNk?+*wV^zb($hNz6Xh6TQ`zu4X z8A%MY<|Qye^`TUA4>@Vt+Y#Yji|4P1IT~00*$S6DEon^0A1N!2)u6S&rd}Pl|qf zciU-gY5(Wz0297v3Kv`LNLTlqoXR0(!{^?vL)=dv<6K%^{v+)nGT~^WQ2BYp;`s%# zm-W+6=q}4WG41bj-cycP{z`xF$vAsJx3$@+^X=>~MxJKFLl6Ft#|)vXg3c@NFK7HV zk-B=0v|R@qNwNeeevwhz+v9EkHIiMcxdsP?E&JVi#{mMhfJ7K=nqXpMXH*V=ywhHs zNtIkbC3NT&UvtH+P}9Si8puBQ0E*K_=terp0N7?Ux6g* zT5+vjp_5Gatni^*;^)t)vgGaC6fL@qvXSx{nwo@!gy{taHAZ{sry`s)Gc8DUTeDe$ zuPCr(E=O`SLf-lz_;W|EbY^SQZod`M1H%fUD@Um!3g7qBy#aWMy?^glas?KiLD8v( zorp`?GXRyuOBN(hWW?#n>FEi#3cy$R#4lQ%;&#Nqg2JO(4)Nkd-{<6f^AT98`8p!j ziuEbJ|9CK}QX;`Kc=v9im$~wj8ua_9I6e1SCM#^d)iNv%o#J?HRdi7bEpgu0*%o%A zx+h5Y2u(%htOQ0-`y9j46aoTst`>8$UUbt9tCzc7WiJTWRSR`)=|5Da03M(>hK@2% z>;l6_&u=~Aox`Q`g6V^Y4whGs@%`rsT`V%Oo{*4(LmD65N4u5FESs;BZ~ig@LfEJcbNeGmug%{GnNGIC z4uVYB`M9~IL$KBf$M4wp5GVGu`qb9M^RNR{6gHaK%J4Q%Vs%woMYk^H<&zGMo1YWX z$+5OH_f^LN``!B4^&%wQ_OP*uqE8vKsTszCV|kd<{T>B4FcJ}E6HGV7*N1(zj*Dsw zv6`%9;GxQV6-M8AYq?IbOzOuwgzoi7r=uPD5oWa0pzVvo>fo|;5CY}#AqT7~M#Lg-o1EMla8Hz1O|)3wLCID%H1~NBP7$v4$cz*~lNg&Z)wOqGPb1%qt4# z=noWct*&f&p6xG1WvBnh*Q#-$bl6JbEv5E}tFCq{d24YDf2&BFOjb70QEFNqD%T=n z1eWjN(nM+O28{xK)-hwCsLf1`u*=fx(O9=xsvA6kkkcq-%?x}b?W$MKB_)|zKI*{! z#y@y$Sg}HJ4Fw?DKCa{_+ebpLuSnm9k2=|Bp z1z5+hTc%b7lD4)EI9#oK1^?Pt4=++t9-sZt*%|(xJuoq{DkS2`gjGLo2>rJ!rfy)- zc<>`9H@Cump`u!a@)@Wq^NP;;kvhpNlH}qP}+G&46^? zsc76INO~9OWi1S=ZHyfI6%i3<0f|RNfeSW=$0(qxPrpp1i7;9_e7FekadL1_1<4QszSqocC&M!59} z4i_uytnaCYf9XpZAv_F`c!#k9^hZh$f0vu&o)WND>qq{)8>PSvvxx|rP$FI`*rS{_ zN|IFQZ4n-AjOf(3U{8u$5ScnVNk_NSCN!iiC4M1nviD}7Tl52YlyrBQ(voHh7{F@a zAM`htSnH*|jOZ6ke4Ln*9_iq%y5lTXw$0hIo9uj?-jtJChjq!ruYZ1hK)qdZ1Y8Q? zSiWw5Zp`m{7gFu>X-Q>^d%spG)_CmCr!`}n^OzDLQ^DXA4H^oT4X+_5$ngS?;g)TrG@l3=TeHq|^6yUu#S@axrfAA zTGIXeN*~)DvbMd%iQCN&G3~vhgJ|U4MPu@0V!rPK@Y2fe*TQ^Ew6S;HvfF&M@9Kc^LG>(AzJc7q(I^4l4Mp_ud_$82oHC6}6f@=!?-I)7l+n zFniu(_8yI7P+MQ0%>H@W?K3!db9}cQ*h#3=!Ho(8h$3SVBj|Ym7t>Ix*dTd|LW4n{ zq?6nU;ja5N&mD#!76a#_cBaEd0Cxqz)rz7~ZFdIaSvH*}2PN1AzABtM953gUG!KPW zeDG0D7y0-0@Xy?YTh#y&J6Vlew0%K?7~aQw=LSN5fQ3qZN^>^uyF4l1_9Cm23!c24 zE`T4}6_VeWz5bnUPI>tbBJlh!JCG#$9#mqVZxUwGtNUTnEm?=Rhdy7;!q)$ArqSh} zCxcN6GO|yS$?CVP3VabyI$6%qNzH8&#nqGR0y?7H*&{R|+l`~1-Q+n(K%k*Gc(Y2E zqn`ay;DUJwK0AOwxCjP}(WMsbou^zvyzXF=qNAd2e1bXs>B3SH4#=v-i(-cd8aFiminV`ei{-c< z!eAbuk7<|LuA(~Fl4MkSYr>70!kkb}E(Cb-{}hJAMDgIf2NlKNU;pP(^#3uh{=Xb@ z*MbryW!e49KmTo)7)t4*ag?N^UX!I8=x_hG>x_aTU3e6PHgx`bIJp0b*#Ex_)Bk_> zH>g$4<1n2s1y_8DPFlSRyiPPgK^6#5Tb3N2=h=DRsAZLq*J0`RXBgpuw5zJh+S;qX zvK4?|{jd52F%T__^l9GjJlG%hAWu3K>$ufFZ3+qbz?i6{tnU#XUb0}~;u3*lMMZh> zTF&7g)IQ6LDCBr=uUs_Dzm75$XVKhF#P$#bdxJLOaKly zg&En{lo`4IJwhMGc!-9UDCpa@^dKXXjN4?P{FZmEwrMXp7B$LUPB{}xZ$Uw5Ljhm* zI85};&3S6UaEpq9K_=YbIY8j;tQ-s+g8R)7gY9+_46DUOfWT^v7iv5ovWdF37Iy5U zmgR-eCXd(VQbq@yR_Qp(!Dw!ABII#7G9dg9nJT+Vw3l|1Y}J2f0q%~To)>>1NY2yF z&YU-X=b+v>glhn1a0-Xw4+}>-x1W#+X?@O)g{zK#s}v}&RA^}0;c6`^Q2F=ZJ8wnB z#l=Obu#_mU$bxJ;E9KAniVw`TuQS8Yf<;&I<42gE zgf3F~D0~(uv)(YNaWLxG2LgG^$_hMd+x)`l68bOPs==NWe_RN$W6`RCU``@0udCa) zkpj>ajXLJ%=N%t2w|F3lA(ez#L?&O4|GoQvf@fahDy*E~9 ze}I!gr}j0CRx+*OKyniYD_4KwQ*}FOX$T=jGlp3V;6HvWY}y>f_L7<*#+lD`ejn~; z>{YAE%kc;6rVb}iMum8IMp7Ew2%ItMMPq!&1g|+j8fjp-Sis2iUBIoi`hnrckUQ>R z|1$iD!NJ8f+vu25Rb0HGeFD&$r@K2Gpv_gAK4li*rcOS~(N|zxQ!`T4nrMQkq>Mr?|ZKeKiPGcC`ZvD$IM=M>h9n_x zKKo8VxDtRZQpvm301T!?t~{NRq{%&OHIyKPx=hwB9ycz9}-m|#evVyOz`O+{=F$mv>X17r9XALUqJ+e zc`(xL!Sr$F0{zLK=fgueBGwFYawAP(v+sNB=R;b%c8GSk%A^+w zlp;NDPlk`5;^}&Ywpn)C_WthBugbzNX$}X7Jr$>&RPKAu@i6GFuo*Jy`7{Hi8qZfz zgR?RPB$k&g;|ZcW(uNsrDRXhm53%5>KjJ( zZjq`KqxB7NU?RcX_T7wCY+1jus~&|*9Q_%!jPF{MO8e8NUw`{el1553N4G+N!>2&^ zz5P*I2S_pbe6|iL%(u;D_IikZx^uN+2Gcrek1aY5kkV9+oxrR1a#&1 zX9^C9MBF>@IE0dNVd^7bwS5!()~)6~kIih2daf5iNy58#LJJs1?(o1x#YkznL**Tr zBsvMq?4~S#C<+W!D~Z8a^IZ$e?N6=#WOt7|^C)tj#It0W6?2(usHY46gKjx)GyBrg zBpAtjLDnQHpc{ZXlYuzg84N_&XHMS-je*Y2!rb_yjq!#ChHdBVXzL3dqkZ0O>uAPb zWvPXHt{F@nu)MS3E;zLYp9D9Iil_&id$Ejq<(iA`AFFsy`()L`R7Ns5_>7+iFxD8naEH8txeMVh?R z_d?##&>$nj@Gogf<16C{Bnd#-Xi=#x3m#`=vGpIhh0YIkBqhdJce8lV?v7iqjNunBq zdPK}Jf%i(!tHO93lA|Z507oYS15o2yKAVQQm2=O@hQ_#@R7*@~y8qmJlQK<`l>B#fCg~&hLCqQh3+_gR= zZh!jv%|+3bn;RTN1_|6&E+t^d_ICKQU_PY}m(7Iz@?_t~BMb1C9?bd$xb6sx_eokA zzJ5(^8yc6a!Ntn;Y9<`6K^Cl+C`d@K3lEHc2nUyZ+Fr%**Xge)ufSedR6AdgKzG<#?fPB&}Fz0x6!Ut^nJIWgXDccQ?=`LtEM=?WtU}ykfnyrxOLg1n^Xt6(S5HPvIAu$t5>6GNh;F=4 zh2Ke-YBp{1#Nm~c=UEYWJL2h;UO2f?xnEM98~@ew4h|}25@BLuel`lb)!^GlIR04p z8x74q+b^{9BQRqe!|e%N-Y}-{x?(j<=bS)mpzrRvm;YjX@toIc(QQA!gO#oz7SfS4 z@MlxQr>|eI|INH5}bJ-0nD^Q$%jahBn7Q7{H#PtX}BtN){l^@ zo2fJ$wgam@(fnO~cGBF>I51njgGA|IZ_ly3N@|>j&pkZ@*I4h6Fz9jU%S=Ab{8%QH zha}05uHH0irhjktms1xL6_V+0*M8A^6Hq_@{c^L7K}J)>_j2j%M1bVWdo$MK%}M)- zZf7vc|89Yony$`FYXhP^=8?CZqPS9Wos*T{?M*UF$Q9p1ZU>jaP(*Z$6xYzj7RWYm zh$-kokRRd(%htW>gGH(B3tN_Ks?b{ss+W%<)A|Sq&bAB-@>>0GF7hedY6Dr~8PbJi z|GiFbPLz@gxv=RG-0D%gKW{nLlfwk+UeM_zS0^m(MFV}Z6(7`QnlU}sbnnN1!x~qY zNUdj#qAN)O2$qHu@5}+zKryy z{O#dXn?9s~@I_=bYE}lSA%$A!i`j41bylTD`^S3{YE!wY4+adJRunn`GUou zZmbVxA0ZYI0jn2{f^|LzL2!BG;CO=1W>FuTWy~cNFv*!mQ$t{JiR9s`UGLQgGq9nr zvt`;+_UN~3DUieQcV62jGUfYKmX6!E#F2!P3X2JnUaf}_jYJ|a4*DMXqX`JihU1^a z{}p42w1<<@pQvX%zEiO^ffCo=-a0uo6}-vlZlI`qU0(zB4>b;3?NC+dY%^yun+}bO z=8%s#-Wg>w{bXf9mZXIJEj@f@w!adD5^nxLy(vDUTE0xh?dFAzptQ1=_qZrW9G!|J zbUxS%_7hd@q$v>XA@3JM-$jl+tr+-rb11MQc_*jd2wGspGQ>qA6Hv3*&);0_JfdEXy<`+tfr2n+`oNh*PnlX zhK?P8h?$&(z-sH!yqWtb^4=FA(>HKa+{XA!yr~49(^Sek*1~IZKQPd*LScja+xUBsM>RWTgCBNte{Q*@i2_+nLE%_7m|HFlFO?k0CLT;z69 z3E6U+srl&|lzm+Q!`sdQ{L8V~S=~u#8^hXQfg*+CrR=wjbhNZ)W1BUc7NdR8C)ZfK z%A$coNJu|-AxND-o0axHPfIt6$GlWLCYVX18vm~IP9sv#POamrO~*@Js7-rodz01k zs&iwnPsrks=geeB=t?OKe92F8N04`O0G?3zd>TLL$~WZuQeD;?ItaANE&i&A8Y~|Z zX(s}XC+u~aV;%x1 z04T}fbNN}FMIY*$46h(e%dV-7LBfX*Cki|RLpVj>Jy;DiCb34hOCq4ix9j!Kk7z_j zT~WMyx^s?Zh5Aj*!P|=+3g{$!)0N&Giyi2Xnf7^`%1TQGUn4eYYY0 zKkR%I^gJ#xmuez?xZ0;(b@N5Q8RESbuHLI@qX4VKxL-nB`a=>PU}_@sk2ozyem5Fs z*VXZ}8I;_+6`+xOE`EulxNp+`<>LA(05B_~{-g)o`L7Cfs%*A~UoHdP_cVUWNS1A= zcz$D5Jx%lskk8!04lLW9*((4a`@f8qKoLNOBqT+zRmgcyeu>FH^KP6dr2%@kk>82B7_CWz_5wkE2_h<8wn9Jzi^Az45|{crbJ zB8N@P5~-rQ+aaPv;7br~EQmYnQC9>cwPc{$X6e~4CD1J8k6_gf_0 zg`BU3XMfTt&Cs~46pM8x*zWJRY|T<)U1@=wc5A9aTPDt*jw5)7u~BLwj(_tRQvz%}7k{FZ*3$ zVVmA@uKTDDe~yDuJ+3-P56y}U6O+~kij&LDMrokfxYaNAuMA~U_N6fchxxRk5N9I% z#Ik<9h+$DbUODQiuo%;*vk`;k?(0`F17k1I%kj$bp|QwowBzIBi<2!pVM<9<@L#A^ zS|$7o`13yN9?etB|09=f;J?2bd%!YV>rQkM+;_Y=qh1+`-N-w*qnmBB)o^j#(b_64 z-lQn%CD6TEK=5mNdRp!2qv$BXavCbC&YpBpl@jF&cGCgt>F<>u6?p~?Inc#E@&|e^ zPRMvDFP=Y;ijSNu`umjShE1qaK5qNhhwfd5{%BrozG|)j&`7}FrpHemTwOqT%XMu! zSN}#txAus~bGv@MIT|s}@4d45>1*A1fnHj6_DO~Hq5XKgli2ybH7U`>Sv~>h)cdvy+P+!&3*yQdi9sU6rB5#eeYJd<*m|=N`UE;HlS#I zwJ~msii!&Fxa?ZCF8C;J1ZmE`_rp<-{LJdz9b_0lNkO4xX+4$v3w6lX{9vUJDxslR zq}kQxooH~hBJ|r2=E-}GTC*8-V&dRX&9xr$+y~WrNMfN@{nzT+5`rbkAcK&Y`_QypdS```_9 zb7^r}zyqQKq!F7*uNz|UO^l3vFq8?^fR7a%0pxT8CNVf=pSDE38a>Bd-0c)sdJ_f{ znXs-NpPy_jA`s6!Ht&duiNR7MW(l5$CC+qFc9&jSWaJAGv5&Ac^#Kr;17GXNi*jWSQ}Ee6GB5q>-pp7O5U{!ZIW5UhtD@5?zb}f$`a(}x z&z2fW>|xNm@GTPh^H@G}PmPRZs_*#Zaein;MRg=;X)58B~^|qoe;` zSs%583fZ;ctmwdN%F}!C@oiQgCrXXQQdk{=!COS$sZnYsLXbU7ThbyKO2}sRHgV*wxFkzSa`a{Tz@2G2e_MuzBU@PMQK%R z7`j7s%c-hDA|G!;wu`H)?QprjhGhIoqI%M5O5@*dr-lo3)jSOmJ(V=CGmt(22bn#P zbaZ@dGn8JlE(Y{k!|VA4-8z?(t&HmRV6C7AxA{=oqTMU1&hgL3Whl|#~n`~%70e+Q9tNie5KtMwSiGv<%3T>3;2ND?c$#m!6 zV%9F@2`fcPqOEv<5q=*H4Z1*V{`SCje#g4Hhl-AZvN2mI7o~3J>Oe1OwEF{oOlt5p z(iox`jxV+n5CDc8dMSu`zBh@-eEFY>xQdwDyQ9j59$TV!QLN@Fc@qen{&PG+G#&km z7=qi&tYcoDs5hpTb#6nKJ?cg1gaWGnxwlLg{(KLaSTWxO&4^gD%*6TX;_~KyH+K)6 zhU8aVUbxTv_x~UdZ(tosd{>xJR``2Oi`TrkU8s?7MkG5QG2*ALBN6P%KZ}v(HE^7}b+eX16*ggEH z=jvX#7ThA?!S$?Y7BAE>86>i=GDDCx4qCRRy>@r^ph1Yg!jjs&5fU;92YB-Mf-k0> zz|#*lz%4CKNFV%7mfK9P4^}QiOkcm=gYs{uLBnO7kjMTeq$43SgU@b1dDzr$5LO@Y zg@o595AtYGez3Q$5dqE00&jS(ur9Kwzoz^wZeza1;k-us9TK zE+>N9m3XkO4w|a!X+W$4io_r?K#S%oja6jM%(a@#J)`)^STj>V_;yGF!!HTC^^r!- zxE+}A4)xSQVkwV^SinI5^uJq55)W}Z<3>fSmw~;SEH{e+UZ&zGMN~|^VD_2O^Bu^Y zPED9-*1g&2Ohd}_#eLWUd7!7Eq`^OT`xmc(qJZJbzL@9pHIJ+IWw5#fD#IMX5`O3h z_lZAM7Ok)QJA2a_0TxW7i0D^ zL8ZOPW2@A2?#bv8c-WyvJaL_HH9$eH!6&*G<(e>t0^4vAu96KWgvtM0U4+AA1FF_0 zD=p|7(LcaMvBGMM7Od$N0T5tZ*^6EcaE#_3Unl&1 zrVlycKKi-0_xP-zMIWl_1XT4>%oXUAOn;y&>yBa0&B*u~5n=adcMnX6(oOMjGyGS% ztO4S^pM6^-W00Wf9;0EO+=bJWe7*O(OLvHOg&kjl8)pzfwJWkU5eF5Rwj!Wp`N(7En({XfJs&;!GrP%xyLlJ3)8B0x z1!^)?F|vSho&s^cSx^w?DF#$sS3#NOK9{|;>!SsU7At7{<;;8K@V%;qsM>tvWr>Km zcxG&@F|d^*E<*O}RsuLE??jq=UKPn*Zl;F@8Q+QgU8v!6zUfaZkvABT$o8gu{Y$iI ze_f-q9I>5OA2{cszU*ppnQ1cmK2Iju2T3bcqZkbrXGhoXRik6)#165{YV}&n4nv0^igi=iHzp6(s|f!VW7N@^L$^yB6> zk@F>{&wpnD+FDwv6X9cY*G29j>f#@y0<-@EA$}ip-IYE=q7|2vc=BR%u8t_q(PV(j zcts#a&D&crMq%?HBR<{@P1Ifg=ZJ*zwM7tVuriZ{`PN{*_Aw8K9`8F z0P=C7-&FyJi><#NoH~)?^{&5Wv3W(iqncI_?Zraz^fG~mEzso}@)afbHf-k_!hPd@ z-)kFa`8xrVvM*C?_BxfOxp^7P1gPXhp-Np3tR^+9NpyxEBNS5hSNr@9ku@tVOagq3 z&CEWpZ?I@(zW!UipsyHaW(>rq$d7mL1-*|&-IoClgv4oTcK%Zv*qY*yVMEvRlq=bv z6{u|hTx)4@BRfCwu*&H5>j;&#M`{eQk3q*=|2{l)^5U)UU_kh6=dCt*UcJjMVU@pc zs`O%rlpqsYA!H7AFWHZ=cszgp2OfsOR|$1? z6Tl;)G0!020PTN60m_&jnY2Tx@Kj)$ilNp&QriCh#st=5o{REu-=jEMk1f!TcKv z&H2`Jv%3e+(5bJvjFySXH?;Nt)!upkWBtE>zam80vS%`jl)d-fq$nd>cymhT8M3#m zvSk!SA+lE@TTZfPr>v08X`a{N`~AK@x9^{DUDvJG53g<&C+B#*p2zbz9*@U;n6h_v zuAuMm;J_0mcYaqZMEWHa(tr6*)9!rk@~2LtkZx@JkY2RN?=?*~#B+_D$9#AhxFPnr z)xKW8S5~4BxIok)^_3MAe!O(fS7M?>ihA#s+p4vIyXqbwY9O6{GcYJfVcLJED_x=h zIRY|^DC(Q}bj{a8DZp{; z0H5ntnGk&@NxF!%Hu^Y~KTO5XO|$`AMb0?pRN)V3A$GCAuYJR|K-^2%YKqomSMg|L z`2?+OwJvNM2Bm=F$`tTKfV-~j+l-8Cv|CG2%yGhp*JH(dwvZ0|Gmg9)qf7Ed9ffiy5@a&$38UEToHNf~d~j+G_I=nA33l$w5ld@lNQdQF zf|v9?{qX~Tj+omOhV+gC>tBtQC4_^~(MI^-QD zCtqRR`3bri_vxNa*qKCR7ecx|z&U0P`p$_Po|zW2J6PzLm>U}Kch@4mk?XU9(gM7j z3S0qZn=p5quC8T8%khqjD}=&Y6?vY{o|LUgm17E%cHv-4l#4mU_EHOr*sSc))yxWz zj_92JbW_9liHnbe&|sl2S;4s>D{C6O?qvQf8uAnbD3WQ9d|`_A*k*-R;au9#i~P*Y zAY#Ki#9Pu)*5_Eqji3>h?F-oo4QCZ)$K@?!$?iy$%%{)2rI}s6(K7ECLbyQPF!-uu zx>?5XG%;Q1W62>`u%qSq57Lsyu(B`mn={?nSxdJl6xl@Na@xb*k={Pk54#`;@w6 zq*=|PYdT)`${ng~Ft?J7SHP~Ln_q}av|9USYsCDfvR0PBz4{5bFk$gbW==)?+f5B$3FC0JHQ;Zamtb<&sbSSW@x0WM_DQ_LW$;2t4mf1IG}=#u z^1uxlu0D@#1LPCv?LFrf7#VINE`z-4n~a7Zp#Y{2Y+Ype$5mC2r$vkMPMUV@fEGoj zXL%;qi{sggkpMz6+e0Slo62MwUBXoF1IA6p{m!3XT2`{58$t+{?JzL-}SsV4oDC$?#q(SqF$*b&#Lu zpQ?x1f34p(aSP=T1(!8U%vUDLyTQ!gw3tcOUrV9&&?zWiSwD7y+A~x136tD?ENkBH zp?A+cTHv!94^T66+LQLTE^Ve@fTko3; zqgEXVMsr@B`BX<|;YUn~S1DYh;3cB&38OB);S#5NS(gPuY5)KeNc=NAT$M5u0@Ay# zpZ|ac`qyjjaC_Sy{pTUbrfPk1acyPgaF|hx#k;?*ztE5I^T_(o1Gx-`KqDg{kl6N` z50p7;hcsc-&J)d4mRN8D`l zw`plJWb*088>h}%4PIZgCeRa=28o#o=nTLqSofe>_Qivfq$V!k8w)!G`f4iw;v?FtOGI+-;A0XW%? zH=S?Rgk<+f9BiNr%w_z8-BUln{B^W_>gzUqgd^HXGn>3FhREq>(XtQE^0K7o1&NNS z0E6o=ZKdkZ?0QQ9y*WKxXsVa)l}t#2+z=|U_dZm(x0%4$`&Yh@B~Ol1 zQL=3DJA)!|V)wAvELj0v#)gmp8?dl~iPTUkTSGSfsOnVntin^jk&MT%RM8qYu3o`p z-slnU5=%2lqBX;g&r;~KgW)zIUQ$y|-0C^dD6F5ZSwzkp{KV*e?OzKug6M6P2O=(? z9;y)YG%{KcY=NTqP z-Z=lBz0>PQvvVtswzeKh-s7=5Zxom^tN6{E$Q1BeqXX>qVG-l(GY*!tCn0u?r_ ze}ASuRkI|?A5qY4-?tyw*iL1!SFUtC@g}_QB6x2c&#?Y`(!w!z1> zYl6#`0X1+c+0xccua*&io3q87M>sqphH47sG=JfoF$*OvhbjK93A@5UGdn3~tm^_J!}<`cSVdK%^Ge=P2T&H>LRj%Z%Wp z`7ukLgW#GckjtQW1SEa{`}US9#XnHPB!AnjefvN#yD`o7N>CT?D(125+j{2_X~>fC@(w!M75Ch{ z{i&sxc<-~L(H$Mf&<-5t-x*_AnJ6B#q?1bqM{i8jy^p3%*Ogsa`sI31Z>;E#QxAO( zG0fl}q{`O9y^r~{bUcsD_MiXG{$HCgwT6O1jW&WJUGCLG*y8(~qIh|^ zC8oQRxo>LMDwprg{+x@9ko?-x=D-i+D2gZNEy_V}Y*R#;hZCqEuUM{f8DF*t$M->* ztpd1^lpatwAT1k}p|#><%-D2*PMoYMeKsOp@nmObcWa?9B(fw|LMoH$>a_oH3M-J+ zX#C1iftCHBNSdy0WV~HkCHe^eNTrvayC+(Vh|F8kck?%hs=C zc6li@>j14;>b{#Er0|q7;!Vgwh64pkP-#Y@auxOF+Tlx7rID}NjEscr?e!j!{RrDT zbdg>&{Vj6a7Q^*zsl^{QMbLbLhKjXicGyQltgQBaudfJOs$yG@Mg|8VXhUBAYED42 z>@sgZ2g$*X+Mgwl(sC-!xzSPUzJyLXF`rP6z<+xIL5+>VCb|Ic+M-oH8+uBF_vawT z1~@pDiKux3D)#|SL*{)PWUSq$>N{GqA0VoRyCKqbsjbDP=ml;L>xV-wTvD#0&UIQc z^6|AT!?&t@3w-y5ZChCo#8JPwc`z}S5pnD$<*Ic9G_iXu$ z^|TPm7assUdIavPcAYH7(W<&*mo}!>4wjLsemLd(Efzpi|? z?}w}=rGm_*aN2Mnc!)ju!#2GQiFIpIUajrz{Br6Whg;Gt*9spLiPQB!^GWsdMl%3O z)MN;pSHT0ACFWmdGpGy&uzd6LWT@An8(M5uMXM7N_kB{Sn}Qq-&9CUU62u8xJ73R& z2_A%9Y=s08O}&7s_35xl3jF)HY;X@LGR;dn=H%SgMmwt{1vDMPcH7xR7}kNSk13 zmE#1&)=A5QtT?q%8FaMF>OVjyG0nU9pv?|{|MY)v(PU(>$&MoLJ}45zo6nO+&%g$E zw!7L#Fr&XcJY*@w{@^yQ`0FAX=?^D1PEENrg~cklQS>oka?gA@ra3E@0VSDU*IsGr zi#p+Zds&ufyIZ1QM0WkEI_{5G+PO4Jt+v#Hy=cnW6@sWB;-26D7ggF41IN zMiFa!3l_SGaW1jVOXj}U?FruX5nO9p^h(-$6@TD1MgjBNSijYBYCfGb@j9LFe2{rU zD_zZe*nnyPM?&Jb7&oSAu0(ZA>X)RD^O zmW+r;#EQbIylnE-cY;Lz%S^8ZNjh0c zl#*T0P_qtkOA4CBb0MI6p5Qj1D1bx2#yM1@y0NhHdzj3hgDnA6cM+aZ$!uzyBm|h$ z?p8L80b^qh-Xp?D!U4i0h9Nlxf>P+{aBU3uQeWw?&MvbGfx# zkh!K!WWB;Y74Y=PK25L2h$Q3Jg0f+oCAR53Wh)+;3(j z+8l;Bjq5fZlCO34pW(-ahGDV!Z5!d=N6xg3YFSiNdNGf^!d8ve)i;n~<_G_MOOojDl1H;BAxGI+RX1R{F zpkpj#vt%{TnuIZk{b@v0{L}1cv16VaoNhs%B-uh zVb+k5F(Y;aB9z#%rrVK7EQ*}#R_x@9BN!Qjp(H=_^ZEf8Z z?R^bgSO&s`xpgr5mb~!P6Ra&NzI7A@-*d*yb288a3HzDMIbZr4dqu=2U`rRDHmeNm6wvN^OR=tisBbLSmfCEkjYxg=Z@-KgJvJR&Pbc84R&D3+`lD=y4o z^f29aG3#Whkot<7af7(fH|;dDyvOtGi*;@7_%aZUq+R61+1}oM?euGb!owJpz(jopz?z-@V@&%a-=dDJobwUb=MZN$gQ6 zP$uv=oi$(fD^-aHW(J#HVpcaghI~40x!OP3b0hJyW&JtnQ;ghCoqA(o6@0{-%gA$} z+OX?=Fj`}evJ_!8E0L~yeIhTzt1EVfS{A?L`)FE*oMd$a%ae; zf@ijy32icAz7-lx7dI?Q$AlI~!gPGR0ZYpBsjw~bA8U6dkN=3SQFG%>_o^UjWKG8X zM@Okc=;dX-ImME+;dgDyx@TNlT-=(Fc0tWof(Tv@F(*9_bd3tBn zr?PSVn^mKHP?xTM`^Yw`NUk>@miVN@X$Is}lzsJkgzTK2K+F6bE z-vI=llhdbq#IvK%p))eN!|#68#ykM$|NS?XVs{J-D1(@NFrhakn56tdUtjUp0&(^~ zlZ{n2NB4W^|EW8L1dq^3rw#jg7E(%aAH+C+Vd^!jMJvf#FDW^Ueew_F(_d1jpA zbCb}Sh>Ckop}1E%FSJ|;mP>dU+@q}@`*R6^T%gFIF4#0_U+4h zJNLMBUMqmmeV>*34hiH-mkfFn>BOJ!?Try(u#=Q_%E+B9GpKJM8!;Y1dGp~$dVvtD zwbOLPc|sidD(lmW-foKz>kz!^rZZWl{aKQT!&R79wq3G$udBPMU0dB~`q}SEa^yU0 zU4r89P8kFrwk2yN24AfkU&`<2x64EB?ES;VP5yOsbQJF47K?0ViszZc+|SW96X>@+ zC;A>3N0KYGdc_6@I|JPx0DSv)^?2unkt8g7>~53XuW-5x7gzU7BU}bk@e1X}Ae`4J zoTD-H8R2vW2bL`2qM`Iwi7`LD=ToA9MJKG^PeOu=z(bFEgWvYj@*TzRs?IO&`PtR;a+i?5@qtSyxjy<8 z;umHaW?ij?Y)NHa0>%8|beDuK#A>fSV3zUT8yH#vi;JlJs+22kU5iA$W$!oG%64Q6 zAxBt3Rgn%Uv$Ld<7o`K-m~w;z=N7S=zfhdtGx+rjWp!N%2Ctm;IfP7>E+ps=;PcOD z(tc?4j*%5wN+<8NxQnKw5`Z~vMu@`UzBt%$vzz>3;nrJJNaVl8MPRx!;~yVS87P@+ z|5JYE32KnN0lR^~=2#iuwv$J@9`A^`N9Gz(uXRf7XP_(@1oD~bXdskG|KX>db!t)B z>|;<4P{g&++;Zwc*9_hrd94-rM}6l!IBJU3f@*=O4s>kf;p&@eq~_S|H$3;ajt*m1 z;Ozs)$8+1aG=i*7lMu9W@F|vf=Be|=9hW=_pRMZnt2{x{rvidJ{kh!Wp6cV>!_Z<2h^C zx|2CHn&W#)*g#--DP+1@x=lWR<^$&F&>r;f-3OKyNdf)uAJ1=LTwWSbR=Qb!yrMRC zs%Yi`l;7y(!6<=?Q0>0pf0_CV_E_uY)ySKjMH;iS!p@a!j$A8->bvB3?6dQNA+AFs zn!Lq8&~fry_PfaWuGw9GF=^FK{-$A&3wwQ`emc4wXHp3aCjrtdGHgrd>FWK8+_JsJ!%v4 zGY~lDgb|#yZH~IrZ2x@At#heBcg=-#3jP05B8pg=nGyr5sc3E3QBb;LbX8wnebc~jXV+b?l52g|MmiYV26b@Mb1Q9rw=3l@HJ3=XVypJs(2|{D_UEtFAARh zAUKl54-PNp9tmp^?$5r(^m|H4O{FZ||DPP_@KaTkKv|Oou1X|617v1xUbT7UmM3_*0ho;v5I~{I^yLG~ z5NYO5lt$T4UY*BhXJ-%hWXE#}Lwnv}WB&IY(id@dc9xPdeI{ip)$o0es-mJIAKT~8 z;7a=6zdG58WS>6>TK?;x`w!Za%s~?GU&&qmCe6QJ9skX$`TzPa-jVTG4ZuBj?wo8t zc4UpfJ@ni;-#{Gr`zQqioSU;|;e2in7az{0oT6|Jkos~S{#d<43}*`+S2+LumcOsq z-w({+>*eq5V)pEA`1fuJFT&r&0$zl_Yvtck;cun%w^00BC;z{_O{kG`Hip~zyqcn% t;s5X3AGl* !a.startsWith('--')) +const inspectDatabase = args.includes('--database') function walk(dir, exts, skip = /node_modules|dist|\.test\.|routeTree\.gen/) { const out = [] @@ -66,7 +67,7 @@ const unused = [...src.entries()] // 3) cost basis per aggregation SQL (total_cost = official list price, actual_cost = rate-multiplied) const repo = readFileSync(join(ROOT, 'src/lib/db/repos/usage-logs-repo.mjs'), 'utf8') const basis = [] -const methodRe = /^ {2}(_?[a-zA-Z]+)\([^)]*\)\s*\{/gm +const methodRe = /^ {2}(?:async )?(_?[a-zA-Z]+)\([^)]*\)\s*\{/gm const starts = [...repo.matchAll(methodRe)].map((m) => ({ name: m[1], at: m.index })) starts.forEach((s, i) => { const body = repo.slice(s.at, starts[i + 1]?.at ?? repo.length) @@ -95,44 +96,50 @@ const backfillRepricesUnpriced = !bf || !/pricing_model IS NULL/.test(bf[2]) // 5) optional DB checks let db = null -if (dbPath) { - if (!existsSync(dbPath)) { - console.error(`db not found: ${dbPath}`) - process.exit(2) - } - const { DatabaseSync } = await import('node:sqlite') - const d = new DatabaseSync(dbPath, { readOnly: true }) - const colNames = d - .prepare("SELECT name FROM pragma_table_info('usage_logs')") - .all() - .map((r) => r.name) - const q = (sql) => d.prepare(sql).all() - db = { - has_service_tier_col: colNames.includes('service_tier'), - has_speed_col: colNames.includes('speed'), - unpriced_rows: q("SELECT COUNT(*) n FROM usage_logs WHERE pricing_model = 'unpriced'")[0], - band_distribution: colNames.includes('service_tier') - ? q(`SELECT service_tier, speed, long_context, COUNT(*) n, ROUND(SUM(COALESCE(total_cost,0)),4) cost +if (inspectDatabase) { + const d = await openAuditDb() + try { + const colNames = ( + await d + .prepare( + "SELECT column_name AS name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'usage_logs'", + ) + .all() + ).map((r) => r.name) + const q = (sql) => d.prepare(sql).all() + db = { + has_service_tier_col: colNames.includes('service_tier'), + has_speed_col: colNames.includes('speed'), + unpriced_rows: (await q("SELECT COUNT(*) n FROM usage_logs WHERE pricing_model = 'unpriced'"))[0], + band_distribution: colNames.includes('service_tier') + ? await q(`SELECT service_tier, speed, long_context, COUNT(*) n, ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) cost FROM usage_logs GROUP BY 1, 2, 3 ORDER BY 4 DESC LIMIT 30`) - : null, - totals: q(`SELECT COUNT(*) n, - SUM(total_cost IS NULL) null_total, - SUM(actual_cost IS NULL AND total_cost IS NOT NULL) null_actual, - ROUND(SUM(COALESCE(total_cost,0)),4) sum_total, - ROUND(SUM(COALESCE(actual_cost,total_cost,0)),4) sum_actual, - SUM(COALESCE(rate_multiplier,1) <> 1) rated_rows - FROM usage_logs`)[0], - unpriced_with_tokens: q(`SELECT COALESCE(upstream_model,model,requested_model) m, COUNT(*) n + : null, + totals: ( + await q(`SELECT COUNT(*) n, + COUNT(*) FILTER (WHERE total_cost IS NULL) null_total, + COUNT(*) FILTER (WHERE actual_cost IS NULL AND total_cost IS NOT NULL) null_actual, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) sum_total, + ROUND((SUM(COALESCE(actual_cost,total_cost,0)))::numeric,4) sum_actual, + COUNT(*) FILTER (WHERE COALESCE(rate_multiplier,1) <> 1) rated_rows + FROM usage_logs`) + )[0], + unpriced_with_tokens: await q(`SELECT COALESCE(upstream_model,model,requested_model) m, COUNT(*) n FROM usage_logs WHERE (total_cost IS NULL OR total_cost = 0) AND COALESCE(input_tokens,0)+COALESCE(output_tokens,0) > 0 GROUP BY 1 ORDER BY 2 DESC LIMIT 20`), - pricing_model: q(`SELECT pricing_model, COUNT(*) n, ROUND(SUM(total_cost),4) cost + pricing_model: await q(`SELECT pricing_model, COUNT(*) n, ROUND((SUM(total_cost))::numeric,4) cost FROM usage_logs GROUP BY 1 ORDER BY 2 DESC LIMIT 30`), - long_context_candidates: q(`SELECT pricing_model, COUNT(*) n FROM usage_logs + long_context_candidates: await q(`SELECT pricing_model, COUNT(*) n FROM usage_logs WHERE COALESCE(input_tokens,0)+COALESCE(cache_read_tokens,0)+COALESCE(cache_creation_tokens,0) > 200000 GROUP BY 1 ORDER BY 2 DESC`), - total_vs_parts_mismatch: q(`SELECT COUNT(*) n FROM usage_logs WHERE total_cost IS NOT NULL - AND ABS(total_cost - (COALESCE(input_cost,0)+COALESCE(output_cost,0)+COALESCE(cache_read_cost,0)+COALESCE(cache_creation_cost,0))) > 1e-6`)[0], + total_vs_parts_mismatch: ( + await q(`SELECT COUNT(*) n FROM usage_logs WHERE total_cost IS NOT NULL + AND ABS(total_cost - (COALESCE(input_cost,0)+COALESCE(output_cost,0)+COALESCE(cache_read_cost,0)+COALESCE(cache_creation_cost,0))) > 1e-6`) + )[0], + } + } finally { + await d.close() } } @@ -170,5 +177,5 @@ if (asJson) { if (db) { console.log('\n== 5. DB ==') console.log(JSON.stringify(db, null, 2)) - } else console.log('\n(未给 DB 路径，跳过数据检查)') + } else console.log('\n(未传 --database，跳过数据检查)') } diff --git a/scripts/audit-billing-gaps.mjs b/scripts/audit-billing-gaps.mjs index c54f2272a..4d4f7dd44 100644 --- a/scripts/audit-billing-gaps.mjs +++ b/scripts/audit-billing-gaps.mjs @@ -1,11 +1,12 @@ -import { DatabaseSync } from 'node:sqlite' +import { openAuditDb } from './audit-db.mjs' -const db = new DatabaseSync(process.argv[2] || '/opt/kin-gateway/data/kin.db', { readOnly: true }) +const db = await openAuditDb() const rows = (sql) => db.prepare(sql).all() const one = (sql) => db.prepare(sql).get() -const out = { - totals: one(` +try { + const out = { + totals: await one(` SELECT COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, SUM(CASE WHEN input_tokens IS NULL THEN 1 ELSE 0 END) AS null_in, @@ -14,80 +15,82 @@ const out = { SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, SUM(CASE WHEN status BETWEEN 200 AND 299 AND (total_cost IS NULL OR total_cost=0) AND COALESCE(input_tokens,0)+COALESCE(output_tokens,0)>0 THEN 1 ELSE 0 END) AS ok_tokens_zero_cost, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost FROM usage_logs `), - protocol: rows(` + protocol: await rows(` SELECT protocol, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, SUM(CASE WHEN total_cost IS NULL THEN 1 ELSE 0 END) AS null_cost, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost, SUM(COALESCE(input_tokens,0)) AS in_tok, SUM(COALESCE(output_tokens,0)) AS out_tok, SUM(COALESCE(cache_read_tokens,0)) AS cache_read, SUM(COALESCE(cache_creation_tokens,0)) AS cache_create FROM usage_logs GROUP BY protocol ORDER BY n DESC `), - ua: rows(` + ua: await rows(` SELECT CASE - WHEN user_agent LIKE '%kin-console-test%' THEN 'vm-test' - WHEN user_agent LIKE '%claude-cli%' OR user_agent LIKE '%claude-code%' - OR user_agent LIKE '%Claude-Code%' OR user_agent LIKE '%Claude Code%' THEN 'official-cc' - WHEN user_agent LIKE '%Rikka%' THEN 'rikka' - WHEN user_agent LIKE '%Hermes%' THEN 'hermes' - WHEN user_agent LIKE '%Cherry%' THEN 'cherry' - WHEN user_agent LIKE '%Go-http-client%' THEN 'go-http' - WHEN user_agent LIKE '%OpenAI%' OR user_agent LIKE '%openai%' THEN 'openai-sdk' + WHEN user_agent ILIKE '%kin-console-test%' THEN 'vm-test' + WHEN user_agent ILIKE '%claude-cli%' OR user_agent ILIKE '%claude-code%' + OR user_agent ILIKE '%Claude-Code%' OR user_agent ILIKE '%Claude Code%' THEN 'official-cc' + WHEN user_agent ILIKE '%Rikka%' THEN 'rikka' + WHEN user_agent ILIKE '%Hermes%' THEN 'hermes' + WHEN user_agent ILIKE '%Cherry%' THEN 'cherry' + WHEN user_agent ILIKE '%Go-http-client%' THEN 'go-http' + WHEN user_agent ILIKE '%OpenAI%' OR user_agent ILIKE '%openai%' THEN 'openai-sdk' WHEN user_agent IS NULL OR user_agent = '' THEN 'empty' ELSE 'other-third' END AS ua_class, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost FROM usage_logs GROUP BY ua_class ORDER BY n DESC `), - path: rows('SELECT path, COUNT(*) AS n FROM usage_logs GROUP BY path ORDER BY n DESC LIMIT 20'), - other_ua: rows(` + path: await rows('SELECT path, COUNT(*) AS n FROM usage_logs GROUP BY path ORDER BY n DESC LIMIT 20'), + other_ua: await rows(` SELECT substr(user_agent,1,90) AS ua, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 AND input_tokens IS NULL THEN 1 ELSE 0 END) AS ok_null_in, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost FROM usage_logs - WHERE user_agent NOT LIKE '%kin-console-test%' - AND user_agent NOT LIKE '%claude-cli%' - AND user_agent NOT LIKE '%claude-code%' - AND user_agent NOT LIKE '%Claude-Code%' - AND user_agent NOT LIKE '%Claude Code%' - AND user_agent NOT LIKE '%Rikka%' - AND user_agent NOT LIKE '%Hermes%' - AND user_agent NOT LIKE '%Cherry%' + WHERE user_agent NOT ILIKE '%kin-console-test%' + AND user_agent NOT ILIKE '%claude-cli%' + AND user_agent NOT ILIKE '%claude-code%' + AND user_agent NOT ILIKE '%Claude-Code%' + AND user_agent NOT ILIKE '%Claude Code%' + AND user_agent NOT ILIKE '%Rikka%' + AND user_agent NOT ILIKE '%Hermes%' + AND user_agent NOT ILIKE '%Cherry%' GROUP BY ua ORDER BY n DESC LIMIT 15 `), - ok_null_in_sample: rows(` + ok_null_in_sample: await rows(` SELECT protocol, status, stream, path, substr(user_agent,1,60) AS ua, model, error_code, stop_reason, final_state, via FROM usage_logs WHERE status BETWEEN 200 AND 299 AND input_tokens IS NULL ORDER BY created_at DESC LIMIT 20 `), - today_by_ua: rows(` + today_by_ua: await rows(` SELECT CASE - WHEN user_agent LIKE '%kin-console-test%' THEN 'vm-test' - WHEN user_agent LIKE '%claude-cli%' OR user_agent LIKE '%claude-code%' - OR user_agent LIKE '%Claude-Code%' OR user_agent LIKE '%Claude Code%' THEN 'official-cc' + WHEN user_agent ILIKE '%kin-console-test%' THEN 'vm-test' + WHEN user_agent ILIKE '%claude-cli%' OR user_agent ILIKE '%claude-code%' + OR user_agent ILIKE '%Claude-Code%' OR user_agent ILIKE '%Claude Code%' THEN 'official-cc' ELSE 'third-or-other' END AS ua_class, COUNT(*) AS n, SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END) AS ok2xx, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost, SUM(COALESCE(input_tokens,0)) AS in_tok, SUM(COALESCE(output_tokens,0)) AS out_tok FROM usage_logs - WHERE created_at >= datetime('now', '+8 hours', 'start of day', '-8 hours') + WHERE created_at >= to_char(date_trunc('day', now() AT TIME ZONE 'Asia/Shanghai') AT TIME ZONE 'Asia/Shanghai' AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') GROUP BY ua_class `), -} + } -console.log(JSON.stringify(out, null, 2)) -db.close() + console.log(JSON.stringify(out, null, 2)) +} finally { + await db.close() +} diff --git a/scripts/audit-billing-openai.mjs b/scripts/audit-billing-openai.mjs index 1b5ca254e..5c5d154e5 100644 --- a/scripts/audit-billing-openai.mjs +++ b/scripts/audit-billing-openai.mjs @@ -1,51 +1,54 @@ -import { DatabaseSync } from 'node:sqlite' +import { openAuditDb } from './audit-db.mjs' -const db = new DatabaseSync(process.argv[2] || '/opt/kin-gateway/data/kin.db', { readOnly: true }) +const db = await openAuditDb() const rows = (sql) => db.prepare(sql).all() -const out = { - openai_chat: rows(` +try { + const out = { + openai_chat: await rows(` SELECT stream, status, final_state, error_code, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, - ROUND(total_cost,6) AS cost, substr(user_agent,1,50) AS ua, model, stop_reason + ROUND((total_cost)::numeric,6) AS cost, substr(user_agent,1,50) AS ua, model, stop_reason FROM usage_logs WHERE protocol = 'openai.chat' ORDER BY created_at DESC LIMIT 40 `), - openai_python: rows(` + openai_python: await rows(` SELECT stream, status, final_state, error_code, - input_tokens, output_tokens, ROUND(total_cost,6) AS cost, model, via - FROM usage_logs WHERE user_agent LIKE 'OpenAI/%' + input_tokens, output_tokens, ROUND((total_cost)::numeric,6) AS cost, model, via + FROM usage_logs WHERE user_agent ILIKE 'OpenAI/%' ORDER BY created_at DESC `), - openai_ok_null: rows(` + openai_ok_null: await rows(` SELECT COUNT(*) AS n, SUM(stream) AS streams, SUM(CASE WHEN final_state = 'incomplete' THEN 1 ELSE 0 END) AS incomplete, SUM(CASE WHEN final_state = 'verified' THEN 1 ELSE 0 END) AS verified, SUM(CASE WHEN error_code IS NOT NULL THEN 1 ELSE 0 END) AS with_err FROM usage_logs - WHERE protocol LIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NULL + WHERE protocol ILIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NULL `), - openai_ok_with_tok: rows(` + openai_ok_with_tok: await rows(` SELECT COUNT(*) AS n, SUM(stream) AS streams, SUM(COALESCE(input_tokens,0)) AS in_tok, SUM(COALESCE(output_tokens,0)) AS out_tok, SUM(COALESCE(cache_read_tokens,0)) AS cache_read, - ROUND(SUM(COALESCE(total_cost,0)),6) AS cost + ROUND((SUM(COALESCE(total_cost,0)))::numeric,6) AS cost FROM usage_logs - WHERE protocol LIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NOT NULL + WHERE protocol ILIKE 'openai%' AND status BETWEEN 200 AND 299 AND input_tokens IS NOT NULL `), - stream_ok_null: rows(` + stream_ok_null: await rows(` SELECT protocol, COUNT(*) AS n, error_code, final_state FROM usage_logs WHERE stream = 1 AND status BETWEEN 200 AND 299 AND input_tokens IS NULL GROUP BY protocol, error_code, final_state ORDER BY n DESC `), - vm_test_anywhere: rows(` + vm_test_anywhere: await rows(` SELECT COUNT(*) AS n FROM usage_logs - WHERE user_agent LIKE '%kin-console-test%' OR path LIKE '%test-chat%' + WHERE user_agent ILIKE '%kin-console-test%' OR path ILIKE '%test-chat%' `), -} + } -console.log(JSON.stringify(out, null, 2)) -db.close() + console.log(JSON.stringify(out, null, 2)) +} finally { + await db.close() +} diff --git a/scripts/audit-db.mjs b/scripts/audit-db.mjs new file mode 100644 index 000000000..b1057d13a --- /dev/null +++ b/scripts/audit-db.mjs @@ -0,0 +1,41 @@ +/** Shared read-only PostgreSQL connection for the four audit commands. */ +import pg from 'pg' +import { quoteIdentifier, resolveDatabaseUrl } from '../src/lib/db/database.mjs' + +export async function openAuditDb() { + const schema = process.env.VM2API_DATABASE_SCHEMA || process.env.KIN_DATABASE_SCHEMA || 'public' + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(schema)) throw new Error('invalid database schema') + const client = new pg.Client({ + connectionString: resolveDatabaseUrl(), + options: `-c default_transaction_read_only=on -c search_path=${quoteIdentifier(schema)},pg_catalog -c timezone=UTC`, + types: { + getTypeParser(oid, format) { + if (format !== 'binary' && (oid === 20 || oid === 1700)) + return (value) => { + const number = Number(value) + return Number.isFinite(number) && Math.abs(number) <= Number.MAX_SAFE_INTEGER ? number : value + } + return pg.types.getTypeParser(oid, format) + }, + }, + }) + try { + await client.connect() + } catch { + throw new Error('PostgreSQL audit connection failed') + } + const query = async (sql, params) => { + try { + return await client.query(sql, params) + } catch (error) { + throw new Error(`PostgreSQL audit query failed (${error.code || 'database_error'})`) + } + } + return { + prepare: (sql) => ({ + all: async (...params) => (await query(sql, params)).rows, + get: async (...params) => (await query(sql, params)).rows[0], + }), + close: () => client.end(), + } +} diff --git a/scripts/audit-fable-week.mjs b/scripts/audit-fable-week.mjs index a9a948c00..2f088009c 100644 --- a/scripts/audit-fable-week.mjs +++ b/scripts/audit-fable-week.mjs @@ -1,38 +1,41 @@ -import { DatabaseSync } from 'node:sqlite' +import { openAuditDb } from './audit-db.mjs' -const db = new DatabaseSync(process.argv[2] || '/opt/kin-gateway/data/kin.db', { readOnly: true }) +const db = await openAuditDb() -const acc = db - .prepare(` +try { + const acc = await db + .prepare(` SELECT vm_id, - json_extract(extra, '$.unified.7d.utilization') AS u7, - json_extract(extra, '$.unified.7d.status') AS s7, - json_extract(extra, '$.unified.7d_oi.utilization') AS uoi, - json_extract(extra, '$.unified.7d_oi.status') AS soi, - json_extract(extra, '$.unified.5h.utilization') AS u5 + extra::jsonb #>> '{unified,7d,utilization}' AS u7, + extra::jsonb #>> '{unified,7d,status}' AS s7, + extra::jsonb #>> '{unified,7d_oi,utilization}' AS uoi, + extra::jsonb #>> '{unified,7d_oi,status}' AS soi, + extra::jsonb #>> '{unified,5h,utilization}' AS u5 FROM accounts `) - .all() + .all() -const logs = db - .prepare(` + const logs = await db + .prepare(` SELECT CASE - WHEN model LIKE '%fable%' OR upstream_model LIKE '%fable%' THEN 'fable' - WHEN model LIKE '%opus%' OR upstream_model LIKE '%opus%' THEN 'opus' - WHEN model LIKE '%sonnet%' OR upstream_model LIKE '%sonnet%' THEN 'sonnet' - WHEN model LIKE '%haiku%' OR upstream_model LIKE '%haiku%' THEN 'haiku' + WHEN model ILIKE '%fable%' OR upstream_model ILIKE '%fable%' THEN 'fable' + WHEN model ILIKE '%opus%' OR upstream_model ILIKE '%opus%' THEN 'opus' + WHEN model ILIKE '%sonnet%' OR upstream_model ILIKE '%sonnet%' THEN 'sonnet' + WHEN model ILIKE '%haiku%' OR upstream_model ILIKE '%haiku%' THEN 'haiku' ELSE 'other' END AS fam, COUNT(*) AS n, - ROUND(SUM(COALESCE(total_cost,0)),4) AS cost, + ROUND((SUM(COALESCE(total_cost,0)))::numeric,4) AS cost, SUM(COALESCE(input_tokens,0)+COALESCE(output_tokens,0)+COALESCE(cache_read_tokens,0)+COALESCE(cache_creation_tokens,0)) AS toks FROM usage_logs - WHERE created_at >= datetime('now', '-7 days') + WHERE created_at >= to_char(now() AT TIME ZONE 'UTC' - interval '7 days', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') GROUP BY fam ORDER BY cost DESC `) - .all() + .all() -console.log(JSON.stringify({ quota: acc, last7d_by_family: logs }, null, 2)) -db.close() + console.log(JSON.stringify({ quota: acc, last7d_by_family: logs }, null, 2)) +} finally { + await db.close() +} diff --git a/scripts/check-postgres-upgrade.mjs b/scripts/check-postgres-upgrade.mjs new file mode 100644 index 000000000..69f4e22f9 --- /dev/null +++ b/scripts/check-postgres-upgrade.mjs @@ -0,0 +1,49 @@ +/** Read-only cutover check, executed from the new image before replacing the live container. */ +import fs from 'node:fs' +import path from 'node:path' +import crypto from 'node:crypto' +import { fileURLToPath } from 'node:url' +import { openAuditDb } from './audit-db.mjs' + +const dataDir = process.env.KIN_DATA_DIR || path.resolve('data') +let db +try { + if (fs.existsSync(path.join(dataDir, 'restore-maintenance.json'))) + throw new Error('offline restore recovery required') + db = await openAuditDb() + const server = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) + if (server < 170000 || server >= 180000) throw new Error('PostgreSQL 17 required') + const directory = fileURLToPath(new URL('../src/lib/db/postgres-migrations/', import.meta.url)) + const expected = fs + .readdirSync(directory) + .filter((name) => /^\d+.*\.sql$/.test(name)) + .sort() + .map((name) => ({ + version: name.split('_')[0], + checksum: crypto + .createHash('sha256') + .update(fs.readFileSync(path.join(directory, name), 'utf8').replaceAll('\r\n', '\n')) + .digest('hex'), + })) + const applied = await db.prepare('SELECT version, checksum FROM schema_migrations ORDER BY version').all() + if ( + !applied.length || + applied.length > expected.length || + applied.some((row, index) => row.version !== expected[index]?.version || row.checksum !== expected[index]?.checksum) + ) { + throw new Error('PostgreSQL migration history is missing, unknown or inconsistent') + } + if (fs.existsSync(path.join(dataDir, 'kin.db'))) { + const row = await db.prepare("SELECT value FROM settings WHERE key = 'sqlite_import_done'").get() + if (!row?.value || !JSON.parse(row.value).source_sha256) throw new Error('missing import marker') + } else { + await db.prepare('SELECT 1 AS ok').get() + } +} catch { + console.error( + 'PostgreSQL cutover check failed. Keep the current container and verify PostgreSQL 17, migration history, import marker or offline recovery; see docs/deploy/postgresql.md.', + ) + process.exitCode = 1 +} finally { + await db?.close() +} diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 0b936259c..617bd966a 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -3,6 +3,10 @@ set -eu ROOT="${KIN_PROJECT_ROOT:-/opt/vm2api}" IMAGE_BIN="/opt/vm2api/image-bin" +if [ -z "${VM2API_DATABASE_URL:-${KIN_DATABASE_URL:-}}" ]; then + echo "vm2api: PostgreSQL 17 connection is required. Configure VM2API_DATABASE_URL; see docs/deploy/postgresql.md for SQLite import." >&2 + exit 1 +fi mkdir -p "$ROOT/vms" "$ROOT/data" "$ROOT/bin" # 镜像安装没有仓库 checkout：把镜像内默认配置补进被 bind-mount 覆盖的 src/config。 diff --git a/scripts/panel-upgrade.sh b/scripts/panel-upgrade.sh index 65fa1bb1c..67a948294 100755 --- a/scripts/panel-upgrade.sh +++ b/scripts/panel-upgrade.sh @@ -9,7 +9,6 @@ TAG=$5 ARCH=$6 shift 6 cd "$ROOT" - COMPOSE_FILE_LIST='' for file do if [ -n "$COMPOSE_FILE_LIST" ]; then COMPOSE_FILE_LIST="$COMPOSE_FILE_LIST:"; fi @@ -126,6 +125,8 @@ else compose pull vm2api fi +# The new image checks the import marker without starting HTTP or applying migrations. +compose run --rm --no-deps -T --entrypoint node vm2api scripts/check-postgres-upgrade.mjs compose up -d --no-deps --force-recreate vm2api CURRENT_ID=$(docker inspect --format '{{.Id}}' "$CONTAINER") if [ "$CURRENT_ID" = "$PREVIOUS_ID" ]; then diff --git a/src/lib/db/database-metrics.mjs b/src/lib/db/database-metrics.mjs index 68925ddcf..5a637033c 100644 --- a/src/lib/db/database-metrics.mjs +++ b/src/lib/db/database-metrics.mjs @@ -1,120 +1,54 @@ -import fs from 'node:fs' - +/** Read-only PostgreSQL and application-cache metrics; never expose connection details. */ function numberOrNull(value) { if (value == null || value === '') return null - const num = Number(value) - return Number.isFinite(num) ? num : null -} - -function scalar(db, sql) { - try { - const row = db.prepare(sql).get() - if (!row || typeof row !== 'object') return null - return Object.values(row)[0] ?? null - } catch { - return null - } + const n = Number(value) + return Number.isFinite(n) && Math.abs(n) <= Number.MAX_SAFE_INTEGER ? n : null } -function probe(db, clock) { - try { - const startedAt = clock() - db.prepare('SELECT 1 AS ok').get() - const elapsed = numberOrNull(clock() - startedAt) - return { - ok: true, - latencyMs: elapsed == null ? null : Math.round(Math.max(elapsed, 0) * 1000) / 1000, - } - } catch { - return { ok: false, latencyMs: null } - } -} - -function fileSize(dbPath, suffix = '') { - if (typeof dbPath !== 'string' || !dbPath) return null +export async function snapshotDatabaseMetrics({ + db, + usageCache, + now = () => Date.now(), + clock = () => performance.now(), +}) { + let ok = false + let latency = null try { - return fs.statSync(`${dbPath}${suffix}`).size - } catch (error) { - return error?.code === 'ENOENT' ? 0 : null - } -} - -function migrationMetrics(db) { - const count = numberOrNull(scalar(db, 'SELECT COUNT(*) AS count FROM schema_migrations')) + const started = clock() + await db.prepare('SELECT 1 AS ok').get() + ok = true + latency = Math.round(Math.max(clock() - started, 0) * 1000) / 1000 + } catch {} + let size = null + let migrations = null let latest = null try { - latest = db - .prepare('SELECT version, applied_at FROM schema_migrations ORDER BY applied_at DESC, version DESC LIMIT 1') - .get() + size = numberOrNull((await db.prepare('SELECT pg_database_size(current_database()) AS bytes').get())?.bytes) } catch {} - return { - migration_count: count, - latest_migration: latest?.version == null ? null : String(latest.version), - latest_migration_at: latest?.applied_at == null ? null : String(latest.applied_at), - } -} - -function pageMetrics(pageSize, pageCount, freelistPages) { - if (pageCount == null || freelistPages == null) { - return { used_pages: null, used_bytes: null, free_ratio: null } - } - const usedPages = Math.max(pageCount - freelistPages, 0) - return { - used_pages: usedPages, - used_bytes: pageSize == null ? null : usedPages * pageSize, - free_ratio: pageCount > 0 ? freelistPages / pageCount : null, - } -} - -function safeUsageStats(usageCache) { try { - return typeof usageCache?.stats === 'function' ? usageCache.stats() : null - } catch { - return null - } -} - -function sampledAt(now) { + migrations = numberOrNull((await db.prepare('SELECT COUNT(*) AS count FROM schema_migrations').get())?.count) + } catch {} try { - return new Date(now()).toISOString() - } catch { - return null - } -} - -export function snapshotDatabaseMetrics({ - db, - dbPath, - usageCache, - now = () => Date.now(), - clock = () => performance.now(), -}) { - const health = probe(db, clock) - const journalMode = scalar(db, 'PRAGMA journal_mode') - const foreignKeys = numberOrNull(scalar(db, 'PRAGMA foreign_keys')) - const pageSize = numberOrNull(scalar(db, 'PRAGMA page_size')) - const pageCount = numberOrNull(scalar(db, 'PRAGMA page_count')) - const freelistPages = numberOrNull(scalar(db, 'PRAGMA freelist_count')) - + latest = await db.prepare('SELECT version, applied_at FROM schema_migrations ORDER BY version DESC LIMIT 1').get() + } catch {} + let usage = null + try { + usage = typeof usageCache?.stats === 'function' ? usageCache.stats() : null + } catch {} return { - sampled_at: sampledAt(now), + sampled_at: new Date(now()).toISOString(), database: { - ok: health.ok, - engine: 'sqlite', - probe_latency_ms: health.latencyMs, - journal_mode: journalMode == null ? null : String(journalMode).toLowerCase(), - synchronous: numberOrNull(scalar(db, 'PRAGMA synchronous')), - foreign_keys: foreignKeys == null ? null : foreignKeys !== 0, - busy_timeout_ms: numberOrNull(scalar(db, 'PRAGMA busy_timeout')), - file_size_bytes: fileSize(dbPath), - wal_size_bytes: fileSize(dbPath, '-wal'), - shm_size_bytes: fileSize(dbPath, '-shm'), - page_size_bytes: pageSize, - page_count: pageCount, - freelist_pages: freelistPages, - ...pageMetrics(pageSize, pageCount, freelistPages), - ...migrationMetrics(db), + ok, + engine: 'postgresql', + probe_latency_ms: latency, + size_bytes: size, + pool_total: numberOrNull(db.pool?.totalCount), + pool_idle: numberOrNull(db.pool?.idleCount), + pool_waiting: numberOrNull(db.pool?.waitingCount), + migration_count: migrations, + latest_migration: latest?.version == null ? null : String(latest.version), + latest_migration_at: latest?.applied_at == null ? null : String(latest.applied_at), }, - usage_cache: safeUsageStats(usageCache), + usage_cache: usage, } } diff --git a/test/deploy/smoke_arm64_image.py b/test/deploy/smoke_arm64_image.py index 062bd8d50..64d048553 100644 --- a/test/deploy/smoke_arm64_image.py +++ b/test/deploy/smoke_arm64_image.py @@ -3,6 +3,9 @@ import argparse import json +import os +import secrets +import time from pathlib import Path import subprocess import uuid @@ -41,6 +44,11 @@ def elf(path, machine, static=False): elf(root / 'image-bin' / 'kin-api-kernel', 183, static=True) elf(Path('/usr/local/bin/docker'), 183, static=True) subprocess.run(['docker', '--version'], check=True, timeout=10) +for name in ('pg_dump', 'pg_restore'): + binary = Path('/usr/lib/postgresql/17/bin') / name + elf(binary, 183) + assert ' 17.' in subprocess.check_output([str(binary), '--version'], text=True) + for name in ('kin-kernel', 'kin-codex-kernel', 'kin-oauth-auth', 'kin-computer'): elf(root / 'image-bin' / name, 62) for name in ('kin-kernel', 'kin-codex-kernel'): @@ -113,18 +121,30 @@ def main(): args = parser.parse_args() expected = {"version": (ROOT / "VERSION").read_text().strip()} name = "vm2api-arm64-smoke-" + uuid.uuid4().hex[:12] + network = name + '-net' + postgres = name + '-pg' + password = secrets.token_hex(32) + db_env = {**os.environ, 'POSTGRES_PASSWORD': password, 'POSTGRES_USER': 'vm2api', 'POSTGRES_DB': 'vm2api_smoke_test', + 'VM2API_DATABASE_URL': f'postgresql://vm2api:{password}@{postgres}:5432/vm2api_smoke_test'} try: + subprocess.run(['docker', 'network', 'create', '--internal', network], check=True, capture_output=True) + subprocess.run(['docker', 'run', '-d', '--name', postgres, '--network', network, + '-e', 'POSTGRES_PASSWORD', '-e', 'POSTGRES_USER', '-e', 'POSTGRES_DB', + 'postgres:17-bookworm'], env=db_env, check=True, capture_output=True) + deadline = time.monotonic() + 60 + while subprocess.run(['docker', 'exec', postgres, 'psql', '-U', 'vm2api', '-d', 'vm2api_smoke_test', '-Atqc', 'SELECT 1'], + capture_output=True).returncode: + if time.monotonic() > deadline: + raise RuntimeError('isolated PostgreSQL did not become ready') + time.sleep(0.5) subprocess.run(["docker", "run", "--name", name, "--platform", "linux/arm64", - "--network", "none", "-i", "--entrypoint", "python3", args.image, - "-", json.dumps(expected)], input=PROGRAM, text=True, check=True, timeout=210) + "--network", network, '-e', 'VM2API_DATABASE_URL', "-i", "--entrypoint", "python3", args.image, + "-", json.dumps(expected)], env=db_env, input=PROGRAM, text=True, check=True, timeout=210) finally: - result = subprocess.run(["docker", "inspect", "--format", "{{.Id}} {{.State.Running}}", name], - capture_output=True, text=True) - if result.returncode == 0: - container_id, running = result.stdout.strip().split() - if running == "true": - subprocess.run(["docker", "stop", "--time", "5", container_id], check=True) - subprocess.run(["docker", "container", "rm", container_id], check=True) + for container in (name, postgres): + subprocess.run(['docker', 'rm', '-fv', container], capture_output=True) + subprocess.run(['docker', 'network', 'rm', network], capture_output=True) + if __name__ == "__main__": diff --git a/test/deploy/smoke_postgres_role.py b/test/deploy/smoke_postgres_role.py new file mode 100644 index 000000000..4b835c062 --- /dev/null +++ b/test/deploy/smoke_postgres_role.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Verify the Compose bootstrap role in a disposable PostgreSQL container.""" +import os +from pathlib import Path +import secrets +import subprocess +import time +import uuid + +name = 'vm2api-pg-role-test-' + uuid.uuid4().hex[:10] +root = Path(__file__).resolve().parents[2] +env = os.environ.copy() +env['VM2API_POSTGRES_PASSWORD'] = secrets.token_hex(32) +env['PGPASSWORD'] = env['VM2API_POSTGRES_PASSWORD'] +env['POSTGRES_INITDB_ARGS'] = '--auth-host=scram-sha-256' + + +def run(args, check=True): + return subprocess.run(['docker', *args], env=env, capture_output=True, text=True, check=check) + + +try: + run(['run', '-d', '--name', name, '--network', 'none', '-e', 'VM2API_POSTGRES_PASSWORD', '-e', 'POSTGRES_INITDB_ARGS', + '-v', str(root / 'deploy/postgres-entrypoint.sh') + ':/usr/local/bin/vm2api-postgres-entrypoint:ro', + '-v', str(root / 'deploy/postgres-init.sh') + ':/docker-entrypoint-initdb.d/10-vm2api.sh:ro', + '--entrypoint', '/bin/bash', 'postgres:17-bookworm', + '/usr/local/bin/vm2api-postgres-entrypoint', 'postgres']) + deadline = time.monotonic() + 60 + while run(['exec', name, 'psql', '-U', 'vm2api', '-d', 'vm2api', '-Atqc', 'SELECT 1'], False).returncode: + if time.monotonic() > deadline: + raise RuntimeError('isolated role test PostgreSQL did not become ready') + time.sleep(0.25) + flags = run(['exec', name, 'psql', '-U', 'postgres', '-d', 'postgres', '-Atqc', + "SELECT rolsuper,rolcreatedb,rolcreaterole,rolreplication FROM pg_roles WHERE rolname='vm2api'"]).stdout.strip() + owner = run(['exec', name, 'psql', '-U', 'postgres', '-d', 'postgres', '-Atqc', + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname='vm2api'"]).stdout.strip() + assert flags == 'f|f|f|f', 'application role permissions differ' + assert owner == 'vm2api', 'application database has a different owner' + run(['exec', '-e', 'PGPASSWORD', name, 'psql', '-h', '127.0.0.1', '-U', 'vm2api', '-d', 'vm2api', + '--no-password', '-v', 'ON_ERROR_STOP=1', '-c', 'CREATE TABLE role_check (id integer); DROP TABLE role_check;']) + bootstrap = run(['exec', '-e', 'PGPASSWORD', name, 'psql', '-h', '127.0.0.1', '-U', 'postgres', '-d', 'postgres', + '--no-password', '-Atqc', 'SELECT 1'], False) + assert bootstrap.returncode != 0 and 'password authentication failed' in bootstrap.stderr, 'bootstrap must use a different password' + denied = run(['exec', '-e', 'PGPASSWORD', name, 'psql', '-h', '127.0.0.1', '-U', 'vm2api', '-d', 'vm2api', + '--no-password', '-v', 'ON_ERROR_STOP=1', '-c', 'CREATE DATABASE forbidden_role_test;'], False) + assert denied.returncode != 0 and 'permission denied' in denied.stderr, 'application role must not create databases' + print('PASS: app role has no superuser/createdb/createrole/replication; owns database; ' + 'password auth and DDL work; bootstrap password differs; CREATE DATABASE denied') +finally: + run(['rm', '-fv', name], False) diff --git a/test/deploy/test_postgres_env.py b/test/deploy/test_postgres_env.py new file mode 100644 index 000000000..79cb4a27f --- /dev/null +++ b/test/deploy/test_postgres_env.py @@ -0,0 +1,36 @@ +"""Validate generated PostgreSQL deployment config without exposing its credentials.""" +from pathlib import Path +import shutil +import stat +import subprocess +import tempfile +import unittest +from urllib.parse import urlparse + + +class PostgreSqlEnvironmentTests(unittest.TestCase): + def test_arm64_initializer_writes_matching_private_database_configuration(self): + with tempfile.TemporaryDirectory(prefix='vm2api-env-') as directory: + root = Path(directory) + (root / 'deploy').mkdir() + script = root / 'deploy' / 'init-arm64-env.py' + shutil.copyfile(Path(__file__).resolve().parents[2] / 'deploy' / 'init-arm64-env.py', script) + result = subprocess.run(['python3', str(script), '--port', '18787'], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + values = dict(line.split('=', 1) for line in (root / '.env').read_text().splitlines() if line and not line.startswith('#')) + url = urlparse(values['VM2API_DATABASE_URL']) + self.assertEqual(url.password, values['VM2API_POSTGRES_PASSWORD']) + self.assertEqual(url.hostname, '127.0.0.1') + self.assertEqual(url.port, 55432) + self.assertEqual(url.path, '/vm2api') + self.assertEqual(values['COMPOSE_PROFILES'], 'postgres') + self.assertEqual(stat.S_IMODE((root / '.env').stat().st_mode), 0o600) + self.assertNotIn(values['VM2API_POSTGRES_PASSWORD'], result.stdout + result.stderr) + original = (root / '.env').read_bytes() + result = subprocess.run(['python3', str(script)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((root / '.env').read_bytes(), original) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/e2e/panel-api.e2e.test.mjs b/test/e2e/panel-api.e2e.test.mjs index a807cbc8f..7ffb26552 100644 --- a/test/e2e/panel-api.e2e.test.mjs +++ b/test/e2e/panel-api.e2e.test.mjs @@ -33,18 +33,18 @@ test('panel login → cookie → /api/panel/me', async () => { } }) -test('admin database metrics endpoint returns a safe SQLite snapshot', async () => { +test('admin database metrics endpoint returns a safe PostgreSQL snapshot', async () => { const gw = await startGateway() try { const response = await api(gw, 'GET', '/api/panel/database/metrics') assert.equal(response.status, 200, response.text) const data = response.json.data || response.json - assert.equal(data.database.engine, 'sqlite') + assert.equal(data.database.engine, 'postgresql') assert.equal(data.database.ok, true) - assert.equal(data.database.journal_mode, 'wal') + assert.equal('wal_size_bytes' in data.database, false) assert.equal(typeof data.database.probe_latency_ms, 'number') - assert.equal(typeof data.database.file_size_bytes, 'number') - assert.equal(typeof data.database.page_count, 'number') + assert.equal(typeof data.database.size_bytes, 'number') + assert.equal(typeof data.database.pool_total, 'number') assert.equal(typeof data.usage_cache.requests, 'number') assert.ok(data.sampled_at) diff --git a/test/unit/audit-postgres.test.mjs b/test/unit/audit-postgres.test.mjs new file mode 100644 index 000000000..e022255c2 --- /dev/null +++ b/test/unit/audit-postgres.test.mjs @@ -0,0 +1,89 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +const exec = promisify(execFile) + +test('audit commands execute PostgreSQL aggregates using a read-only connection', async () => { + const db = getDb() + for (const [script, args] of [ + ['audit-billing-gaps.mjs', []], + ['audit-billing-openai.mjs', []], + ['audit-fable-week.mjs', []], + ['audit-billing-contract.mjs', ['--database', '--json']], + ]) { + const { stdout } = await exec(process.execPath, [`scripts/${script}`, ...args], { + env: { ...process.env, VM2API_DATABASE_URL: db.connectionString, VM2API_DATABASE_SCHEMA: db.schema }, + }) + const report = JSON.parse(stdout) + assert.equal(typeof report, 'object', script) + if (script === 'audit-billing-gaps.mjs') assert.equal(typeof report.totals.n, 'number') + if (script === 'audit-billing-contract.mjs') assert.equal(typeof report.db.totals.n, 'number') + assert.equal(stdout.includes('postgresql://'), false, script) + } +}) + +async function preflight() { + const db = getDb() + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-upgrade-preflight-')) + try { + return await exec(process.execPath, ['scripts/check-postgres-upgrade.mjs'], { + env: { + ...process.env, + KIN_DATA_DIR: directory, + VM2API_DATABASE_URL: db.connectionString, + VM2API_DATABASE_SCHEMA: db.schema, + }, + }) + } finally { + fs.rmSync(directory, { recursive: true, force: true }) + } +} + +test('upgrade preflight reads valid source checksums without modifying migration history', async () => { + const before = await getDb().prepare('SELECT * FROM schema_migrations ORDER BY version').all() + await preflight() + assert.deepEqual(await getDb().prepare('SELECT * FROM schema_migrations ORDER BY version').all(), before) +}) + +test('upgrade preflight refuses an applied checksum mismatch without repairing it', async () => { + const row = await getDb().prepare('SELECT version FROM schema_migrations ORDER BY version LIMIT 1').get() + await getDb() + .prepare('UPDATE schema_migrations SET checksum = $1 WHERE version = $2') + .run('synthetic-mismatch', row.version) + await assert.rejects(preflight(), (error) => error.code === 1 && /preflight|cutover/i.test(error.stderr)) + assert.equal( + (await getDb().prepare('SELECT checksum FROM schema_migrations WHERE version = $1').get(row.version)).checksum, + 'synthetic-mismatch', + ) +}) + +test('upgrade preflight refuses unknown migrations and an empty history', async () => { + await getDb() + .prepare('INSERT INTO schema_migrations (version,name,checksum,applied_at) VALUES ($1,$2,$3,$4)') + .run('999-unknown', 'unknown', 'unknown', new Date().toISOString()) + await assert.rejects(preflight(), (error) => error.code === 1) + await getDb().exec('DELETE FROM schema_migrations') + await assert.rejects(preflight(), (error) => error.code === 1) +}) + +test('upgrade preflight refuses missing migration history and persisted restore maintenance', async () => { + await getDb().exec('DROP TABLE schema_migrations') + await assert.rejects(preflight(), (error) => error.code === 1) + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-preflight-marker-')) + try { + fs.writeFileSync(path.join(directory, 'restore-maintenance.json'), '{}') + await assert.rejects( + exec(process.execPath, ['scripts/check-postgres-upgrade.mjs'], { + env: { ...process.env, KIN_DATA_DIR: directory }, + }), + (error) => error.code === 1, + ) + } finally { + fs.rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/test/unit/database-metrics.test.mjs b/test/unit/database-metrics.test.mjs index 63b1a8b9f..a3fc79d62 100644 --- a/test/unit/database-metrics.test.mjs +++ b/test/unit/database-metrics.test.mjs @@ -1,138 +1,61 @@ import test from 'node:test' import assert from 'node:assert/strict' -import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' -import { createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import { snapshotDatabaseMetrics } from '../../src/lib/db/database-metrics.mjs' -function fakeDb(values = {}, failures = new Set()) { - const statements = [] - return { - statements, - prepare(sql) { - statements.push(sql) - if (failures.has(sql)) throw new Error('read failed') - return { - get() { - if (failures.has(`${sql}:get`)) throw new Error('read failed') - if (sql === 'SELECT 1 AS ok') return { ok: 1 } - if (sql.startsWith('SELECT COUNT(*)')) return { count: values.migration_count ?? 0 } - if (sql.startsWith('SELECT version')) return values.latest ?? undefined - return { value: values[sql] } - }, - } - }, - } -} - -test('snapshot reports the live SQLite/WAL state without exposing its path', () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-db-metrics-')) - const dbPath = path.join(tmp, 'private-name.db') - const db = createDatabase({ dbPath }) - try { - const snapshot = snapshotDatabaseMetrics({ - db, - dbPath, - usageCache: { stats: () => ({ requests: 0, hit_rate: null }) }, - now: () => Date.parse('2026-08-31T00:00:00.000Z'), - clock: (() => { - const samples = [10, 10.375] - return () => samples.shift() - })(), - }) - - assert.equal(snapshot.sampled_at, '2026-08-31T00:00:00.000Z') - assert.equal(snapshot.database.ok, true) - assert.equal(snapshot.database.engine, 'sqlite') - assert.equal(snapshot.database.probe_latency_ms, 0.375) - assert.equal(snapshot.database.journal_mode, 'wal') - assert.equal(snapshot.database.synchronous, 1) - assert.equal(snapshot.database.foreign_keys, true) - assert.equal(snapshot.database.busy_timeout_ms, 5000) - assert.ok(snapshot.database.file_size_bytes > 0) - assert.ok(snapshot.database.page_size_bytes > 0) - assert.ok(snapshot.database.page_count > 0) - assert.ok(snapshot.database.migration_count > 0) - assert.match(snapshot.database.latest_migration, /^\d+$/) - assert.equal(snapshot.usage_cache.requests, 0) - assert.equal(JSON.stringify(snapshot).includes(tmp), false) - assert.equal(JSON.stringify(snapshot).includes('private-name.db'), false) - } finally { - db.close() - fs.rmSync(tmp, { recursive: true, force: true }) - } +const fakeDb = (failures = new Set()) => ({ + pool: { totalCount: 3, idleCount: 2, waitingCount: 1 }, + prepare(sql) { + assert.match(sql, /^SELECT /) + return { + async get() { + if (failures.has(sql)) throw new Error('private connection failure') + if (sql.includes('pg_database_size')) return { bytes: '4096' } + if (sql.includes('COUNT(*)')) return { count: '2' } + if (sql.includes('version,')) return { version: '001', applied_at: '2026-10-21T00:00:00.000Z' } + return { ok: 1 } + }, + } + }, }) -test('snapshot derives page usage and treats absent database files as real zeroes', () => { - const db = fakeDb({ - 'PRAGMA journal_mode': 'wal', - 'PRAGMA synchronous': 1, - 'PRAGMA foreign_keys': 1, - 'PRAGMA busy_timeout': 5000, - 'PRAGMA page_size': 4096, - 'PRAGMA page_count': 10, - 'PRAGMA freelist_count': 3, - migration_count: 2, - latest: { version: '002', applied_at: '2026-08-30T00:00:00.000Z' }, - }) - const missing = path.join(os.tmpdir(), `missing-kin-${process.pid}-${Date.now()}.db`) - const snapshot = snapshotDatabaseMetrics({ db, dbPath: missing, usageCache: null, clock: () => 1 }) - - assert.equal(snapshot.database.used_pages, 7) - assert.equal(snapshot.database.used_bytes, 28_672) - assert.equal(snapshot.database.free_ratio, 0.3) - assert.equal(snapshot.database.file_size_bytes, 0) - assert.equal(snapshot.database.wal_size_bytes, 0) - assert.equal(snapshot.database.shm_size_bytes, 0) - assert.equal(snapshot.usage_cache, null) - assert.ok(db.statements.every((sql) => /^(SELECT 1|SELECT (COUNT|version)|PRAGMA )/.test(sql))) - assert.ok(db.statements.every((sql) => !/(checkpoint|vacuum|optimize|integrity|quick_check)/i.test(sql))) +test('PostgreSQL metrics report database size, migration state and this process pool', async () => { + const snapshot = await snapshotDatabaseMetrics({ db: getDb(), usageCache: { stats: () => ({ requests: 4 }) } }) + assert.equal(snapshot.database.engine, 'postgresql') + assert.equal(snapshot.database.ok, true) + assert.ok(snapshot.database.size_bytes > 0) + assert.ok(snapshot.database.migration_count > 0) + assert.equal(typeof snapshot.database.pool_total, 'number') + assert.equal(snapshot.usage_cache.requests, 4) + assert.equal(JSON.stringify(snapshot).includes('postgresql://'), false) + assert.equal('wal_size_bytes' in snapshot.database, false) }) -test('snapshot preserves partial results when the probe or a metric read fails', () => { - const db = fakeDb( - { - 'PRAGMA journal_mode': 'wal', - 'PRAGMA synchronous': 1, - 'PRAGMA foreign_keys': 1, - 'PRAGMA busy_timeout': 5000, - 'PRAGMA page_size': 4096, - 'PRAGMA page_count': 0, - 'PRAGMA freelist_count': 0, - }, - new Set(['SELECT 1 AS ok', 'PRAGMA synchronous']), - ) - const snapshot = snapshotDatabaseMetrics({ db, dbPath: null, usageCache: { stats: () => null } }) - +test('read-only snapshot retains independent metrics when the probe fails', async () => { + const snapshot = await snapshotDatabaseMetrics({ + db: fakeDb(new Set(['SELECT 1 AS ok'])), + now: () => Date.parse('2026-10-21T00:00:00.000Z'), + usageCache: null, + }) + assert.equal(snapshot.sampled_at, '2026-10-21T00:00:00.000Z') assert.equal(snapshot.database.ok, false) assert.equal(snapshot.database.probe_latency_ms, null) - assert.equal(snapshot.database.journal_mode, 'wal') - assert.equal(snapshot.database.synchronous, null) - assert.equal(snapshot.database.page_count, 0) - assert.equal(snapshot.database.used_pages, 0) - assert.equal(snapshot.database.used_bytes, 0) - assert.equal(snapshot.database.free_ratio, null) - assert.equal(snapshot.database.file_size_bytes, null) + assert.equal(snapshot.database.size_bytes, 4096) + assert.equal(snapshot.database.pool_total, 3) + assert.equal(snapshot.database.pool_idle, 2) + assert.equal(snapshot.database.pool_waiting, 1) + assert.equal(JSON.stringify(snapshot).includes('private'), false) }) -test('snapshot returns null for unavailable page inputs and cache stats failures', () => { - const db = new DatabaseSync(':memory:') - try { - const snapshot = snapshotDatabaseMetrics({ - db, - dbPath: null, - usageCache: { - stats() { - throw new Error('cache unavailable') - }, +test('cache-stat failure does not hide PostgreSQL health', async () => { + const snapshot = await snapshotDatabaseMetrics({ + db: fakeDb(), + usageCache: { + stats() { + throw new Error('cache failed') }, - }) - assert.equal(snapshot.database.migration_count, null) - assert.equal(snapshot.database.latest_migration, null) - assert.equal(snapshot.usage_cache, null) - } finally { - db.close() - } + }, + }) + assert.equal(snapshot.database.ok, true) + assert.equal(snapshot.usage_cache, null) }) diff --git a/test/unit/panel-upgrade-script.test.mjs b/test/unit/panel-upgrade-script.test.mjs index 3728bed73..715515d0c 100644 --- a/test/unit/panel-upgrade-script.test.mjs +++ b/test/unit/panel-upgrade-script.test.mjs @@ -17,6 +17,7 @@ const args = process.argv.slice(2) args.push('--compose-file-env', process.env.COMPOSE_FILE || '') fs.appendFileSync(process.env.TEST_LOG, JSON.stringify(args) + '\n') if (args[0] === 'compose') { + if (args.includes('run') && args.includes('scripts/check-postgres-upgrade.mjs') && process.env.FAIL_CUTOVER === '1') process.exit(1) if (args.includes('build') && process.env.FAIL_BUILD === '1') process.exit(1) if (args.includes('up') && process.env.STALE_BACKEND !== '1') { fs.writeFileSync(process.env.BOOT_VERSION, 'v1.2.7') @@ -227,3 +228,10 @@ test('ARM64 source upgrade keeps the native override and rejects a missing one b assert.match(missing.stderr, /lacks docker-compose.arm64.yml/) assert.ok(!missing.calls.some((args) => args[0] === 'compose')) }) + +test('failed PostgreSQL cutover preflight leaves the running control plane intact', async () => { + const result = await runUpgrade({ source: false, env: { FAIL_CUTOVER: '1' } }) + assert.equal(result.code, 1) + assert.ok(result.calls.some((args) => args.includes('scripts/check-postgres-upgrade.mjs'))) + assert.ok(!result.calls.some((args) => args[0] === 'compose' && args.includes('up'))) +}) diff --git a/web/src/features/database/database-overview.tsx b/web/src/features/database/database-overview.tsx index 584bd148c..29f8dbfec 100644 --- a/web/src/features/database/database-overview.tsx +++ b/web/src/features/database/database-overview.tsx @@ -4,7 +4,6 @@ import { PanelCard, StatCell } from '@/features/overview/panel-card' import { formatMetricBytes, formatMetricNumber, - formatMetricPercent, formatMetricTime, } from './metrics' @@ -14,17 +13,6 @@ function latency(value: number | null): string { return `${Math.round(value)}ms` } -function synchronousLabel(value: number | null): string { - if (value == null) return '—' - return ['OFF', 'NORMAL', 'FULL', 'EXTRA'][value] || String(value) -} - -function timeout(value: number | null): string { - if (value == null || !Number.isFinite(value)) return '—' - if (value >= 1000 && value % 1000 === 0) return `${value / 1000}s` - return `${value.toLocaleString()}ms` -} - function Detail({ label, value }: { label: string; value: string }) { return (

@@ -43,11 +31,6 @@ export function DatabaseOverview({ metrics: DatabaseRuntimeMetrics sampledAt: string | null }) { - const pageUse = - metrics.used_pages == null && metrics.page_count == null - ? '—' - : `${formatMetricNumber(metrics.used_pages)} / ${formatMetricNumber(metrics.page_count)}` - return ( - - - - -

- 当前 node:sqlite 接口不提供 SQLite 页缓存 - hit/miss；这里展示的是文件、页使用和只读探针，不把应用缓存冒充数据库缓存。 + 连接池指标属于当前控制面进程。数据库大小和健康探针通过只读查询获取。

) diff --git a/web/src/types/panel-database.ts b/web/src/types/panel-database.ts index 0f827f90e..ada58f63a 100644 --- a/web/src/types/panel-database.ts +++ b/web/src/types/panel-database.ts @@ -1,20 +1,11 @@ export type DatabaseRuntimeMetrics = { ok: boolean - engine: 'sqlite' + engine: 'postgresql' probe_latency_ms: number | null - journal_mode: string | null - synchronous: number | null - foreign_keys: boolean | null - busy_timeout_ms: number | null - file_size_bytes: number | null - wal_size_bytes: number | null - shm_size_bytes: number | null - page_size_bytes: number | null - page_count: number | null - freelist_pages: number | null - used_pages: number | null - used_bytes: number | null - free_ratio: number | null + size_bytes: number | null + pool_total: number | null + pool_idle: number | null + pool_waiting: number | null migration_count: number | null latest_migration: string | null latest_migration_at: string | null From f41e58933f435d9693c27762ddfe1080bc174968 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:55:15 +0800 Subject: [PATCH 017/473] test: cover pending scheduled result persistence --- test/unit/vm-test-schedule.test.mjs | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test/unit/vm-test-schedule.test.mjs b/test/unit/vm-test-schedule.test.mjs index 21249af08..8217717ad 100644 --- a/test/unit/vm-test-schedule.test.mjs +++ b/test/unit/vm-test-schedule.test.mjs @@ -296,3 +296,56 @@ test('sync with an invalid test writes nothing', () => { } assert.deepEqual(settings.all(), before) }) + +test('flush waits until a scheduled result finishes persisting', async () => { + const { settings, schedule } = runner() + await schedule.save('vm-01', claudeConfig) + schedule.started = true + let release + let entered + const gate = new Promise((resolve) => { release = resolve }) + const writing = new Promise((resolve) => { entered = resolve }) + const originalSet = settings.set.bind(settings) + settings.set = async (...args) => { + entered() + await gate + return originalSet(...args) + } + const run = schedule._run('vm-01') + await writing + let flushed = false + const flush = schedule.flush().then(() => { flushed = true }) + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(flushed, false, 'the scheduled result is still being written') + } finally { + release() + await run + await flush + schedule.stop() + } +}) + +test('flush reports a failed scheduled result write', async () => { + const { settings, schedule } = runner() + await schedule.save('vm-01', claudeConfig) + schedule.started = true + const failure = new Error('synthetic scheduled result write failure') + let release + let entered + const gate = new Promise((resolve) => { release = resolve }) + const writing = new Promise((resolve) => { entered = resolve }) + settings.set = async () => { + entered() + await gate + throw failure + } + const run = schedule._run('vm-01') + run.catch(() => {}) + await writing + const flush = schedule.flush() + release() + await assert.rejects(run, (error) => error === failure) + await assert.rejects(flush, (error) => error === failure) + schedule.stop() +}) From 6cb132b6040e70b9e1f322b641401245b4aad158 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:56:15 +0800 Subject: [PATCH 018/473] fix(db): preserve integer precision across PostgreSQL usage counters --- src/lib/admin/api-keys.mjs | 13 +- src/lib/admin/request-log.mjs | 33 ++-- src/lib/db/integers.mjs | 19 +++ src/lib/db/repos/accounts-repo.mjs | 25 +-- src/lib/db/repos/api-keys-repo.mjs | 20 ++- src/lib/db/repos/statistics-repo.mjs | 67 +++++--- src/lib/db/repos/usage-log-preds.mjs | 9 +- src/lib/db/repos/usage-logs-repo.mjs | 235 ++++++++++++++------------- src/lib/db/repos/usage-logs-view.mjs | 57 +++---- src/lib/db/repos/users-repo.mjs | 65 +++++--- test/unit/request-log.test.mjs | 12 +- 11 files changed, 320 insertions(+), 235 deletions(-) create mode 100644 src/lib/db/integers.mjs diff --git a/src/lib/admin/api-keys.mjs b/src/lib/admin/api-keys.mjs index a03a1f909..a2c35fa5c 100644 --- a/src/lib/admin/api-keys.mjs +++ b/src/lib/admin/api-keys.mjs @@ -19,6 +19,7 @@ import { keyScopeFromRecord, normalizeKeyScope } from './key-scope.mjs' import { VmPoolsRepo } from '../db/repos/vm-pools-repo.mjs' import crypto from 'node:crypto' import { resolveStoreDb, withTransaction } from '../db/database.mjs' +import { integerValue } from '../db/integers.mjs' import { ApiKeysRepo } from '../db/repos/api-keys-repo.mjs' import { maybeEncrypt, maybeDecrypt } from '../db/secure.mjs' import { calculateCost, normalizeUsage } from './pricing.mjs' @@ -301,7 +302,7 @@ export class ApiKeyStore { if ( rec.status === 'quota_exhausted' && !((rec.quota ?? 0) > 0 && (rec.quota_used ?? 0) >= rec.quota) && - !(rec.quota_requests > 0 && (rec.quota_requests_used ?? 0) >= rec.quota_requests) + !(rec.quota_requests > 0 && BigInt(rec.quota_requests_used ?? 0) >= BigInt(rec.quota_requests)) ) { rec.status = 'active' } @@ -386,7 +387,7 @@ export class ApiKeyStore { if (this._expired(rec, now)) { return { ok: false, code: 'api_key_expired', message: 'API key has expired', status: 403 } } - if (rec.quota_requests > 0 && (rec.quota_requests_used ?? 0) >= rec.quota_requests) { + if (rec.quota_requests > 0 && BigInt(rec.quota_requests_used ?? 0) >= BigInt(rec.quota_requests)) { return { ok: false, code: 'api_key_quota_exhausted', @@ -504,10 +505,10 @@ export class ApiKeyStore { const rate = Number.isFinite(Number(rateMultiplier)) ? Number(rateMultiplier) : 1 const cost = billed * rate const rec = await this.repo.recordUsage(id, { - tokens_in: Number(normalized.input_tokens) || Number(normalized.tokens_in) || 0, - tokens_out: Number(normalized.output_tokens) || Number(normalized.tokens_out) || 0, - cache_read_tokens: Number(normalized.cache_read_input_tokens ?? normalized.cache_read_tokens) || 0, - cache_creation_tokens: Number(normalized.cache_creation_input_tokens ?? normalized.cache_creation_tokens) || 0, + tokens_in: integerValue(normalized.input_tokens ?? normalized.tokens_in), + tokens_out: integerValue(normalized.output_tokens ?? normalized.tokens_out), + cache_read_tokens: integerValue(normalized.cache_read_input_tokens ?? normalized.cache_read_tokens), + cache_creation_tokens: integerValue(normalized.cache_creation_input_tokens ?? normalized.cache_creation_tokens), cost, }) return rec diff --git a/src/lib/admin/request-log.mjs b/src/lib/admin/request-log.mjs index 4216c999b..7c1d71bb6 100644 --- a/src/lib/admin/request-log.mjs +++ b/src/lib/admin/request-log.mjs @@ -19,6 +19,7 @@ import path from 'node:path' import crypto from 'node:crypto' import { redactSecrets } from '../core/security.mjs' import { resolveStoreDb } from '../db/database.mjs' +import { integerValue } from '../db/integers.mjs' import { UsageLogsRepo } from '../db/repos/usage-logs-repo.mjs' import { classifyRequestError, resolveMutedErrorClasses } from './error-class.mjs' import { costColumnsFromUsage, normalizeUsage } from './pricing.mjs' @@ -114,9 +115,9 @@ function cacheCreationBreakdown(usage) { return { cache_creation_5m_tokens: null, cache_creation_1h_tokens: null } } const nested = usage.cache_creation - let five = Number(nested?.ephemeral_5m_input_tokens) || 0 - let hour = Number(nested?.ephemeral_1h_input_tokens) || 0 - const total = Number(usage.cache_creation_input_tokens ?? usage.cache_creation_tokens) || 0 + let five = integerValue(nested?.ephemeral_5m_input_tokens) + let hour = integerValue(nested?.ephemeral_1h_input_tokens) + const total = integerValue(usage.cache_creation_input_tokens ?? usage.cache_creation_tokens) if (five === 0 && hour === 0 && total > 0) { if (usage.cache_ttl === '1h') hour = total else five = total @@ -128,8 +129,8 @@ function cacheCreationBreakdown(usage) { } function webSearchRequestsOf(usage) { - const n = Number(usage?.server_tool_use?.web_search_requests ?? usage?.web_search_requests) - return Number.isFinite(n) && n > 0 ? Math.trunc(n) : 0 + const n = integerValue(usage?.server_tool_use?.web_search_requests ?? usage?.web_search_requests) + return n > 0 ? n : 0 } const PRICED_COLUMN_KEYS = [ @@ -388,6 +389,7 @@ export class RequestLogStore { this._mem = [] // recent normal summaries for panel hot path this._memMax = 200 this._cleanupTimer = null + this._cleanupPending = new Set() } async setConfig({ mode, retainDays, debugRetainDays, maxMb, mutedErrorClasses } = {}) { @@ -502,9 +504,10 @@ export class RequestLogStore { async finish(ctx, extra = {}) { if (!ctx || ctx.mode === 'off') return null const duration_ms = Math.max(0, Date.now() - (ctx.t0 || Date.now())) - const usage = normalizeUsage(disjointOpenaiUsage(extra.usage || {}) || {}) + const rawUsage = disjointOpenaiUsage(extra.usage || {}) || {} + const usage = normalizeUsage(rawUsage) const breakdown = cacheCreationBreakdown(usage) - const webSearch = extra.web_search_requests ?? webSearchRequestsOf(usage) + const webSearch = extra.web_search_requests ?? webSearchRequestsOf(rawUsage) const billedUsage = { ...usage, input_tokens: extra.input_tokens ?? usage.input_tokens, @@ -764,10 +767,14 @@ export class RequestLogStore { startCleanupScheduler({ intervalMs = CLEANUP_INTERVAL_MS } = {}) { this.stopCleanupScheduler() const ms = Math.max(60_000, Number(intervalMs) || CLEANUP_INTERVAL_MS) - const run = async () => { - try { - await this.cleanup() - } catch {} + const run = () => { + const pending = (async () => { + try { + await this.cleanup() + } catch {} + })() + this._cleanupPending.add(pending) + pending.finally(() => this._cleanupPending.delete(pending)) } this._cleanupTimer = setInterval(run, ms) if (typeof this._cleanupTimer.unref === 'function') this._cleanupTimer.unref() @@ -778,6 +785,10 @@ export class RequestLogStore { this._cleanupTimer = null } + async flush() { + while (this._cleanupPending.size) await Promise.all([...this._cleanupPending]) + } + _cleanupLegacyFiles(days, dir) { const cutoff = Date.now() - days * 86400_000 try { diff --git a/src/lib/db/integers.mjs b/src/lib/db/integers.mjs new file mode 100644 index 000000000..1e21dc661 --- /dev/null +++ b/src/lib/db/integers.mjs @@ -0,0 +1,19 @@ +/** Exact arithmetic for PostgreSQL bigint counters exposed as numbers or decimal strings. */ +const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER) + +function integer(value) { + if (value == null || value === '') return 0n + if (typeof value === 'number' && !Number.isSafeInteger(value)) { + throw new RangeError('counter must be a safe integer number or an exact decimal string') + } + return BigInt(value) +} + +export function addInteger(left, right) { + const result = integer(left) + integer(right) + return result >= -MAX_SAFE && result <= MAX_SAFE ? Number(result) : result.toString() +} + +export function integerValue(value) { + return addInteger(value, 0) +} diff --git a/src/lib/db/repos/accounts-repo.mjs b/src/lib/db/repos/accounts-repo.mjs index 7d74c7f02..fd707673a 100644 --- a/src/lib/db/repos/accounts-repo.mjs +++ b/src/lib/db/repos/accounts-repo.mjs @@ -9,6 +9,7 @@ */ import { getDb, withTransaction } from '../database.mjs' +import { integerValue } from '../integers.mjs' const MAX_ALLOCATIONS_PER_ACCOUNT = 50 @@ -136,11 +137,11 @@ export class AccountsRepo { acc.concurrency_override ? 1 : 0, acc.max_rpm ?? 0, acc.rpm_override ? 1 : 0, - acc.requests ?? 0, - acc.tokens_in ?? 0, - acc.tokens_out ?? 0, - acc.cache_read_tokens ?? 0, - acc.cache_creation_tokens ?? 0, + integerValue(acc.requests), + integerValue(acc.tokens_in), + integerValue(acc.tokens_out), + integerValue(acc.cache_read_tokens), + integerValue(acc.cache_creation_tokens), buildExtra(acc), acc.created_at ?? now, acc.updated_at ?? now, @@ -161,11 +162,11 @@ export class AccountsRepo { acc.concurrency_override ? 1 : 0, acc.max_rpm ?? 0, acc.rpm_override ? 1 : 0, - acc.requests ?? 0, - acc.tokens_in ?? 0, - acc.tokens_out ?? 0, - acc.cache_read_tokens ?? 0, - acc.cache_creation_tokens ?? 0, + integerValue(acc.requests), + integerValue(acc.tokens_in), + integerValue(acc.tokens_out), + integerValue(acc.cache_read_tokens), + integerValue(acc.cache_creation_tokens), buildExtra(acc), new Date().toISOString(), acc.account_id, @@ -193,8 +194,8 @@ export class AccountsRepo { alloc.status_5h ?? null, alloc.status_7d ?? null, alloc.claim ?? null, - alloc.tokens_in ?? null, - alloc.tokens_out ?? null, + alloc.tokens_in == null ? null : integerValue(alloc.tokens_in), + alloc.tokens_out == null ? null : integerValue(alloc.tokens_out), ) await this._trimAlloc.run(accountId, accountId, max) }) diff --git a/src/lib/db/repos/api-keys-repo.mjs b/src/lib/db/repos/api-keys-repo.mjs index d3aafd340..99177b264 100644 --- a/src/lib/db/repos/api-keys-repo.mjs +++ b/src/lib/db/repos/api-keys-repo.mjs @@ -8,6 +8,7 @@ */ import { getDb, withTransaction } from '../database.mjs' +import { integerValue } from '../integers.mjs' const COLUMNS = [ 'id', @@ -75,6 +76,16 @@ const NUMERIC_DEFAULT_0 = new Set([ 'usage_7d', ]) +const INTEGER_COUNTERS = new Set([ + 'quota_requests', + 'quota_requests_used', + 'requests', + 'tokens_in', + 'tokens_out', + 'cache_read_tokens', + 'cache_creation_tokens', +]) + function rowToRec(row) { if (!row) return null return { ...row } @@ -88,6 +99,7 @@ function toValue(rec, c) { if (c === 'allowed_vms') return '[]' return NUMERIC_DEFAULT_0.has(c) ? 0 : null } + if (INTEGER_COUNTERS.has(c)) return integerValue(v) return v } @@ -185,10 +197,10 @@ export class ApiKeysRepo { const usd = Number(cost) || 0 const w = WINDOWS.map((win) => this._rollWindow(rec, win, usd, nowMs)) await this._recordUsage.run( - Number(tokens_in) || 0, - Number(tokens_out) || 0, - Number(cache_read_tokens) || 0, - Number(cache_creation_tokens) || 0, + integerValue(tokens_in), + integerValue(tokens_out), + integerValue(cache_read_tokens), + integerValue(cache_creation_tokens), usd, w[0].usage, w[0].start, diff --git a/src/lib/db/repos/statistics-repo.mjs b/src/lib/db/repos/statistics-repo.mjs index 1795f0f01..44ac9d1dd 100644 --- a/src/lib/db/repos/statistics-repo.mjs +++ b/src/lib/db/repos/statistics-repo.mjs @@ -11,8 +11,16 @@ */ import { getDb } from '../database.mjs' +import { addInteger, integerValue } from '../integers.mjs' import { reportTimezone, zonedDayStartMs, zonedParts, zonedWallToMs } from '../../core/timezone.mjs' -import { ERROR_PRED, PROMPT_TOKENS_SQL, SUCCESS_PRED, TOTAL_TOKENS_SQL, ownerPred } from './usage-log-preds.mjs' +import { + ERROR_PRED, + PROMPT_TOKENS_SQL, + SUCCESS_PRED, + TOTAL_TOKENS_SQL, + ownerPred, + totalTokens, +} from './usage-log-preds.mjs' import { lookupNames } from './usage-logs-view.mjs' export const STATS_RANGES = ['today', '7days', '30days', 'thisMonth'] @@ -133,17 +141,17 @@ export class StatisticsRepo { FROM usage_logs ${cond} `) .get(...params) - const input = num(row.input_tokens) - const output = num(row.output_tokens) - const cacheRead = num(row.cache_read_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const prompt = num(row.prompt_tokens) + const input = integerValue(row.input_tokens) + const output = integerValue(row.output_tokens) + const cacheRead = integerValue(row.cache_read_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const prompt = integerValue(row.prompt_tokens) return { - requests: num(row.requests), - errors: num(row.errors), + requests: integerValue(row.requests), + errors: integerValue(row.errors), cost: num(row.cost), actualCost: num(row.actual_cost), - tokens: input + output + cacheRead + cacheCreation, + tokens: totalTokens(row), inputTokens: input, outputTokens: output, cacheReadTokens: cacheRead, @@ -194,13 +202,14 @@ export class StatisticsRepo { row.id = id const cur = volume.get(id) || { id, cost: 0, requests: 0, tokens: 0 } cur.cost += num(row.cost) - cur.requests += num(row.requests) - cur.tokens += num(row.tokens) + cur.requests = addInteger(cur.requests, row.requests) + cur.tokens = addInteger(cur.tokens, row.tokens) volume.set(id, cur) } - const ranked = [...volume.values()].sort( - (a, b) => b.cost - a.cost || b.requests - a.requests || a.id.localeCompare(b.id), - ) + const ranked = [...volume.values()].sort((a, b) => { + const requests = BigInt(b.requests) - BigInt(a.requests) + return b.cost - a.cost || (requests > 0n ? 1 : requests < 0n ? -1 : 0) || a.id.localeCompare(b.id) + }) const kept = ranked.slice(0, MAX_SERIES) const names = await this._names( dim, @@ -227,9 +236,13 @@ export class StatisticsRepo { if (!dataKey) continue const idx = bucketIndex(buckets, Date.parse(`${row.slot}:00.000Z`)) for (const m of metricNames) { - const v = num(row[m]) - metrics[m][idx][dataKey] += v - seriesTotals[dataKey][m] += v + if (m === 'cost') { + metrics[m][idx][dataKey] += num(row[m]) + seriesTotals[dataKey][m] += num(row[m]) + } else { + metrics[m][idx][dataKey] = addInteger(metrics[m][idx][dataKey], row[m]) + seriesTotals[dataKey][m] = addInteger(seriesTotals[dataKey][m], row[m]) + } } } @@ -298,20 +311,20 @@ export class StatisticsRepo { range: r, tz: zone, entries: rows.map((row, i) => { - const input = num(row.input_tokens) - const output = num(row.output_tokens) - const cacheRead = num(row.cache_read_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const requests = num(row.requests) - const prompt = num(row.prompt_tokens) + const input = integerValue(row.input_tokens) + const output = integerValue(row.output_tokens) + const cacheRead = integerValue(row.cache_read_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const requests = integerValue(row.requests) + const prompt = integerValue(row.prompt_tokens) return { id: ids[i], name: this._nameOf(names, ids[i]), totalRequests: requests, - successRequests: num(row.success), - errorRequests: num(row.errors), - successRate: requests > 0 ? num(row.success) / requests : 0, - totalTokens: input + output + cacheRead + cacheCreation, + successRequests: integerValue(row.success), + errorRequests: integerValue(row.errors), + successRate: requests > 0 ? integerValue(row.success) / requests : 0, + totalTokens: totalTokens(row), inputTokens: input, outputTokens: output, cacheReadTokens: cacheRead, diff --git a/src/lib/db/repos/usage-log-preds.mjs b/src/lib/db/repos/usage-log-preds.mjs index 893824e22..28bc49b1e 100644 --- a/src/lib/db/repos/usage-log-preds.mjs +++ b/src/lib/db/repos/usage-log-preds.mjs @@ -5,6 +5,7 @@ */ import { ignoredErrorSqlList, slaOkErrorSqlList } from '../../admin/error-class.mjs' +import { addInteger } from '../integers.mjs' const IGNORED_CODES_SQL = ignoredErrorSqlList() const SLA_OK_CODES_SQL = slaOkErrorSqlList() @@ -23,11 +24,9 @@ export const TOTAL_TOKENS_SQL = 'COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)' export function totalTokens(row = {}) { - return ( - (Number(row.input_tokens) || 0) + - (Number(row.output_tokens) || 0) + - (Number(row.cache_read_tokens) || 0) + - (Number(row.cache_creation_tokens) || 0) + return [row.input_tokens, row.output_tokens, row.cache_read_tokens, row.cache_creation_tokens].reduce( + (sum, value) => addInteger(sum, value), + 0, ) } diff --git a/src/lib/db/repos/usage-logs-repo.mjs b/src/lib/db/repos/usage-logs-repo.mjs index 8fd8ea17b..381ce3d60 100644 --- a/src/lib/db/repos/usage-logs-repo.mjs +++ b/src/lib/db/repos/usage-logs-repo.mjs @@ -9,6 +9,7 @@ */ import { getDb, withTransaction } from '../database.mjs' +import { addInteger, integerValue } from '../integers.mjs' import { collectErrors, enrichLogRow, excludeErrorClassSql, ingressAuthSql } from '../../admin/error-class.mjs' import { costColumnsFromUsage, @@ -56,10 +57,24 @@ function billingStatsCacheKey(accountWindows) { // Do not re-apply a single Anthropic formula on the mixed SUM (homepage/usage // billing). Database-page PromptCacheMetricsPanel still uses input+read+write. function sumCostBuckets(rows) { - return { - ...sumCostColumns(rows), - prompt_tokens: rows.reduce((sum, row) => sum + (Number(row.prompt_tokens) || 0), 0), + const out = sumCostColumns(rows) + for (const key of [ + 'requests', + 'success', + 'errors', + 'input_tokens', + 'output_tokens', + 'cache_read_tokens', + 'cache_creation_tokens', + 'prompt_tokens', + ]) { + out[key] = rows.reduce((sum, row) => addInteger(sum, row[key]), 0) } + return out +} + +function withCacheHitRate(bucket) { + return { ...bucket, cache_hit_rate: cacheHitStats(bucket).cache_hit_rate } } const SUMMARY_COLUMNS = [ @@ -140,7 +155,7 @@ function toRow(rec) { c === 'cache_creation_1h_tokens' || c === 'web_search_requests' ) - return Number(rec[c]) || 0 + return integerValue(rec[c]) return rec[c] ?? null }) } @@ -216,13 +231,13 @@ function bucketEventsSince(rows = [], sinceMs) { ...sumCostBuckets( picked.map((row) => ({ requests: 1, - success: Number(row.success || 0), - errors: Number(row.errors || 0), - input_tokens: Number(row.input_tokens || 0), - output_tokens: Number(row.output_tokens || 0), - cache_read_tokens: Number(row.cache_read_tokens || 0), - cache_creation_tokens: Number(row.cache_creation_tokens || 0), - prompt_tokens: Number(row.prompt_tokens || 0), + success: integerValue(row.success), + errors: integerValue(row.errors), + input_tokens: integerValue(row.input_tokens), + output_tokens: integerValue(row.output_tokens), + cache_read_tokens: integerValue(row.cache_read_tokens), + cache_creation_tokens: integerValue(row.cache_creation_tokens), + prompt_tokens: integerValue(row.prompt_tokens), input_cost: Number(row.input_cost || 0), output_cost: Number(row.output_cost || 0), cache_read_cost: Number(row.cache_read_cost || 0), @@ -574,7 +589,7 @@ export class UsageLogsRepo { FROM usage_logs `) .get() - return { ...row, ...cacheHitStats(row) } + return withCacheHitRate(row) } _usageForPrice(row) { @@ -747,7 +762,7 @@ export class UsageLogsRepo { .all(...params) ).map((r) => ({ model: r.model, - requests: Number(r.requests || 0), + requests: integerValue(r.requests), last_seen: r.last_seen || null, })) } @@ -772,20 +787,20 @@ export class UsageLogsRepo { `) .get(...params) const out = { - requests: Number(row?.requests || 0), - input_tokens: Number(row?.input_tokens || 0), - output_tokens: Number(row?.output_tokens || 0), - cache_read_tokens: Number(row?.cache_read_tokens || 0), - cache_creation_tokens: Number(row?.cache_creation_tokens || 0), - prompt_tokens: Number(row?.prompt_tokens || 0), + requests: integerValue(row?.requests), + input_tokens: integerValue(row?.input_tokens), + output_tokens: integerValue(row?.output_tokens), + cache_read_tokens: integerValue(row?.cache_read_tokens), + cache_creation_tokens: integerValue(row?.cache_creation_tokens), + prompt_tokens: integerValue(row?.prompt_tokens), input_cost: Number(row?.input_cost || 0), output_cost: Number(row?.output_cost || 0), cache_read_cost: Number(row?.cache_read_cost || 0), cache_creation_cost: Number(row?.cache_creation_cost || 0), total_cost: Number(row?.total_cost || 0), - unpriced: Number(row?.unpriced || 0), + unpriced: integerValue(row?.unpriced), } - return { ...out, ...cacheHitStats(out) } + return withCacheHitRate(out) } async costByAccount({ since = null, until = null } = {}) { @@ -817,20 +832,20 @@ export class UsageLogsRepo { return rows.map((r) => ({ account_id: r.account_id, vm_id: r.vm_id || null, - requests: Number(r.requests || 0), - success: Number(r.success || 0), - errors: Number(r.errors || 0), - input_tokens: Number(r.input_tokens || 0), - output_tokens: Number(r.output_tokens || 0), - cache_read_tokens: Number(r.cache_read_tokens || 0), - cache_creation_tokens: Number(r.cache_creation_tokens || 0), - prompt_tokens: Number(r.prompt_tokens || 0), + requests: integerValue(r.requests), + success: integerValue(r.success), + errors: integerValue(r.errors), + input_tokens: integerValue(r.input_tokens), + output_tokens: integerValue(r.output_tokens), + cache_read_tokens: integerValue(r.cache_read_tokens), + cache_creation_tokens: integerValue(r.cache_creation_tokens), + prompt_tokens: integerValue(r.prompt_tokens), input_cost: Number(r.input_cost || 0), output_cost: Number(r.output_cost || 0), cache_read_cost: Number(r.cache_read_cost || 0), cache_creation_cost: Number(r.cache_creation_cost || 0), total_cost: Number(r.total_cost || 0), - unpriced: Number(r.unpriced || 0), + unpriced: integerValue(r.unpriced), })) } @@ -874,13 +889,13 @@ export class UsageLogsRepo { service_tier: r.service_tier || null, speed: r.speed || null, long_context: Number(r.long_context || 0), - requests: Number(r.requests || 0), - unpriced_requests: Number(r.unpriced_requests || 0), - input_tokens: Number(r.input_tokens || 0), - output_tokens: Number(r.output_tokens || 0), - cache_read_tokens: Number(r.cache_read_tokens || 0), - cache_creation_tokens: Number(r.cache_creation_tokens || 0), - prompt_tokens: Number(r.prompt_tokens || 0), + requests: integerValue(r.requests), + unpriced_requests: integerValue(r.unpriced_requests), + input_tokens: integerValue(r.input_tokens), + output_tokens: integerValue(r.output_tokens), + cache_read_tokens: integerValue(r.cache_read_tokens), + cache_creation_tokens: integerValue(r.cache_creation_tokens), + prompt_tokens: integerValue(r.prompt_tokens), input_cost: Number(r.input_cost || 0), output_cost: Number(r.output_cost || 0), cache_read_cost: Number(r.cache_read_cost || 0), @@ -942,16 +957,16 @@ export class UsageLogsRepo { .all(...params) ).map((r) => ({ day: r.day, - requests: Number(r.requests || 0), - errors: Number(r.errors || 0), - input_tokens: Number(r.input_tokens || 0), - output_tokens: Number(r.output_tokens || 0), - cache_read_tokens: Number(r.cache_read_tokens || 0), - cache_creation_tokens: Number(r.cache_creation_tokens || 0), - prompt_tokens: Number(r.prompt_tokens || 0), + requests: integerValue(r.requests), + errors: integerValue(r.errors), + input_tokens: integerValue(r.input_tokens), + output_tokens: integerValue(r.output_tokens), + cache_read_tokens: integerValue(r.cache_read_tokens), + cache_creation_tokens: integerValue(r.cache_creation_tokens), + prompt_tokens: integerValue(r.prompt_tokens), total_cost: Number(r.total_cost || 0), - duration_ms_sum: Number(r.duration_ms_sum || 0), - duration_n: Number(r.duration_n || 0), + duration_ms_sum: integerValue(r.duration_ms_sum), + duration_n: integerValue(r.duration_n), })) const group = async (expr, limit) => ( @@ -967,8 +982,8 @@ export class UsageLogsRepo { .all(...params) ).map((r) => ({ name: r.name, - requests: Number(r.requests || 0), - tokens: Number(r.tokens || 0), + requests: integerValue(r.requests), + tokens: integerValue(r.tokens), total_cost: Number(r.total_cost || 0), })) return { @@ -1039,9 +1054,9 @@ export class UsageLogsRepo { const folded = foldInterceptRows(grouped) return { since, - total: Number(row?.total || 0), - blocked: Number(row?.blocked || 0), - passed: Number(row?.passed || 0), + total: integerValue(row?.total), + blocked: integerValue(row?.blocked), + passed: integerValue(row?.passed), keywords: collapseBlockKeywords(keywords), blocks: folded.blocks, passes: folded.passes, @@ -1093,8 +1108,8 @@ export class UsageLogsRepo { // blocks unrelated HTTP requests on the Node event loop. const totalBucket = sumCostBuckets(accountsTotal) const todayBucket = sumCostBuckets(accountsToday) - const total = { ...totalBucket, ...cacheHitStats(totalBucket) } - const today = { ...todayBucket, ...cacheHitStats(todayBucket) } + const total = withCacheHitRate(totalBucket) + const today = withCacheHitRate(todayBucket) const keyOf = (a) => a.account_id + '\0' + (a.vm_id || '') const todayById = new Map(accountsToday.map((a) => [keyOf(a), a])) const aligned = Array.isArray(accountWindows) && accountWindows.length > 0 @@ -1139,8 +1154,8 @@ export class UsageLogsRepo { window_fable = { cost: Number(fable.total_cost || 0), tokens: totalTokens(fable), - requests: Number(fable.requests || 0), - unpriced: Number(fable.unpriced || 0), + requests: integerValue(fable.requests), + unpriced: integerValue(fable.unpriced), start: new Date(sinceFable).toISOString(), } } @@ -1150,47 +1165,41 @@ export class UsageLogsRepo { window_5h: { ...w }, window_7d: { ...w7 }, today_cost: Number(t.total_cost || 0), - today_requests: Number(t.requests || 0), - today_input_tokens: Number(t.input_tokens || 0), - today_output_tokens: Number(t.output_tokens || 0), - today_cache_read_tokens: Number(t.cache_read_tokens || 0), - today_cache_creation_tokens: Number(t.cache_creation_tokens || 0), + today_requests: integerValue(t.requests), + today_input_tokens: integerValue(t.input_tokens), + today_output_tokens: integerValue(t.output_tokens), + today_cache_read_tokens: integerValue(t.cache_read_tokens), + today_cache_creation_tokens: integerValue(t.cache_creation_tokens), today_input_cost: Number(t.input_cost || 0), today_output_cost: Number(t.output_cost || 0), today_cache_cost: Number(t.cache_read_cost || 0) + Number(t.cache_creation_cost || 0), window_5h_cost: Number(w.total_cost || 0), - window_5h_requests: Number(w.requests || 0), + window_5h_requests: integerValue(w.requests), window_5h_tokens: totalTokens(w), - window_5h_unpriced: Number(w.unpriced || 0), - window_5h_input_tokens: Number(w.input_tokens || 0), - window_5h_output_tokens: Number(w.output_tokens || 0), - window_5h_cache_read_tokens: Number(w.cache_read_tokens || 0), - window_5h_cache_creation_tokens: Number(w.cache_creation_tokens || 0), + window_5h_unpriced: integerValue(w.unpriced), + window_5h_input_tokens: integerValue(w.input_tokens), + window_5h_output_tokens: integerValue(w.output_tokens), + window_5h_cache_read_tokens: integerValue(w.cache_read_tokens), + window_5h_cache_creation_tokens: integerValue(w.cache_creation_tokens), window_5h_input_cost: Number(w.input_cost || 0), window_5h_output_cost: Number(w.output_cost || 0), window_5h_cache_cost: Number(w.cache_read_cost || 0) + Number(w.cache_creation_cost || 0), window_5h_start: aligned ? new Date(since5).toISOString() : fiveHStart, window_7d_cost: Number(w7.total_cost || 0), - window_7d_requests: Number(w7.requests || 0), - window_7d_success: Number(w7.success || 0), - window_7d_errors: Number(w7.errors || 0), + window_7d_requests: integerValue(w7.requests), + window_7d_success: integerValue(w7.success), + window_7d_errors: integerValue(w7.errors), window_7d_tokens: totalTokens(w7), - window_7d_unpriced: Number(w7.unpriced || 0), + window_7d_unpriced: integerValue(w7.unpriced), window_7d_start: aligned ? new Date(since7).toISOString() : sevenDStart, window_fable, } }) const window5h = aligned - ? { - ...sumCostBuckets(accounts.map((a) => a.window_5h)), - ...cacheHitStats(sumCostBuckets(accounts.map((a) => a.window_5h))), - } + ? withCacheHitRate(sumCostBuckets(accounts.map((a) => a.window_5h))) : await this._costSelect('WHERE created_at >= $1', [fiveHStart]) const window7d = aligned - ? { - ...sumCostBuckets(accounts.map((a) => a.window_7d)), - ...cacheHitStats(sumCostBuckets(accounts.map((a) => a.window_7d))), - } + ? withCacheHitRate(sumCostBuckets(accounts.map((a) => a.window_7d))) : await this._costSelect('WHERE created_at >= $1', [sevenDStart]) const coverage = await this.logCoverageStart() return { @@ -1319,22 +1328,22 @@ export class UsageLogsRepo { since: since ? new Date(since).toISOString() : null, until: until ? new Date(until).toISOString() : null, totals: { - requests: num(totals, 'requests'), - ok: num(totals, 'ok'), - fail: num(totals, 'fail'), - tokens_in: num(totals, 'tokens_in'), - tokens_out: num(totals, 'tokens_out'), + requests: integerValue(totals.requests), + ok: integerValue(totals.ok), + fail: integerValue(totals.fail), + tokens_in: integerValue(totals.tokens_in), + tokens_out: integerValue(totals.tokens_out), cost_usd: num(totals, 'cost_usd'), official_cost_usd: num(totals, 'official_cost_usd'), }, items: items.map((row) => ({ vm_id: grouped === 'vm' ? row.id || null : undefined, api_key_id: grouped === 'key' ? row.id || null : undefined, - requests: num(row, 'requests'), - ok: num(row, 'ok'), - fail: num(row, 'fail'), - tokens_in: num(row, 'tokens_in'), - tokens_out: num(row, 'tokens_out'), + requests: integerValue(row.requests), + ok: integerValue(row.ok), + fail: integerValue(row.fail), + tokens_in: integerValue(row.tokens_in), + tokens_out: integerValue(row.tokens_out), cost_usd: num(row, 'cost_usd'), official_cost_usd: num(row, 'official_cost_usd'), })), @@ -1374,20 +1383,20 @@ export class UsageLogsRepo { FROM usage_logs ${where} `) .get(...allParams) - const requests = Number(counts?.requests || 0) - const success = Number(counts?.success || 0) - const errors = Number(counts?.errors || 0) - const inputTokens = Number(counts?.input_tokens || 0) - const outputTokens = Number(counts?.output_tokens || 0) - const cacheRead = Number(counts?.cache_read_tokens || 0) - const cacheWrite = Number(counts?.cache_creation_tokens || 0) - const cache = cacheHitStats({ + const requests = integerValue(counts?.requests) + const success = integerValue(counts?.success) + const errors = integerValue(counts?.errors) + const inputTokens = integerValue(counts?.input_tokens) + const outputTokens = integerValue(counts?.output_tokens) + const cacheRead = integerValue(counts?.cache_read_tokens) + const cacheWrite = integerValue(counts?.cache_creation_tokens) + const cache = withCacheHitRate({ input_tokens: inputTokens, cache_read_tokens: cacheRead, cache_creation_tokens: cacheWrite, - prompt_tokens: Number(counts?.prompt_tokens || 0), + prompt_tokens: integerValue(counts?.prompt_tokens), }) - const tokens = inputTokens + outputTokens + cacheRead + cacheWrite + const tokens = totalTokens(counts) const startMs = since ? Date.parse(since) : null const endMs = until ? Date.parse(until) : Date.now() @@ -1403,8 +1412,8 @@ export class UsageLogsRepo { GROUP BY minute `) .all(...allParams) - const peakReq = minuteRows.reduce((m, r) => Math.max(m, Number(r.requests || 0)), 0) - const peakTok = minuteRows.reduce((m, r) => Math.max(m, Number(r.tokens || 0)), 0) + const peakReq = minuteRows.reduce((m, r) => Math.max(m, integerValue(r.requests)), 0) + const peakTok = minuteRows.reduce((m, r) => Math.max(m, integerValue(r.tokens)), 0) const lastMinute = minuteRows.length ? minuteRows[minuteRows.length - 1] : null const ttftWhere = where ? `${where} AND first_token_ms IS NOT NULL` : 'WHERE first_token_ms IS NOT NULL' @@ -1440,10 +1449,10 @@ export class UsageLogsRepo { errors, sla: requests ? success / requests : 1, error_rate: requests ? errors / requests : 0, - status_429: Number(counts?.status_429 || 0), - status_529: Number(counts?.status_529 || 0), - status_503: Number(counts?.status_503 || 0), - stream_requests: Number(counts?.stream_requests || 0), + status_429: integerValue(counts?.status_429), + status_529: integerValue(counts?.status_529), + status_503: integerValue(counts?.status_503), + stream_requests: integerValue(counts?.stream_requests), input_tokens: inputTokens, output_tokens: outputTokens, cache_read_tokens: cacheRead, @@ -1466,9 +1475,9 @@ export class UsageLogsRepo { ttft: summarizeLatency(ttfts), by_model: byModel.map((r) => ({ model: r.model, - requests: Number(r.requests || 0), - errors: Number(r.errors || 0), - ttft_samples: Number(r.ttft_samples || 0), + requests: integerValue(r.requests), + errors: integerValue(r.errors), + ttft_samples: integerValue(r.ttft_samples), avg_first_token_ms: r.avg_first_token_ms ?? null, avg_duration_ms: r.avg_duration_ms ?? null, })), @@ -1505,9 +1514,9 @@ export class UsageLogsRepo { let selections = 0 let hits = 0 for (const row of rows) { - const n = Number(row.n || 0) - selections += n - if (String(row.selection_reason || '') === 'sticky') hits += n + const n = integerValue(row.n) + selections = addInteger(selections, n) + if (String(row.selection_reason || '') === 'sticky') hits = addInteger(hits, n) } return { selections, @@ -1563,7 +1572,7 @@ export class UsageLogsRepo { ).map((row) => ({ api_key_presented: row.api_key_presented || '', ip: row.ip || '', - count: Number(row.count || 0), + count: integerValue(row.count), last_ts: row.last_ts || null, error_code: row.error_code || null, })) @@ -1616,11 +1625,11 @@ export class UsageLogsRepo { } async debugBytes() { - return Number((await this._debugBytes.get()).n || 0) + return integerValue((await this._debugBytes.get()).n) } async countDebug() { - return Number((await this._debugCount.get()).n || 0) + return integerValue((await this._debugCount.get()).n) } /** Delete oldest debug rows until the stored JSON bytes fit maxBytes. */ diff --git a/src/lib/db/repos/usage-logs-view.mjs b/src/lib/db/repos/usage-logs-view.mjs index 52e9fcb42..91a91c09c 100644 --- a/src/lib/db/repos/usage-logs-view.mjs +++ b/src/lib/db/repos/usage-logs-view.mjs @@ -10,6 +10,7 @@ */ import { getDb } from '../database.mjs' +import { addInteger, integerValue } from '../integers.mjs' import { DEFAULT_MUTED_ERROR_CLASSES, enrichLogRow, excludeErrorClassSql } from '../../admin/error-class.mjs' import { reportTimezone, zonedDayStartMs } from '../../core/timezone.mjs' import { @@ -105,8 +106,8 @@ export async function lookupNames(db, { userIds = [], keyIds = [], vmIds = [], a } function cacheTtlApplied(row) { - const short = num(row.cache_creation_5m_tokens) - const long = num(row.cache_creation_1h_tokens) + const short = integerValue(row.cache_creation_5m_tokens) + const long = integerValue(row.cache_creation_1h_tokens) if (short > 0 && long > 0) return 'mixed' if (long > 0) return '1h' if (short > 0) return '5m' @@ -155,10 +156,10 @@ function blockedBy(row) { /** Raw `usage_logs` row (already `enrichLogRow`-ed) + batched names → `UsageLogRow`. */ export function toUsageLogRow(row, names, chain = []) { const accountId = row.final_account_id || row.account_id || null - const inputTokens = num(row.input_tokens) - const outputTokens = num(row.output_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const cacheRead = num(row.cache_read_tokens) + const inputTokens = integerValue(row.input_tokens) + const outputTokens = integerValue(row.output_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const cacheRead = integerValue(row.cache_read_tokens) const blocked = blockedBy(row) return { id: row.id, @@ -188,8 +189,8 @@ export function toUsageLogRow(row, names, chain = []) { outputTokens, cacheCreationInputTokens: cacheCreation, cacheReadInputTokens: cacheRead, - cacheCreation5mInputTokens: num(row.cache_creation_5m_tokens), - cacheCreation1hInputTokens: num(row.cache_creation_1h_tokens), + cacheCreation5mInputTokens: integerValue(row.cache_creation_5m_tokens), + cacheCreation1hInputTokens: integerValue(row.cache_creation_1h_tokens), cacheTtlApplied: cacheTtlApplied(row), totalTokens: rowTotalTokens(row), costUsd: nullableNum(row.total_cost), @@ -199,7 +200,7 @@ export function toUsageLogRow(row, names, chain = []) { durationMs: nullableNum(row.duration_ms), ttftMs: nullableNum(row.first_token_ms), stopReason: row.stop_reason ?? null, - attemptCount: nullableNum(row.attempt_count), + attemptCount: row.attempt_count == null ? null : integerValue(row.attempt_count), finalState: row.final_state ?? null, via: row.via ?? null, stream: row.stream == null ? null : !!row.stream, @@ -557,7 +558,7 @@ export class UsageLogsView { 'cache_creation_1h_tokens', 'prompt_tokens', ]) - acc[k] += num(r[k]) + acc[k] = addInteger(acc[k], r[k]) if (r.duration_ms != null) { durSum += num(r.duration_ms) durN++ @@ -741,7 +742,7 @@ export class UsageLogsView { }) return { minutes: mins, - total: num(total), + total: integerValue(total), sessions: rows.map((r) => ({ sessionId: r.sid, clientSessionId: r.client_session_id ?? null, @@ -753,10 +754,10 @@ export class UsageLogsView { model: r.model ?? null, firstAt: r.first_at, lastAt: r.last_at, - requests: num(r.requests), + requests: integerValue(r.requests), lastStatus: nullableNum(r.status), lastDurationMs: nullableNum(r.duration_ms), - totalTokens: num(r.tokens), + totalTokens: integerValue(r.tokens), totalCost: num(r.cost), })), } @@ -798,14 +799,14 @@ export class UsageLogsView { const lastMinute = await agg(now - 60_000, now + 1) const avg = (v) => (v == null ? null : Math.round(Number(v))) return { - activeSessions: num(active.sessions), - rpm: num(lastMinute.requests), - todayRequests: num(today.requests), + activeSessions: integerValue(active.sessions), + rpm: integerValue(lastMinute.requests), + todayRequests: integerValue(today.requests), todayCost: num(today.cost), todayActualCost: num(today.actual_cost), - todayErrors: num(today.errors), + todayErrors: integerValue(today.errors), todayAvgDurationMs: avg(today.avg_duration_ms), - yesterdayRequests: num(yesterday.requests), + yesterdayRequests: integerValue(yesterday.requests), yesterdayCost: num(yesterday.cost), yesterdayAvgDurationMs: avg(yesterday.avg_duration_ms), } @@ -813,15 +814,15 @@ export class UsageLogsView { } function toSummary(row) { - const input = num(row.input_tokens) - const output = num(row.output_tokens) - const cacheCreation = num(row.cache_creation_tokens) - const cacheRead = num(row.cache_read_tokens) - const prompt = num(row.prompt_tokens) + const input = integerValue(row.input_tokens) + const output = integerValue(row.output_tokens) + const cacheCreation = integerValue(row.cache_creation_tokens) + const cacheRead = integerValue(row.cache_read_tokens) + const prompt = integerValue(row.prompt_tokens) return { - totalRequests: num(row.requests), - successRequests: num(row.success), - errorRequests: num(row.errors), + totalRequests: integerValue(row.requests), + successRequests: integerValue(row.success), + errorRequests: integerValue(row.errors), totalCost: num(row.cost), totalActualCost: num(row.actual_cost), totalTokens: rowTotalTokens(row), @@ -829,8 +830,8 @@ function toSummary(row) { totalOutputTokens: output, totalCacheCreationTokens: cacheCreation, totalCacheReadTokens: cacheRead, - totalCacheCreation5mTokens: num(row.cache_creation_5m_tokens), - totalCacheCreation1hTokens: num(row.cache_creation_1h_tokens), + totalCacheCreation5mTokens: integerValue(row.cache_creation_5m_tokens), + totalCacheCreation1hTokens: integerValue(row.cache_creation_1h_tokens), cacheHitRate: prompt > 0 ? Math.min(1, cacheRead / prompt) : 0, avgDurationMs: row.avg_duration_ms == null ? null : Math.round(Number(row.avg_duration_ms)), avgTtftMs: row.avg_ttft_ms == null ? null : Math.round(Number(row.avg_ttft_ms)), diff --git a/src/lib/db/repos/users-repo.mjs b/src/lib/db/repos/users-repo.mjs index c0b29ed90..48e98d57a 100644 --- a/src/lib/db/repos/users-repo.mjs +++ b/src/lib/db/repos/users-repo.mjs @@ -56,35 +56,44 @@ export class UsersRepo { constructor(db) { this.db = db this._list = db.prepare('SELECT * FROM users WHERE deleted_at IS NULL ORDER BY created_at, username') - this._get = db.prepare('SELECT * FROM users WHERE id = ? AND deleted_at IS NULL') - this._getByName = db.prepare('SELECT * FROM users WHERE username = ? COLLATE NOCASE AND deleted_at IS NULL') + this._get = db.prepare('SELECT * FROM users WHERE id = $1 AND deleted_at IS NULL') + this._getByName = db.prepare( + 'SELECT * FROM users WHERE ascii_fold(username) = ascii_fold($1) AND deleted_at IS NULL', + ) this._insert = db.prepare(` INSERT INTO users (${COLUMNS.join(', ')}) - VALUES (${COLUMNS.map(() => '?').join(', ')}) + VALUES (${COLUMNS.map((_, i) => '$' + (i + 1)).join(', ')}) `) - this._softDelete = db.prepare('UPDATE users SET deleted_at = ?, updated_at = ? WHERE id = ? AND deleted_at IS NULL') + this._softDelete = db.prepare( + 'UPDATE users SET deleted_at = $1, updated_at = $2 WHERE id = $3 AND deleted_at IS NULL', + ) this._countRole = db.prepare( - "SELECT COUNT(*) AS c FROM users WHERE role = ? AND status = 'active' AND deleted_at IS NULL", + "SELECT COUNT(*) AS c FROM users WHERE role = $1 AND status = 'active' AND deleted_at IS NULL", ) this._addBalance = db.prepare( - 'UPDATE users SET balance = balance + ?, updated_at = ? WHERE id = ? AND deleted_at IS NULL', + 'UPDATE users SET balance = balance + $1, updated_at = $2 WHERE id = $3 AND deleted_at IS NULL', ) } - list() { - return this._list.all().map(rowToRec) + async list() { + return (await this._list.all()).map(rowToRec) } - getById(id) { - return rowToRec(this._get.get(id)) + async getById(id, { forUpdate = false } = {}) { + return rowToRec( + await (forUpdate + ? this.db.prepare('SELECT * FROM users WHERE id = $1 AND deleted_at IS NULL FOR UPDATE') + : this._get + ).get(id), + ) } - getByUsername(username) { - return rowToRec(this._getByName.get(String(username || '').trim())) + async getByUsername(username) { + return rowToRec(await this._getByName.get(String(username || '').trim())) } - insert(rec) { - this._insert.run( + async insert(rec) { + await this._insert.run( ...COLUMNS.map((c) => { if (c === 'email') return rec.email ?? `${rec.username}@panel.local` if (c === 'status') return statusOf(rec) @@ -96,13 +105,13 @@ export class UsersRepo { return rec[c] ?? null }), ) - return this.getById(rec.id) + return await this.getById(rec.id) } - update(rec) { + async update(rec) { const cols = COLUMNS.filter((c) => c !== 'id' && c !== 'deleted_at') - const sql = `UPDATE users SET ${cols.map((c) => `${c} = ?`).join(', ')} WHERE id = ? AND deleted_at IS NULL` - this.db.prepare(sql).run( + const sql = `UPDATE users SET ${cols.map((c, i) => `${c} = $${i + 1}`).join(', ')} WHERE id = $${cols.length + 1} AND deleted_at IS NULL` + await this.db.prepare(sql).run( ...cols.map((c) => { if (c === 'email') return rec.email ?? `${rec.username}@panel.local` if (c === 'status') return statusOf(rec) @@ -114,23 +123,27 @@ export class UsersRepo { }), rec.id, ) - return this.getById(rec.id) + return await this.getById(rec.id) } /** Soft delete (sub2api SoftDeleteMixin semantics). */ - remove(id) { + async remove(id) { const now = new Date().toISOString() - return this._softDelete.run(now, now, id).changes > 0 + return (await this._softDelete.run(now, now, id)).changes > 0 } - countEnabledRole(role) { - const row = this._countRole.get(role) + async countEnabledRole(role) { + const row = await this._countRole.get(role) return Number(row?.c || 0) } + async touchLogin(id, at) { + await this.db.prepare('UPDATE users SET last_login_at = $1 WHERE id = $2 AND deleted_at IS NULL').run(at, id) + } + /** Atomic balance credit/debit (redeem, billing). Negative delta = charge. */ - addBalance(id, delta) { - this._addBalance.run(Number(delta) || 0, new Date().toISOString(), id) - return this.getById(id) + async addBalance(id, delta) { + await this._addBalance.run(Number(delta) || 0, new Date().toISOString(), id) + return await this.getById(id) } } diff --git a/test/unit/request-log.test.mjs b/test/unit/request-log.test.mjs index 38aff8991..482fc6937 100644 --- a/test/unit/request-log.test.mjs +++ b/test/unit/request-log.test.mjs @@ -1132,9 +1132,13 @@ test('stopping retention scheduling still waits for its running cleanup during f }) t.mock.method(globalThis, 'clearInterval', () => {}) let releaseCleanup - const cleanupPending = new Promise((resolve) => { releaseCleanup = resolve }) + const cleanupPending = new Promise((resolve) => { + releaseCleanup = resolve + }) let signalCleanup - const cleanupStarted = new Promise((resolve) => { signalCleanup = resolve }) + const cleanupStarted = new Promise((resolve) => { + signalCleanup = resolve + }) store.cleanup = async () => { signalCleanup() await cleanupPending @@ -1146,7 +1150,9 @@ test('stopping retention scheduling still waits for its running cleanup during f let settled = false let pending try { - pending = store.flush().then(() => { settled = true }) + pending = store.flush().then(() => { + settled = true + }) await Promise.resolve() assert.equal(settled, false) } finally { From 777d2e22a5d7d7b623a3152796528bb4850ef3da Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:57:07 +0800 Subject: [PATCH 019/473] test(migration): retain fractional weights in legacy runtime rows --- test/unit/sqlite-import.test.mjs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/unit/sqlite-import.test.mjs b/test/unit/sqlite-import.test.mjs index efbbec726..aa94247af 100644 --- a/test/unit/sqlite-import.test.mjs +++ b/test/unit/sqlite-import.test.mjs @@ -71,3 +71,17 @@ test('fractional scheduling weights survive SQLite import and PostgreSQL updates const record = await new AccountRuntimeRepo(db).upsert({ account_id: 'fractional', vm_id: 'vm-01', weight: 2.75 }) assert.equal(record.weight, 2.75) }) + +test('fractional weights in the preserved legacy runtime table survive import', async () => { + const old = sqlite({ dbPath: source }) + old + .prepare('INSERT INTO account_runtime_states_legacy(account_id,vm_id,weight,updated_at) VALUES(?,?,?,?)') + .run('legacy-fractional', 'vm-01', 1.25, '2026-10-21T00:00:00.000Z') + old.close() + const db = getDb() + await importSqlite({ source, db }) + const row = await db + .prepare('SELECT weight FROM account_runtime_states_legacy WHERE account_id=$1') + .get('legacy-fractional') + assert.equal(row.weight, 1.25) +}) From 3fbdc4bc31307936bf321cd4b6b3c7883ae5433c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:58:26 +0800 Subject: [PATCH 020/473] test(pg): isolate integration tests on PostgreSQL and preserve restart state --- .github/workflows/test.yml | 33 +++++++++++++- scripts/run-isolated-tests.mjs | 18 ++++++-- test/e2e/long-session.e2e.test.mjs | 35 ++++++++------- test/e2e/panel-users.e2e.test.mjs | 7 ++- test/e2e/persistence.e2e.test.mjs | 21 ++++----- test/harness.mjs | 30 ++++++++++++- test/pg-setup.mjs | 70 ++++++++++++++++++++++++++++++ 7 files changed, 175 insertions(+), 39 deletions(-) create mode 100644 test/pg-setup.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b07b3ae0..69782f761 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,20 @@ permissions: jobs: unit: runs-on: ubuntu-latest - timeout-minutes: 8 + timeout-minutes: 20 + services: + postgres: + image: postgres:17-bookworm + env: + POSTGRES_USER: vm2api + POSTGRES_PASSWORD: vm2api-ci-test + POSTGRES_DB: vm2api_test + ports: [5432:5432] + options: >- + --health-cmd "pg_isready -U vm2api -d vm2api_test" + --health-interval 5s --health-timeout 3s --health-retries 12 + env: + VM2API_TEST_DATABASE_URL: postgresql://vm2api:vm2api-ci-test@127.0.0.1:5432/vm2api_test steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 @@ -29,9 +42,22 @@ jobs: with: go-version: '1.25.x' cache-dependency-path: worker/go.sum + - name: Keep disposable database cleanup immediate + run: | + docker exec ${{ job.services.postgres.id }} psql -U vm2api -d vm2api_test -v ON_ERROR_STOP=1 -c 'ALTER SYSTEM SET checkpoint_completion_target=0' -c 'SELECT pg_reload_conf()' - run: npm ci + - name: PostgreSQL 17 backup clients + run: | + sudo install -d /usr/share/postgresql-common/pgdg + curl --fail --silent --show-error https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc >/dev/null + . /etc/os-release + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null + sudo apt-get update + sudo apt-get install -y postgresql-client-17 + echo /usr/lib/postgresql/17/bin >> "$GITHUB_PATH" + /usr/lib/postgresql/17/bin/pg_dump --version - name: panel user and onboarding regression tests - run: node --test test/unit/panel-users.test.mjs test/unit/onboarding.test.mjs test/e2e/panel-users.e2e.test.mjs + run: node --import ./test/pg-setup.mjs --test test/unit/panel-users.test.mjs test/unit/onboarding.test.mjs test/e2e/panel-users.e2e.test.mjs - name: format-check run: npx biome format - name: syntax-check Node source and scripts @@ -42,8 +68,11 @@ jobs: npm run fetch:payload npm run build:egress - run: npm run test:unit + - run: npm run test:e2e - name: ARM64 host preparation and rollback tests run: python3 -m unittest discover -s test/deploy -v + - name: PostgreSQL application role permissions + run: python3 test/deploy/smoke_postgres_role.py arm64: runs-on: ubuntu-24.04-arm diff --git a/scripts/run-isolated-tests.mjs b/scripts/run-isolated-tests.mjs index 9699a3029..8ac6c6427 100644 --- a/scripts/run-isolated-tests.mjs +++ b/scripts/run-isolated-tests.mjs @@ -3,7 +3,7 @@ * `--test-isolation=process` is Node 24+; CI stays on Node 22 to match prod. */ import { spawnSync } from 'node:child_process' -import { readdirSync } from 'node:fs' +import { readdirSync, readFileSync } from 'node:fs' import path from 'node:path' const dir = process.argv[2] @@ -25,11 +25,23 @@ if (!files.length) { let failed = 0 for (const file of files) { - const result = spawnSync(process.execPath, ['--test', file], { + const source = readFileSync(file, 'utf8') + const offline = /(?:sqlite-legacy|db-migration-sub2api)\.test\.mjs$/.test(file) + const database = + !offline && + /db\/|database\.mjs|api-keys\.mjs|panel-users\.mjs|account-quota\.mjs|sticky-router\.mjs|proxy-pool\.mjs|request-log\.mjs|api-endpoints\.mjs|thinking-owner\.mjs|harness\.mjs/.test( + source, + ) + const args = database ? ['--import', './test/pg-setup.mjs', '--test', file] : ['--test', file] + const result = spawnSync(process.execPath, args, { stdio: 'inherit', + timeout: 90_000, env: process.env, }) - if (result.status !== 0) failed += 1 + if (result.status !== 0) { + failed += 1 + if (result.error?.code === 'ETIMEDOUT') console.error(`test file timed out: ${file}`) + } } if (failed) { diff --git a/test/e2e/long-session.e2e.test.mjs b/test/e2e/long-session.e2e.test.mjs index cc0e8cc73..28121ae7e 100644 --- a/test/e2e/long-session.e2e.test.mjs +++ b/test/e2e/long-session.e2e.test.mjs @@ -4,7 +4,7 @@ */ import test from 'node:test' import assert from 'node:assert/strict' -import path from 'node:path' +import { createDatabase } from '../../src/lib/db/database.mjs' import { startGateway, api, takeTrace } from '../harness.mjs' const MODEL = 'claude-haiku-4-5-20251001' @@ -60,7 +60,7 @@ test('same x-session-id keeps native full history without CLI resume', async () } }) -test('one sticky session holds across three protocols and stream', async () => { +test('one sticky session holds across allowed protocols and stream', async () => { const gw = await startGateway({ mockText: 'ack' }) try { const h = { 'x-session-id': 'conv-multi-proto-1', 'x-kin-forward': 'cli' } @@ -86,10 +86,9 @@ test('one sticky session holds across three protocols and stream', async () => { headers: h, body: { model: MODEL, input: 'p-openai-responses' }, }) - assert.equal(c.status, 200, c.text) - const trResp = takeTrace(gw) - assert.equal(trResp.argv, undefined) - assert.match(JSON.stringify(trResp.body.messages), /p-openai-responses/) + assert.equal(c.status, 400, c.text) + assert.equal(c.json.error.code, 'protocol_not_allowed') + assert.equal(takeTrace(gw), null) const res = await fetch(gw.baseUrl + '/v1/messages', { method: 'POST', @@ -117,10 +116,10 @@ test('one sticky session holds across three protocols and stream', async () => { const snap = await api(gw, 'GET', '/admin/routing') assert.equal(snap.status, 200, snap.text) const sessions = snap.json?.sticky?.sessions || {} - const hit = sessions['conv-multi-proto-1'] + const hit = sessions['p:anthropic:conv-multi-proto-1'] assert.ok(hit, `missing sticky bind: ${JSON.stringify(snap.json?.sticky)}`) - assert.equal(hit.session_id, null) - assert.ok(hit.hits >= 4, `hits=${hit.hits}`) + assert.equal(typeof hit.session_id, 'string') + assert.ok(hit.hits >= 3, `hits=${hit.hits}`) assert.equal(hit.vm_id, 'vm-sim-01') assert.equal(hit.account_id, 'acct-seed') } finally { @@ -145,14 +144,14 @@ test('different x-session-id creates independent sticky bindings', async () => { assert.equal(trB.argv, undefined) assert.ok(msgText(trB.body.messages[0]).endsWith('only-B')) const snap = await api(gw, 'GET', '/admin/routing') - assert.ok(snap.json.sticky.sessions['conv-A']) - assert.ok(snap.json.sticky.sessions['conv-B']) + assert.ok(snap.json.sticky.sessions['p:anthropic:conv-A']) + assert.ok(snap.json.sticky.sessions['p:anthropic:conv-B']) } finally { await gw.stop() } }) -test('sticky bindings persist in sqlite after a long sequential session', async () => { +test('sticky bindings persist in PostgreSQL after a long sequential session', async () => { const gw = await startGateway({ mockText: 'n' }) try { const h = { 'x-session-id': 'conv-long-n', 'x-kin-forward': 'cli' } @@ -166,16 +165,16 @@ test('sticky bindings persist in sqlite after a long sequential session', async assert.equal(tr.argv, undefined) assert.ok(msgText(tr.body.messages[0]).endsWith(`n-${i}`)) } - // sticky bindings now live in the SQLite store (data/kin.db) - const { DatabaseSync } = await import('node:sqlite') - const db = new DatabaseSync(path.join(gw.project, 'data', 'kin.db'), { readOnly: true }) + // Shutdown must drain queued writes before reporting the gateway stopped. + await gw.stop() + const db = await createDatabase({ connectionString: gw.connectionString, schema: gw.schema }) try { - const row = db.prepare('SELECT * FROM sticky_sessions WHERE key = ?').get('conv-long-n') + const row = await db.prepare('SELECT * FROM sticky_sessions WHERE key = $1').get('p:anthropic:conv-long-n') assert.ok(row, 'sticky binding row should exist in DB') - assert.equal(row.session_id, null) + assert.equal(typeof row.session_id, 'string') assert.ok(row.hits >= 6) } finally { - db.close() + await db.close() } } finally { await gw.stop() diff --git a/test/e2e/panel-users.e2e.test.mjs b/test/e2e/panel-users.e2e.test.mjs index 6ecf7d019..41e2d2ced 100644 --- a/test/e2e/panel-users.e2e.test.mjs +++ b/test/e2e/panel-users.e2e.test.mjs @@ -2,7 +2,6 @@ import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' import { startGateway, api } from '../harness.mjs' async function login(gw, username, password) { @@ -167,13 +166,13 @@ test('deleting a tenant reclaims its assigned VM in the runtime file and SQLite assert.equal(vm.owner_user_id, null) assert.equal(vm.origin, 'platform') assert.equal(vm.schedulable, true) - const db = new DatabaseSync(path.join(gw.project, 'data', 'kin.db'), { readOnly: true }) + const db = gw.db try { - const row = db.prepare('SELECT owner_user_id, origin FROM vms WHERE id = ?').get(vm.id) + const row = await db.prepare('SELECT owner_user_id, origin FROM vms WHERE id = $1').get(vm.id) assert.equal(row.owner_user_id, null) assert.equal(row.origin, 'platform') } finally { - db.close() + // The gateway harness owns this connection. } assert.equal((await panel(gw, 'GET', '/api/panel/me', { cookie: session.cookie })).status, 401) } finally { diff --git a/test/e2e/persistence.e2e.test.mjs b/test/e2e/persistence.e2e.test.mjs index 5c7dd21df..a4dc6d7c8 100644 --- a/test/e2e/persistence.e2e.test.mjs +++ b/test/e2e/persistence.e2e.test.mjs @@ -1,18 +1,17 @@ /** - * E2E: SQLite persistence across gateway restarts + legacy JSON import. + * E2E: PostgreSQL persistence across gateway restarts + legacy JSON import. */ import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' import { startGateway, api } from '../harness.mjs' const MODEL = 'claude-haiku-4-5-20251001' function openDb(gw) { - return new DatabaseSync(path.join(gw.project, 'data', 'kin.db'), { readOnly: true }) + return gw.db } test('data survives a full gateway restart on the same data dir', async () => { @@ -72,7 +71,7 @@ test('vm credentials are mirrored into the vms table (入库)', async () => { // seeded VM has oauth in vms/vm-sim-01.json — reconcile mirrors it at boot const db = openDb(gw) try { - const row = db.prepare('SELECT * FROM vms WHERE id = ?').get('vm-sim-01') + const row = await db.prepare('SELECT * FROM vms WHERE id = $1').get('vm-sim-01') assert.ok(row, 'vms row must exist') assert.equal(row.access_token, null) assert.equal(row.refresh_token, null) @@ -81,10 +80,10 @@ test('vm credentials are mirrored into the vms table (入库)', async () => { assert.equal(vmJson.claude.access_token, undefined) assert.equal(vmJson.claude.has_access, true) assert.equal(vmJson.claude.has_refresh, true) - const active = db.prepare("SELECT value FROM settings WHERE key = 'active_vm'").get() + const active = await db.prepare("SELECT value FROM settings WHERE key = 'active_vm'").get() assert.equal(JSON.parse(active.value), 'vm-sim-01') } finally { - db.close() + // The gateway harness owns this connection. } // sessionKey import (fake) writes through persistOauthToVm → DB mirror updates @@ -94,14 +93,14 @@ test('vm credentials are mirrored into the vms table (入库)', async () => { assert.equal(imp.status, 200, imp.text) const db2 = openDb(gw) try { - const row = db2.prepare('SELECT access_token, session_key, vm_json FROM vms WHERE id = ?').get('vm-sim-01') + const row = await db2.prepare('SELECT access_token, session_key, vm_json FROM vms WHERE id = $1').get('vm-sim-01') assert.equal(row.access_token, null) assert.equal(row.session_key, null) const vmJson = JSON.parse(row.vm_json) assert.equal(vmJson.claude.session_key, undefined) assert.equal(vmJson.claude.has_access, true) } finally { - db2.close() + // The gateway harness owns this connection. } } finally { await gw.stop() @@ -172,11 +171,13 @@ test('legacy JSON files are imported once at first boot', async () => { // import flag recorded; original files untouched const db = openDb(gw) try { - const flag = JSON.parse(db.prepare("SELECT value FROM settings WHERE key = 'legacy_import_done'").get().value) + const flag = JSON.parse( + (await db.prepare("SELECT value FROM settings WHERE key = 'legacy_import_done'").get()).value, + ) assert.equal(flag.counts.api_keys, 1) assert.equal(flag.counts.request_logs, 1) } finally { - db.close() + // The gateway harness owns this connection. } assert.ok(fs.existsSync(path.join(dataDir, 'api-keys.json'))) } finally { diff --git a/test/harness.mjs b/test/harness.mjs index fd85f1232..82609f8d4 100644 --- a/test/harness.mjs +++ b/test/harness.mjs @@ -9,6 +9,8 @@ import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' import { createRequire } from 'node:module' +import crypto from 'node:crypto' +import { createDatabase } from '../src/lib/db/database.mjs' const __dirname = path.dirname(fileURLToPath(import.meta.url)) export const GATEWAY_ROOT = path.resolve(__dirname, '../src') @@ -81,9 +83,20 @@ export async function startGateway(opts = {}) { } const port = opts.port || (await pickFreePort()) const traceFile = opts.traceFile || path.join(project, 'mock-trace.json') + const connectionString = process.env.VM2API_TEST_DATABASE_URL + if (!connectionString || !/_test$/.test(new URL(connectionString).pathname)) { + throw new Error('gateway tests require VM2API_TEST_DATABASE_URL for a dedicated *_test database') + } + const schema = + (process.env.KIN_DATABASE_SCHEMA || 'gateway_test') + + '_gw_' + + crypto.createHash('sha256').update(project).digest('hex').slice(0, 12) const env = { ...process.env, KIN_API_KEY: opts.apiKey || TEST_API_KEY, + VM2API_DATABASE_URL: connectionString, + KIN_DATABASE_URL: connectionString, + KIN_DATABASE_SCHEMA: schema, KIN_ADMIN_USER: 'admin', KIN_ADMIN_PASSWORD: 'testpass', PORT: String(port), @@ -102,6 +115,7 @@ export async function startGateway(opts = {}) { KIN_MOCK_TRACE_FILE: traceFile, KIN_MOCK_TEXT: opts.mockText || 'pong', KIN_CRS_MOCK: '1', + KIN_KERNEL_BIN: path.join(REPO_ROOT, 'bin', 'kin-kernel'), } if (opts.env) Object.assign(env, opts.env) @@ -140,7 +154,11 @@ export async function startGateway(opts = {}) { throw new Error(`gateway failed to start\nstdout:\n${stdout}\nstderr:\n${stderr}`) } + const db = await createDatabase({ connectionString, schema: env.KIN_DATABASE_SCHEMA }) return { + db, + schema: env.KIN_DATABASE_SCHEMA, + connectionString, baseUrl, port, project, @@ -150,10 +168,18 @@ export async function startGateway(opts = {}) { stdout: () => stdout, stderr: () => stderr, async stop() { + await db.close() if (child.exitCode == null) { child.kill('SIGTERM') - await new Promise((r) => setTimeout(r, 150)) - if (child.exitCode == null) child.kill('SIGKILL') + await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill('SIGKILL') + }, 5000) + child.once('exit', () => { + clearTimeout(timer) + resolve() + }) + }) } }, } diff --git a/test/pg-setup.mjs b/test/pg-setup.mjs new file mode 100644 index 000000000..dd06ca9af --- /dev/null +++ b/test/pg-setup.mjs @@ -0,0 +1,70 @@ +/** Per-test PostgreSQL isolation for the Node test runner. Never use a deployment URL. */ +import { beforeEach, afterEach, after } from 'node:test' +import crypto from 'node:crypto' +import pg from 'pg' +import { openDatabase, closeDatabase, closeDatabasesForSchema } from '../src/lib/db/database.mjs' + +const url = process.env.VM2API_TEST_DATABASE_URL +if (!url || !/_test$/.test(new URL(url).pathname)) { + throw new Error('tests require VM2API_TEST_DATABASE_URL pointing to a dedicated *_test database') +} +let schema +let dedicated +let testUrl = url +let fileUrl = url +const backupTest = process.argv.some((arg) => + /(?:^|[/\\])(?:backup-service\.test\.mjs|backup\.e2e\.test\.mjs)$/.test(arg), +) +beforeEach(async () => { + await closeDatabase() + schema = `test_${process.pid}_${crypto.randomBytes(8).toString('hex')}` + testUrl = fileUrl + if (backupTest && !dedicated) { + dedicated = `vm2api_${crypto.randomBytes(8).toString('hex')}_test` + const admin = new pg.Client({ connectionString: url }) + await admin.connect() + try { + await admin.query(`CREATE DATABASE "${dedicated}"`) + } finally { + await admin.end() + } + const target = new URL(url) + target.pathname = '/' + dedicated + testUrl = target.toString() + fileUrl = testUrl + } + process.env.VM2API_TEST_DATABASE_URL = testUrl + process.env.KIN_DATABASE_SCHEMA = schema + process.env.KIN_DATABASE_URL = testUrl + process.env.VM2API_DATABASE_URL = testUrl + const db = await openDatabase({ connectionString: testUrl, schema }) + const { loadSettingsState } = await import('../src/lib/db/settings-state.mjs') + await loadSettingsState(db) +}) +afterEach(async () => { + await closeDatabase() + if (!schema) return + await closeDatabasesForSchema(schema, testUrl) + const client = new pg.Client({ connectionString: testUrl }) + await client.connect() + try { + const schemas = (await client.query('SELECT nspname FROM pg_namespace WHERE left(nspname,length($1))=$1', [schema])) + .rows + for (const row of schemas) await client.query(`DROP SCHEMA IF EXISTS "${row.nspname}" CASCADE`) + } finally { + await client.end() + } + delete process.env.KIN_DATABASE_SCHEMA +}) + +after(async () => { + if (!dedicated) return + const admin = new pg.Client({ connectionString: url }) + await admin.connect() + try { + await admin.query(`DROP DATABASE "${dedicated}" WITH (FORCE)`) + } finally { + await admin.end() + } + process.env.VM2API_TEST_DATABASE_URL = url +}) From 9feedbb383eddf9d0170848cd0c9b4639b0c2906 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:59:08 +0800 Subject: [PATCH 021/473] test(backup): cover scheduler shutdown race --- test/unit/backup-service.test.mjs | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 301ce0986..87ef351fe 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -232,6 +232,50 @@ test('shouldRunScheduled: true when never run, false right after a backup', asyn assert.equal(await svc.shouldRunScheduled(Date.now() + 25 * 3600_000), true) }) +test('stopping scheduling drains an in-flight check without starting a backup afterward', async (t) => { + let check + t.mock.method(globalThis, 'setInterval', (callback) => { + check = callback + return { unref() {} } + }) + t.mock.method(globalThis, 'clearInterval', () => {}) + let releaseCheck, signalCheck + const gate = new Promise((resolve) => { + releaseCheck = resolve + }) + const started = new Promise((resolve) => { + signalCheck = resolve + }) + svc.shouldRunScheduled = async () => { + signalCheck() + await gate + return true + } + let backups = 0 + svc.createBackup = async () => { + backups += 1 + return { size_bytes: 0 } + } + svc.startScheduler() + const checking = check() + await started + svc.stopScheduler() + let flushed = false + const pending = svc.flush().then(() => { + flushed = true + }) + let flushedDuringCheck + try { + await new Promise((resolve) => setImmediate(resolve)) + flushedDuringCheck = flushed + } finally { + releaseCheck() + await checking + await pending + } + assert.deepEqual({ flushedDuringCheck, backups }, { flushedDuringCheck: false, backups: 0 }) +}) + test('remove deletes record + file', async () => { seedVm() const rec = await svc.createBackup({ kind: 'manual' }) From 38f6db6e0be600be1a09ea5c0b9b98ad64505738 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:59:16 +0800 Subject: [PATCH 022/473] feat(runtime): await PostgreSQL persistence throughout application lifecycle --- src/lib/admin/anthropic-console.mjs | 10 +- src/lib/admin/catalog-monitor.mjs | 31 +- src/lib/admin/health-probe.mjs | 3 + src/lib/admin/key-scope.mjs | 12 +- src/lib/admin/notify.mjs | 3 + src/lib/admin/onboarding.mjs | 8 +- src/lib/admin/panel-api.mjs | 46 +- src/lib/admin/panel-routes.mjs | 419 ++++++++++-------- src/lib/admin/panel-tenant.mjs | 4 +- src/lib/admin/price-catalog.mjs | 34 +- src/lib/admin/pricing-view.mjs | 4 +- src/lib/admin/resource-owner.mjs | 4 +- src/lib/admin/routing-runtime.mjs | 12 +- src/lib/admin/usage-profile.mjs | 8 +- src/lib/admin/vm-pools.mjs | 30 +- src/lib/admin/vm-test-chat.mjs | 24 +- src/lib/admin/vm-test-schedule.mjs | 117 ++--- src/lib/cluster/cluster-manager.mjs | 193 +++++--- src/lib/cluster/cluster-routes.mjs | 62 +-- src/lib/core/refusal-guard.mjs | 19 +- src/lib/core/security.mjs | 14 +- src/lib/db/pending-writes.mjs | 20 + src/lib/db/settings-state.mjs | 40 ++ src/lib/oauth/credential-refresh-monitor.mjs | 3 + src/lib/oauth/usage-probe-monitor.mjs | 3 + src/lib/pool/account-quota.mjs | 90 +++- src/lib/pool/api-protocol.mjs | 4 +- src/lib/pool/failover-runner.mjs | 16 +- src/lib/pool/scheduling-runtime.mjs | 208 +++++++++ src/lib/pool/sticky-router.mjs | 63 ++- src/lib/pool/thinking-owner.mjs | 41 +- src/lib/pool/write-queue.mjs | 31 ++ src/lib/protocol/gpt-model-policy.mjs | 27 +- src/lib/protocol/handle-codex.mjs | 12 +- src/lib/protocol/handle-protocol.mjs | 145 +++--- src/lib/protocol/intercept-gate.mjs | 14 +- src/lib/protocol/jev-intercept.mjs | 6 +- src/lib/protocol/model-policy.mjs | 40 +- src/lib/protocol/models.mjs | 9 - src/lib/protocol/responses-ws.mjs | 6 +- src/lib/protocol/user-count-tokens.mjs | 10 +- src/lib/transport/kernel-watchdog.mjs | 3 + src/lib/vm/cli-node-guard.mjs | 3 + src/lib/vm/egress.mjs | 12 +- src/lib/vm/proxy-policy.mjs | 2 +- src/lib/vm/proxy-pool.mjs | 55 ++- src/lib/vm/vm-db-sync.mjs | 213 +++++---- src/server.mjs | 261 +++++++---- test/unit/account-quota.test.mjs | 254 +++++++---- test/unit/anthropic-console.test.mjs | 2 +- test/unit/api-backend.test.mjs | 40 +- test/unit/catalog-sync.test.mjs | 30 +- test/unit/cluster-direct-link.test.mjs | 69 ++- test/unit/cluster-placement.test.mjs | 3 +- test/unit/distill-detect.test.mjs | 4 +- test/unit/failover-runner.test.mjs | 6 +- test/unit/haiku-55-policy.test.mjs | 12 +- test/unit/handle-codex-search.test.mjs | 2 + test/unit/handle-codex.test.mjs | 42 +- test/unit/jev-account-scope.test.mjs | 6 +- test/unit/jev-intercept.test.mjs | 32 +- test/unit/key-scope.test.mjs | 12 +- test/unit/local-egress-per-node.test.mjs | 47 +- test/unit/model-policy-fable51.test.mjs | 14 +- test/unit/onboarding.test.mjs | 50 +-- test/unit/opus-55-policy.test.mjs | 12 +- test/unit/panel-api-proxy.test.mjs | 12 +- test/unit/panel-persistence.test.mjs | 39 +- test/unit/panel-tenant-guards.test.mjs | 72 +-- test/unit/panel-vm-create.test.mjs | 4 +- test/unit/panel-vm-delete.test.mjs | 8 +- test/unit/panel-vm-exit-probe.test.mjs | 12 +- test/unit/pool-scheduler.test.mjs | 105 +++-- test/unit/price-catalog.test.mjs | 22 +- test/unit/probe-status.test.mjs | 50 +-- test/unit/proxy-cli-enabled.test.mjs | 8 +- test/unit/proxy-geo.test.mjs | 22 +- test/unit/proxy-ipv6.test.mjs | 32 +- test/unit/proxy-pool-bind.test.mjs | 54 +-- test/unit/proxy-pool-import.test.mjs | 20 +- test/unit/proxy-pool-update.test.mjs | 62 +-- test/unit/proxy-resolve.test.mjs | 8 +- test/unit/quota-critical-band.test.mjs | 56 +-- test/unit/quota-tiers.test.mjs | 44 +- test/unit/rate-limit-service.test.mjs | 68 +-- test/unit/resource-owner.test.mjs | 10 + .../routing-concurrency-overrides.test.mjs | 48 +- test/unit/routing-runtime-load.test.mjs | 41 +- test/unit/seat-planner.test.mjs | 16 +- test/unit/security.test.mjs | 22 +- test/unit/settings-state.test.mjs | 30 ++ test/unit/sticky-router.test.mjs | 211 +++++---- test/unit/thinking-owner.test.mjs | 38 +- test/unit/usage-profile.test.mjs | 19 +- test/unit/vm-db-sync.test.mjs | 174 +++++--- test/unit/vm-package.test.mjs | 22 +- test/unit/vm-quota-override.test.mjs | 14 +- test/unit/vm-test-chat-billing.test.mjs | 2 +- test/unit/vm-test-chat-codex.test.mjs | 8 +- test/unit/vm-test-schedule.test.mjs | 89 ++-- test/unit/weekly-split.test.mjs | 6 +- test/unit/write-queue.test.mjs | 38 ++ 102 files changed, 2788 insertions(+), 1729 deletions(-) create mode 100644 src/lib/db/pending-writes.mjs create mode 100644 src/lib/db/settings-state.mjs create mode 100644 src/lib/pool/scheduling-runtime.mjs create mode 100644 src/lib/pool/write-queue.mjs create mode 100644 test/unit/settings-state.test.mjs create mode 100644 test/unit/write-queue.test.mjs diff --git a/src/lib/admin/anthropic-console.mjs b/src/lib/admin/anthropic-console.mjs index 5d3f242b2..9581b2327 100644 --- a/src/lib/admin/anthropic-console.mjs +++ b/src/lib/admin/anthropic-console.mjs @@ -62,12 +62,12 @@ function headerMap(raw) { return out } -function directKeys(input = {}, store = null) { +async function directKeys(input = {}, store = null) { const listed = Array.isArray(input.keys) ? input.keys : [input] const keys = [] for (const [row, item] of listed.entries()) { const source = item && typeof item === 'object' ? item : { api_key: item } - const saved = source.key_id || source.keyId ? savedEndpointKey(store, source.key_id || source.keyId) : null + const saved = source.key_id || source.keyId ? await savedEndpointKey(store, source.key_id || source.keyId) : null const apiKey = String(source.api_key || source.apiKey || saved?.api_key || '').trim() if (!apiKey) continue const label = String(source.label || '') @@ -97,10 +97,10 @@ function chosen(source, snake, camel, fallback) { return String(fallback || '').trim() } -export function savedEndpointKey(store, keyId) { +export async function savedEndpointKey(store, keyId) { const id = String(keyId || '').trim() if (!id || !store?.listRaw) return null - for (const endpoint of store.listRaw()) { + for (const endpoint of await store.listRaw()) { const key = (endpoint.api_key_entries || []).find((item) => item.id === id) if (!key) continue return { @@ -254,7 +254,7 @@ export async function runAnthropicConsole({ return forwardConsoleRequest(request, { cfg, forward }) } - const keys = directKeys(input, store) + const keys = await directKeys(input, store) const index = Number.isInteger(Number(input?.key_index)) ? Number(input.key_index) : null const selected = index == null ? keys : keys.filter((key) => key.row === index) if (!selected.length) { diff --git a/src/lib/admin/catalog-monitor.mjs b/src/lib/admin/catalog-monitor.mjs index 3437d409c..62ee4b892 100644 --- a/src/lib/admin/catalog-monitor.mjs +++ b/src/lib/admin/catalog-monitor.mjs @@ -6,7 +6,7 @@ * Results land in settings `model_catalog_sync` so the models page can show them. */ import path from 'node:path' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { runSlotOauth } from '../transport/slot-oauth.mjs' import { getVm, listVms, vmHasClaudeCredential } from '../vm/vm-registry.mjs' import { isCodexVm } from '../vm/vm-kind.mjs' @@ -32,10 +32,10 @@ export function readCatalogSync() { } } -function writeCatalogSync(platform, status) { +async function writeCatalogSync(platform, status) { const repo = new SettingsRepo() const cur = repo.get(CATALOG_SYNC_KEY, null) || { claude: null, codex: null } - repo.set(CATALOG_SYNC_KEY, { ...cur, [platform]: status }) + await repo.set(CATALOG_SYNC_KEY, { ...cur, [platform]: status }) return status } @@ -54,7 +54,7 @@ function claudeCatalogTargets(projectRoot) { /** Live Claude catalog → model_policy. */ export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth } = {}) { const targets = claudeCatalogTargets(projectRoot) - if (!targets.length) return writeCatalogSync('claude', syncStatus('skipped', { error: 'no_claude_slot' })) + if (!targets.length) return await writeCatalogSync('claude', syncStatus('skipped', { error: 'no_claude_slot' })) const ids = new Set() let successes = 0 let lastError = null @@ -70,12 +70,12 @@ export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth } successes++ for (const row of list) if (row?.id) ids.add(String(row.id)) } - if (!successes) return writeCatalogSync('claude', syncStatus('error', { error: lastError || 'models_failed' })) - const { added } = syncWorkerModelsIntoPolicy([...ids], { + if (!successes) return await writeCatalogSync('claude', syncStatus('error', { error: lastError || 'models_failed' })) + const { added } = await syncWorkerModelsIntoPolicy([...ids], { displayNameOf: (id) => modelsDevMeta('anthropic', id)?.name, }) seedModelCatalog() - return writeCatalogSync('claude', syncStatus('ok', { added })) + return await writeCatalogSync('claude', syncStatus('ok', { added })) } /** Live ChatGPT Codex catalog → gpt_model_policy. Timers never rotate OAuth; the panel button may. */ @@ -83,10 +83,10 @@ export async function syncGptCatalog({ projectRoot, vmId = '', catalogClientVers const result = await syncCodexCatalog({ projectRoot, vmId, rotate, catalogClientVersion }) if (!result.ok) { const skipped = result.error === 'no_codex_slot' || result.error === 'proxy_required' - writeCatalogSync('codex', syncStatus(skipped ? 'skipped' : 'error', { error: result.error })) + await writeCatalogSync('codex', syncStatus(skipped ? 'skipped' : 'error', { error: result.error })) return result } - writeCatalogSync('codex', syncStatus('ok', { added: result.added || [] })) + await writeCatalogSync('codex', syncStatus('ok', { added: result.added || [] })) return result } @@ -96,15 +96,21 @@ export async function syncGptCatalog({ projectRoot, vmId = '', catalogClientVers export function createCatalogMonitor({ projectRoot, codexCatalogVersion = () => 'auto', log = () => {} } = {}) { const timers = [] let running = false + const pending = new Set() const guard = (name, fn) => async () => { + if (!running) return + const operation = Promise.resolve().then(fn) + pending.add(operation) try { - const out = await fn() + const out = await operation log({ event: 'catalog-sync', job: name, status: out?.status ?? (out?.ok === false ? 'error' : 'ok') }) } catch (error) { log({ event: 'catalog-sync', job: name, status: 'error', error: String(error?.message || error) }) + } finally { + pending.delete(operation) } } - const modelsDev = guard('models.dev', () => syncModelsDev()) + const modelsDev = guard('models.dev', async () => await syncModelsDev()) const upstream = guard('upstream', async () => { const claude = await syncClaudeCatalog({ projectRoot }) await syncGptCatalog({ projectRoot, catalogClientVersion: codexCatalogVersion() }) @@ -129,6 +135,9 @@ export function createCatalogMonitor({ projectRoot, codexCatalogVersion = () => timers.push(t) } }, + async flush() { + while (pending.size) await Promise.allSettled([...pending]) + }, stop() { running = false for (const t of timers.splice(0)) { diff --git a/src/lib/admin/health-probe.mjs b/src/lib/admin/health-probe.mjs index efb810665..267417cd4 100644 --- a/src/lib/admin/health-probe.mjs +++ b/src/lib/admin/health-probe.mjs @@ -819,6 +819,9 @@ export function createHealthProbeMonitor(opts = {}) { decide, runRealProbe, start, + flush: async () => { + while (inflight) await inflight + }, stop, } } diff --git a/src/lib/admin/key-scope.mjs b/src/lib/admin/key-scope.mjs index 660124722..e83a8b831 100644 --- a/src/lib/admin/key-scope.mjs +++ b/src/lib/admin/key-scope.mjs @@ -79,15 +79,15 @@ export function keyScopeFromRecord(rec) { } } -function defaultPoolLookup(id) { +async function defaultPoolLookup(id) { if (!isDbOpen()) return null - return new VmPoolsRepo(getDb()).get(id) + return await new VmPoolsRepo(getDb()).get(id) } /** Live membership. A missing, disabled, or empty pool never widens to the global set. */ -export function applyVmPool(scope, lookup = defaultPoolLookup) { +export async function applyVmPool(scope, lookup = defaultPoolLookup) { if (!scope?.vm_pool_id) return scope - const pool = lookup?.(scope.vm_pool_id) || null + const pool = (await lookup?.(scope.vm_pool_id)) || null if (!pool) return { ...scope, allowed_vms: [], vm_pool_error: 'missing' } if (!pool.enabled) return { ...scope, allowed_vms: [], vm_pool_error: 'disabled' } const ids = Array.isArray(pool.vm_ids) ? pool.vm_ids.map((id) => String(id || '').trim()).filter(Boolean) : [] @@ -95,9 +95,9 @@ export function applyVmPool(scope, lookup = defaultPoolLookup) { return { ...scope, allowed_vms: ids, vm_pool_error: null } } -export function keyScopeFromRequest(req, lookup = defaultPoolLookup) { +export async function keyScopeFromRequest(req, lookup = defaultPoolLookup) { if (!req || req.apiKeyKind === 'master') return { group_type: 'all', allowed_vms: [], vm_pool_id: null } - return applyVmPool(keyScopeFromRecord(req.apiKeyRecord), lookup) + return await applyVmPool(keyScopeFromRecord(req.apiKeyRecord), lookup) } export function vmPoolDenial(scope) { diff --git a/src/lib/admin/notify.mjs b/src/lib/admin/notify.mjs index d59c51f4c..6acccbab1 100644 --- a/src/lib/admin/notify.mjs +++ b/src/lib/admin/notify.mjs @@ -958,6 +958,9 @@ export function createNotifyMonitor(opts = {}) { getStatus, runOnce, start, + flush: async () => { + while (inflight) await inflight + }, stop, } } diff --git a/src/lib/admin/onboarding.mjs b/src/lib/admin/onboarding.mjs index d8167854b..b9e441463 100644 --- a/src/lib/admin/onboarding.mjs +++ b/src/lib/admin/onboarding.mjs @@ -113,14 +113,14 @@ export function buildOnboardingPayload({ } } -export function loadOnboardingState(settings) { - const raw = settings?.get?.(ONBOARDING_STATE_KEY, {}) +export async function loadOnboardingState(settings) { + const raw = await settings?.get?.(ONBOARDING_STATE_KEY, {}) if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {} return raw } -export function saveOnboardingState(settings, map) { - settings.set(ONBOARDING_STATE_KEY, map && typeof map === 'object' ? map : {}) +export async function saveOnboardingState(settings, map) { + await settings.set(ONBOARDING_STATE_KEY, map && typeof map === 'object' ? map : {}) return map } diff --git a/src/lib/admin/panel-api.mjs b/src/lib/admin/panel-api.mjs index bd406181c..5e88752ce 100644 --- a/src/lib/admin/panel-api.mjs +++ b/src/lib/admin/panel-api.mjs @@ -628,9 +628,9 @@ export async function buildDashboard({ const billing = stampVmBilling( vms, - (() => { + await (async () => { try { - return attachBillingMeta(panelBillingStats(requestLog, accountQuota, vms), snap.accounts || []) + return attachBillingMeta(await panelBillingStats(requestLog, accountQuota, vms), snap.accounts || []) } catch { return null } @@ -686,18 +686,18 @@ export async function buildDashboard({ }, gateway_stats: stats, // historic totals straight from the request_logs table (survives restarts) - db_totals: (() => { + db_totals: await (async () => { try { - return requestLog?.totals() || null + return (await requestLog?.totals()) || null } catch { return null } })(), billing, // last-hour ops cards (SLA / QPS / TTFT) — console can refetch other windows via /request-logs/stats - ops: (() => { + ops: await (async () => { try { - return requestLog?.windowStats?.({ since: new Date(Date.now() - 3600_000).toISOString() }) || null + return (await requestLog?.windowStats?.({ since: new Date(Date.now() - 3600_000).toISOString() })) || null } catch { return null } @@ -744,9 +744,9 @@ export async function buildVmList({ })() stampVmBilling( vms, - (() => { + await (async () => { try { - return attachBillingMeta(panelBillingStats(requestLog, accountQuota, vms), accounts) + return attachBillingMeta(await panelBillingStats(requestLog, accountQuota, vms), accounts) } catch { return null } @@ -803,9 +803,9 @@ export async function buildVmDetail({ role, }) const acc = findAccount(accountQuota, summary) - const billing = (() => { + const billing = await (async () => { try { - return panelBillingStats(requestLog, accountQuota, [summary]) + return await panelBillingStats(requestLog, accountQuota, [summary]) } catch { return null } @@ -813,7 +813,7 @@ export async function buildVmDetail({ const cost = lookupBilling(indexBillingAccounts(billing), acc || summary) const gpt = isCodexVm(vm) applyCostFields(summary, cost, { coveredSince: billing?.log_coverage_start }) - const detail = buildAccountBilling(cost, billing, { + const detail = await buildAccountBilling(cost, billing, { vmId: summary.id, accountId: acc?.account_id || summary.account_uuid, requestLog, @@ -1146,7 +1146,7 @@ export async function buildProbeOne({ probed_at: new Date().toISOString(), ...(headerProbe ? { ...headerProbe, error: null } : {}), } - : await cache.load(accountId, () => probeAccount({ exec, vm, includeFable }), { force: !!force }) + : await cache.load(accountId, async () => await probeAccount({ exec, vm, includeFable }), { force: !!force }) if (fatalUsageCredentialFailure(result)) { try { markVmRefreshError(path.join(cfg.paths.project, 'vms', `${id}.json`), { @@ -1306,12 +1306,12 @@ export async function buildProbeAll({ cfg, accountQuota, hop = true, force = tru return ok({ items, total: items.length }) } -export function buildUsage({ accountQuota, cfg, requestLog = null }) { +export async function buildUsage({ accountQuota, cfg, requestLog = null }) { const snap = accountQuota.snapshot() const listed = listVms(cfg?.paths?.project) - const billing = (() => { + const billing = await (async () => { try { - return panelBillingStats(requestLog, accountQuota, listed) + return await panelBillingStats(requestLog, accountQuota, listed) } catch { return null } @@ -1488,9 +1488,9 @@ export async function snapshotAccountPool({ })() const billing = stampVmBilling( vms, - (() => { + await (async () => { try { - return attachBillingMeta(panelBillingStats(requestLog, accountQuota, vms), accounts) + return attachBillingMeta(await panelBillingStats(requestLog, accountQuota, vms), accounts) } catch { return null } @@ -1498,7 +1498,7 @@ export async function snapshotAccountPool({ ) let today = null try { - today = requestLog?.windowStats?.({ since: shanghaiDayStartIso() }) || null + today = (await requestLog?.windowStats?.({ since: shanghaiDayStartIso() })) || null } catch { today = null } @@ -2318,9 +2318,9 @@ function billingWindowOf(accountId, vmId, vm, q) { } } -function panelBillingStats(requestLog, accountQuota, vms) { +async function panelBillingStats(requestLog, accountQuota, vms) { if (!requestLog?.billingStats) return null - return requestLog.billingStats({ accountWindows: accountWindowsForBilling(accountQuota, vms) }) + return await requestLog.billingStats({ accountWindows: accountWindowsForBilling(accountQuota, vms) }) } function stampVmBilling(vms, billing) { @@ -2431,7 +2431,7 @@ function windowEstimates(quota, cost, coveredSince, codex) { } } -function buildAccountBilling(cost, billing, { vmId, accountId, requestLog } = {}) { +async function buildAccountBilling(cost, billing, { vmId, accountId, requestLog } = {}) { if (!cost && !vmId && !accountId) return null const today = periodView(cost?.today) || { requests: Number(cost?.today_requests || 0), @@ -2475,14 +2475,14 @@ function buildAccountBilling(cost, billing, { vmId, accountId, requestLog } = {} const total = periodView(cost) let byModel = [] try { - byModel = requestLog?.costByModel?.({ vmId, accountId }) || [] + byModel = (await requestLog?.costByModel?.({ vmId, accountId })) || [] } catch { byModel = [] } if (!cost && !byModel.length) return null let usageStats = null try { - usageStats = vmId ? requestLog?.vmUsageStats?.({ vmId, days: 30 }) || null : null + usageStats = vmId ? (await requestLog?.vmUsageStats?.({ vmId, days: 30 })) || null : null } catch { usageStats = null } diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 4d2ab9902..5fa6a2d94 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -1,3 +1,4 @@ +import { flushPendingWrites } from '../db/pending-writes.mjs' /** * /admin and /api/panel routes. Login / logout / me and { ok, data } * envelopes are unchanged; slot start/reload go through startSlotReady. @@ -26,7 +27,7 @@ import { } from '../protocol/jev-intercept.mjs' import { RefusalGuardsRepo } from '../db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../db/repos/refusal-device-blocks-repo.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { parseCodexImportPayload, upsertCodexAccount, readCodexAccounts } from '../vm/codex-slot.mjs' import { generateAuthUrl, exchangeAuthCode, normalizeOauthFlavor } from '../oauth/oauth-auth-url.mjs' import { sessionKeyToOAuth, panelImportErrorPayload } from '../oauth/cookie-auth.mjs' @@ -173,7 +174,7 @@ import { GATEWAY_CAPABILITIES } from '../vm/execution-context.mjs' import { withVmLock, atomicWriteJson, isValidVmId, invalidateVmSlotReadCache } from '../vm/vm-file.mjs' import { snapshotDatabaseMetrics } from '../db/database-metrics.mjs' import { getUsageCache } from '../oauth/usage-cache.mjs' -import { getDb, getDbPath } from '../db/database.mjs' +import { getDb } from '../db/database.mjs' import { readLocalVersion, loadChangelog, @@ -182,7 +183,7 @@ import { startHostUpgrade, } from './release.mjs' import { downloadReleaseKernel, kernelReleaseHttpStatus } from './release-kernel.mjs' -import { clearActiveVmFromDb, removeVmFromDb } from '../vm/vm-db-sync.mjs' +import { clearActiveVmFromDb, removeVmFromDb, flushVmDbSync } from '../vm/vm-db-sync.mjs' import { normalizeCredentialMode } from '../oauth/credential-mode.mjs' import { reloadActiveVm } from '../core/config.mjs' import { @@ -300,7 +301,7 @@ function clusterNodesOf(ctx) { } } -function collectOnboardingFacts(req, ctx) { +async function collectOnboardingFacts(req, ctx) { const ident = panelIdentity(req) const ownerUserId = ident.role === 'user' ? req.panelUserId : null const vms = listVms(ctx.cfg.paths.project, ctx.routingConfig) @@ -311,24 +312,24 @@ function collectOnboardingFacts(req, ctx) { nodes: clusterNodesOf(ctx), }) const visible = filterVmsForPanel(vms, { role: ident.role, userId: ownerUserId }) - const keys = ctx.apiKeyStore.snapshot().keys || [] + const keys = (await ctx.apiKeyStore.snapshot()).keys || [] const mine = ident.role === 'user' ? keys.filter((k) => normalizeOwnerId(k.user_id) === normalizeOwnerId(req.panelUserId)) : keys const admin = getPanelAdmin() return { envPassword: admin.password, - envAdmin: ctx.panelUsers.getByUsername(admin.username), + envAdmin: await ctx.panelUsers.getByUsername(admin.username), proxyCount: (snap.proxies || []).length, ...vmStepFacts(visible), activeKeyCount: mine.filter((k) => k.status === 'active').length, - hasFirstRequest: ctx.requestLog.hasSuccessfulRequest({ owner_user_id: ownerUserId }), + hasFirstRequest: await ctx.requestLog.hasSuccessfulRequest({ owner_user_id: ownerUserId }), } } -function onboardingPayloadFor(req, ctx) { +async function onboardingPayloadFor(req, ctx) { const user = onboardingUserOf(req) const role = onboardingRoleOf(req) - const rec = recordOf(loadOnboardingState(new SettingsRepo()), user) + const rec = recordOf(await loadOnboardingState(new SettingsRepo()), user) return buildOnboardingPayload({ user, role, @@ -337,7 +338,7 @@ function onboardingPayloadFor(req, ctx) { guide_used: rec.guide_used, // Onboarding matches /health: base_url only when PUBLIC_* is configured; else null → browser origin. base_url: process.env.PUBLIC_BASE_URL || null, - facts: collectOnboardingFacts(req, ctx), + facts: await collectOnboardingFacts(req, ctx), }) } @@ -591,7 +592,27 @@ function slotProxyHint(px = {}, label = null) { export function createPanelHandler(ctx) { // A checkout can change under a live Node process; report its boot version. const runningVersion = readLocalVersion(ctx.cfg?.paths?.project) - const json = (...args) => ctx.json(...args) + const json = (...args) => { + const state = args[0]?._panelResponseState + if (state) { + state.response = args + return + } + return ctx.json(...args) + } + async function flushPanelWrites() { + for (const store of [ctx.accountQuota, ctx.stickyRouter, ctx.proxyPool, ctx.runtimeRepo]) { + await store?.writes?.flush?.() + } + await flushVmDbSync() + } + async function persistenceFailure(res, error) { + await ctx.onPersistenceFailure?.(error) + return ctx.json(res, 503, { + ok: false, + error: { code: 'database_persistence_failed', message: 'Database persistence failed; maintenance is required' }, + }) + } const hostUnsupported = (res, vm, cap) => { if (slotHost(vm).supports(cap)) return false json(res, 409, { @@ -603,7 +624,7 @@ export function createPanelHandler(ctx) { const readBody = (...args) => ctx.readBody(...args) const readRawBody = (...args) => (ctx.readRawBody || defaultReadRawBody)(...args) const refreshCluster = ctx.refreshClusterSlotPayloads || refreshClusterSlotPayloads - const requireAuth = (...args) => ctx.requireAuth(...args) + const requireAuth = async (...args) => await ctx.requireAuth(...args) const cfg = ctx.cfg const routingConfigPath = ctx.routingConfigPath const stickyRouter = ctx.stickyRouter @@ -617,13 +638,13 @@ export function createPanelHandler(ctx) { const vmTestSchedule = ctx.vmTestSchedule const backupService = ctx.backupService const stats = ctx.stats - const persistRoutingPatch = (...args) => ctx.persistRoutingPatch(...args) + const persistRoutingPatch = async (...args) => ctx.persistRoutingPatch(...args) const applyVmConcurrency = (...args) => ctx.applyVmConcurrency(...args) const applyVmRpm = (...args) => ctx.applyVmRpm(...args) const applyVmSessionSlots = (...args) => ctx.applyVmSessionSlots(...args) const inheritVmScheduling = (...args) => ctx.inheritVmScheduling(...args) const applyVmQuotaOverride = (...args) => ctx.applyVmQuotaOverride(...args) - const initPoolRuntime = (...args) => ctx.initPoolRuntime(...args) + const initPoolRuntime = async (...args) => ctx.initPoolRuntime(...args) const poolSchedulerConfig = (...args) => ctx.poolSchedulerConfig(...args) const commitImportedOauth = (...args) => ctx.commitImportedOauth(...args) const requireSlotProxy = (...args) => ctx.requireSlotProxy(...args) @@ -639,8 +660,8 @@ export function createPanelHandler(ctx) { setVmSchedulable(cfg.paths.project, vmId, true) } - function fillTimezoneFromExit(id) { - return fillVmTimezoneFromExit(cfg.paths.project, proxyPool, id) + async function fillTimezoneFromExit(id) { + return await fillVmTimezoneFromExit(cfg.paths.project, proxyPool, id) } /** @@ -752,7 +773,7 @@ export function createPanelHandler(ctx) { return json(res, 200, panel.ok({ schedule: vmTestSchedule.get(id) })) } if (req.method === 'DELETE') { - vmTestSchedule.remove(id) + await vmTestSchedule.remove(id) return json(res, 200, panel.ok({ schedule: null })) } let body @@ -765,7 +786,7 @@ export function createPanelHandler(ctx) { }) } try { - return json(res, 200, panel.ok({ schedule: vmTestSchedule.save(id, body) })) + return json(res, 200, panel.ok({ schedule: await vmTestSchedule.save(id, body) })) } catch (error) { const typed = error instanceof VmTestScheduleError return json(res, typed ? error.status : 500, { @@ -793,10 +814,10 @@ export function createPanelHandler(ctx) { } try { if (body == null || body.schedule == null) { - vmTestSchedule.clearDefaultTemplate() + await vmTestSchedule.clearDefaultTemplate() return json(res, 200, panel.ok({ schedule: null })) } - return json(res, 200, panel.ok({ schedule: vmTestSchedule.saveDefaultTemplate(body.schedule) })) + return json(res, 200, panel.ok({ schedule: await vmTestSchedule.saveDefaultTemplate(body.schedule) })) } catch (error) { const typed = error instanceof VmTestScheduleError return json(res, typed ? error.status : 500, { @@ -838,7 +859,7 @@ export function createPanelHandler(ctx) { body = { ...body, vm_ids: vmIds } } try { - const result = vmTestSchedule.syncSchedules({ + const result = await vmTestSchedule.syncSchedules({ scope: body?.scope, vmIds: body?.vm_ids, schedule: body?.schedule, @@ -905,17 +926,17 @@ export function createPanelHandler(ctx) { return cfg.distill } - function refusalGuardSnapshot() { + async function refusalGuardSnapshot() { const settings = new SettingsRepo() const repo = new RefusalGuardsRepo() const devices = new RefusalDeviceBlocksRepo() const read = (key, fallback) => settings.get(key, fallback) return { ...refusalGuardPolicy(read), - count: repo.count(), - items: repo.list(), - device_count: devices.count(), - devices: devices.list(), + count: await repo.count(), + items: await repo.list(), + device_count: await devices.count(), + devices: await devices.list(), } } @@ -1104,14 +1125,14 @@ export function createPanelHandler(ctx) { }) } - function restoreRoutingRuntime(previous) { + async function restoreRoutingRuntime(previous) { ctx.routingConfig = previous setManualScheduleWins(ctx.routingConfig.pool?.manual_schedule_wins) ctx.healthMonitor?.setConfig(ctx.routingConfig.health_probe) ctx.usageProbeMonitor?.setConfig(ctx.routingConfig.usage_probe) ctx.notifyMonitor?.setConfig(ctx.routingConfig.notify) if (ctx.routingConfig.logging) { - requestLog.setConfig({ + await requestLog.setConfig({ mode: ctx.routingConfig.logging.mode, retainDays: ctx.routingConfig.logging.retain_days, debugRetainDays: ctx.routingConfig.logging.debug_retain_days, @@ -1130,16 +1151,16 @@ export function createPanelHandler(ctx) { const target = normalizeInferenceConfig({ ...(ctx.routingConfig.inference || {}), ...body.inference }).engine if (previous === target) return { ok: true, changed: false, items: [] } const previousRoutingConfig = structuredClone(ctx.routingConfig) - return switchInheritedInferenceEngines({ + return await switchInheritedInferenceEngines({ projectRoot: cfg.paths.project, previousEngine: previous, targetEngine: target, - commit: () => { + commit: async () => { try { - return persistRoutingPatch(body) + return await persistRoutingPatch(body) } catch (error) { - restoreRoutingRuntime(previousRoutingConfig) - if (body.pool || body.failover) initPoolRuntime() + await restoreRoutingRuntime(previousRoutingConfig) + if (body.pool || body.failover) await initPoolRuntime() throw error } }, @@ -1161,12 +1182,13 @@ export function createPanelHandler(ctx) { } } - return async function handlePanel(req, res, url) { + async function handlePanel(req, res, url) { const p = url.pathname + if (req.method === 'GET') await flushPendingWrites() if (!p.startsWith('/admin') && !p.startsWith('/api/panel')) return false if (p.startsWith('/admin')) { - if (!requireAuth(req, res)) return true + if (!(await requireAuth(req, res))) return true if (panelIdentity(req).role !== 'admin') { json( res, @@ -1184,7 +1206,7 @@ export function createPanelHandler(ctx) { if (p === '/admin/models/refresh') return false if (p === '/admin/routing' && req.method === 'GET') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 200, { routing: publicRoutingNotify(ctx.routingConfig), sticky: stickyRouter.stats(), @@ -1193,11 +1215,11 @@ export function createPanelHandler(ctx) { }) } if (p === '/admin/routing' && (req.method === 'PUT' || req.method === 'POST')) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const body = await readBody(req, cfg.limits.max_body_bytes) let applied try { - applied = persistRoutingPatch(body) + applied = await persistRoutingPatch(body) } catch (error) { if (error instanceof TypeError || error instanceof RangeError) { return json(res, 400, { @@ -1219,19 +1241,19 @@ export function createPanelHandler(ctx) { }) } if (p === '/admin/quota' && req.method === 'GET') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 200, accountQuota.snapshot()) } // ---- Multi-VM + usage management ---- if (req.method === 'GET' && p === '/admin/vms') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const active = getActiveVmId(cfg.paths.project) const vms = listVms(cfg.paths.project, ctx.routingConfig).map((v) => ({ ...v, active: v.id === active })) return json(res, 200, { vms, active_vm: active, total: vms.length }) } if (req.method === 'GET' && p.startsWith('/admin/vms/') && !p.includes('/probe') && !p.endsWith('/activate')) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const id = p.split('/')[3] const vm = getVm(cfg.paths.project, id) if (!vm) return json(res, 404, { error: { message: 'vm not found' } }) @@ -1241,13 +1263,13 @@ export function createPanelHandler(ctx) { }) } if (req.method === 'POST' && /^\/admin\/vms\/[^/]+\/activate$/.test(p)) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const id = p.split('/')[3] if (!getVm(cfg.paths.project, id)) return json(res, 404, { error: { message: 'vm not found' } }) return json(res, 200, activateVmSlot(id)) } if (req.method === 'POST' && /^\/admin\/vms\/[^/]+\/probe$/.test(p)) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const id = p.split('/')[3] const hop = (await readBody(req, 4096).catch(() => ({})))?.hop !== false const result = await panel.buildProbeOne({ cfg, accountQuota, id, hop, routingConfig: ctx.routingConfig }) @@ -1255,13 +1277,13 @@ export function createPanelHandler(ctx) { return json(res, 200, { vm_id: id, account_uuid: result.data?.account_uuid, probe: result.data }) } if (req.method === 'POST' && p === '/admin/vms/probe-all') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const hop = (await readBody(req, 4096).catch(() => ({})))?.hop === true const result = await panel.buildProbeAll({ cfg, accountQuota, hop, routingConfig: ctx.routingConfig }) return json(res, 200, result) } if (req.method === 'GET' && p === '/admin/usage/summary') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const snap = accountQuota.snapshot() const vms = listVms(cfg.paths.project, ctx.routingConfig) const accounts = snap.accounts || [] @@ -1297,16 +1319,16 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 4096) const username = body.username || body.user || body.u || '' const password = body.password || body.pass || body.p || '' - const authed = verifyPanelLogin(username, password) + const authed = await verifyPanelLogin(username, password) if (!authed) { return json(res, 401, { ok: false, error: { message: '用户名或密码错误', type: 'auth' } }) } if (authed.source === 'env') { try { - panelUsers.bootstrapFromEnv({ username: authed.username, password }) + await panelUsers.bootstrapFromEnv({ username: authed.username, password }) } catch {} } - const token = createPanelSession(authed.username, { role: authed.role }) + const token = await createPanelSession(authed.username, { role: authed.role }) const secure = panelCookieSecure(req) res.setHeader('Set-Cookie', panelSessionCookie(token, { secure })) const me = { user: authed.username, role: authed.role } @@ -1329,7 +1351,7 @@ export function createPanelHandler(ctx) { } if (p.startsWith('/api/panel')) { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return // Managed client keys may only call protocol endpoints — never panel/admin. // Only panel session or master KIN_API_KEY may administer. const isPanelOperator = !!req.panelUser || req.apiKeyKind === 'master' @@ -1386,10 +1408,10 @@ export function createPanelHandler(ctx) { try { const settings = new SettingsRepo() const user = onboardingUserOf(req) - const state = loadOnboardingState(settings) + const state = await loadOnboardingState(settings) let next if (body?.action === 'profile') { - applyUsageProfile({ + await applyUsageProfile({ mode: body?.mode, persistRouting: (patch) => persistRoutingPatch(patch), settings, @@ -1402,7 +1424,7 @@ export function createPanelHandler(ctx) { } else { next = applyOnboardingAction(state, user, body?.action) } - saveOnboardingState(settings, next) + await saveOnboardingState(settings, next) } catch (e) { if (e.code === 'invalid_action' || e.code === 'invalid_profile') { return json(res, 400, { @@ -1413,13 +1435,12 @@ export function createPanelHandler(ctx) { throw e } } - return json(res, 200, panel.ok(onboardingPayloadFor(req, ctx))) + return json(res, 200, panel.ok(await onboardingPayloadFor(req, ctx))) } - if (p.startsWith('/api/panel/cluster/')) return ctx.clusterRoutes.handle(req, res, url) + if (p.startsWith('/api/panel/cluster/')) return ctx.clusterRoutes.handle(req, res, url, json) if (req.method === 'GET' && p === '/api/panel/database/metrics') { - const snapshot = snapshotDatabaseMetrics({ + const snapshot = await snapshotDatabaseMetrics({ db: getDb(), - dbPath: getDbPath(), usageCache: getUsageCache(), }) return json(res, 200, panel.ok(snapshot)) @@ -1455,12 +1476,12 @@ export function createPanelHandler(ctx) { return json(res, result.status, panel.ok(result.data)) } if (req.method === 'GET' && p === '/api/panel/users') { - return json(res, 200, panel.ok({ items: panelUsers.list() })) + return json(res, 200, panel.ok({ items: await panelUsers.list() })) } if (req.method === 'POST' && p === '/api/panel/users') { const body = await readBody(req, 8192).catch(() => ({})) try { - const rec = panelUsers.create({ + const rec = await panelUsers.create({ username: body.username || body.user, password: body.password || body.pass, role: body.role || 'user', @@ -1480,8 +1501,8 @@ export function createPanelHandler(ctx) { const id = p.split('/').pop() const body = await readBody(req, 8192).catch(() => ({})) try { - const before = panelUsers.getById(id) - const rec = panelUsers.update(id, body || {}, { actorId: req.panelUserId }) + const before = await panelUsers.getById(id) + const rec = await panelUsers.update(id, body || {}, { actorId: req.panelUserId }) if (!rec) return json(res, 404, { ok: false, error: { message: 'user not found' } }) // Sessions cache the role from login time, so a role or enable change must log them out. if (before && (before.role !== rec.role || !!before.enabled !== !!rec.enabled)) { @@ -1500,9 +1521,9 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/users\/[^/]+$/.test(p)) { const id = p.split('/').pop() - const existing = panelUsers.getById(id) + const existing = await panelUsers.getById(id) try { - const ok = panelUsers.remove(id, { actorId: req.panelUserId }) + const ok = await panelUsers.remove(id, { actorId: req.panelUserId }) if (!ok) return json(res, 404, { ok: false, error: { message: 'user not found' } }) if (existing?.username) revokePanelSessionsForUser(existing.username) return json(res, 200, { ok: true, deleted: id }) @@ -1515,12 +1536,12 @@ export function createPanelHandler(ctx) { } } if (req.method === 'GET' && p === '/api/panel/vm-pools') { - return json(res, 200, panel.ok({ pools: vmPoolsFor(apiKeyStore.db).list() })) + return json(res, 200, panel.ok({ pools: await vmPoolsFor(apiKeyStore.db).list() })) } if (req.method === 'POST' && p === '/api/panel/vm-pools') { const body = await readBody(req, 64 * 1024).catch(() => ({})) try { - const pool = createVmPool(vmPoolsFor(apiKeyStore.db), body || {}, listVms(cfg.paths.project)) + const pool = await createVmPool(vmPoolsFor(apiKeyStore.db), body || {}, listVms(cfg.paths.project)) return json(res, 201, panel.ok({ pool })) } catch (e) { return json(res, vmPoolHttpStatus(e.code), { @@ -1533,7 +1554,7 @@ export function createPanelHandler(ctx) { const id = decodeURIComponent(p.split('/').pop()) const body = await readBody(req, 64 * 1024).catch(() => ({})) try { - const pool = updateVmPool(vmPoolsFor(apiKeyStore.db), id, body || {}, listVms(cfg.paths.project)) + const pool = await updateVmPool(vmPoolsFor(apiKeyStore.db), id, body || {}, listVms(cfg.paths.project)) if (!pool) return json(res, 404, { ok: false, error: { message: '账号池不存在', code: 'vm_pool_not_found' } }) return json(res, 200, panel.ok({ pool })) } catch (e) { @@ -1545,7 +1566,7 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/vm-pools\/[^/]+$/.test(p)) { const id = decodeURIComponent(p.split('/').pop()) - const result = deleteVmPool(vmPoolsFor(apiKeyStore.db), id) + const result = await deleteVmPool(vmPoolsFor(apiKeyStore.db), id) if (!result.ok) { return json(res, vmPoolHttpStatus(result.error), { ok: false, @@ -1559,7 +1580,7 @@ export function createPanelHandler(ctx) { return json(res, 200, { ok: true, deleted: id }) } if (req.method === 'GET' && p === '/api/panel/api-keys') { - const snap = apiKeyStore.snapshot() + const snap = await apiKeyStore.snapshot() if (panelIdentity(req).role === 'user') { const mine = (snap.keys || []).filter( (k) => normalizeOwnerId(k.user_id) === normalizeOwnerId(req.panelUserId), @@ -1575,10 +1596,10 @@ export function createPanelHandler(ctx) { } if (req.method === 'GET' && /^\/api\/panel\/api-keys\/[^/]+\/stats$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const rec = apiKeyStore.getById(id) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const rec = await apiKeyStore.getById(id) if (!rec) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) - const usage_stats = requestLog?.keyUsageStats?.({ apiKeyId: id, days: 30 }) || null + const usage_stats = (await requestLog?.keyUsageStats?.({ apiKeyId: id, days: 30 })) || null return json(res, 200, { ok: true, item: publicKeyView(rec, { reveal: false }), usage_stats }) } if (req.method === 'POST' && p === '/api/panel/api-keys') { @@ -1598,14 +1619,14 @@ export function createPanelHandler(ctx) { } const scope = normalizeKeyScope(scopeInput) const pools = vmPoolsFor(apiKeyStore.db) - if (scope.vm_pool_id) assertVmPoolExists(pools, scope.vm_pool_id) + if (scope.vm_pool_id) await assertVmPoolExists(pools, scope.vm_pool_id) else assertVmScope(keyScopeVms(req), scope) const defaultConc = Number( ctx.routingConfig?.concurrency?.default_key_concurrency ?? ctx.routingConfig?.concurrency?.default_max_per_account ?? 20, ) - const rec = apiKeyStore.create({ + const rec = await apiKeyStore.create({ name: body?.name, key: body?.key || body?.custom_key || undefined, category: body?.category, @@ -1644,7 +1665,7 @@ export function createPanelHandler(ctx) { } if (req.method === 'PATCH' && /^\/api\/panel\/api-keys\/[^/]+$/.test(p)) { const id = p.split('/').pop() - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true const body = await readBody(req, 8192).catch(() => ({})) const ident = panelIdentity(req) if (ident.role !== 'admin' && body?.vm_pool_id != null && String(body.vm_pool_id).trim() !== '') { @@ -1655,7 +1676,7 @@ export function createPanelHandler(ctx) { } try { const input = normalizePanelApiKeyInput(body || {}, req) - const currentRec = apiKeyStore.getById(id) + const currentRec = await apiKeyStore.getById(id) const scopeInput = { group_type: body?.group_type, allowed_vms: body?.allowed_vms } if (ident.role === 'admin' && Object.prototype.hasOwnProperty.call(body || {}, 'vm_pool_id')) { scopeInput.vm_pool_id = body.vm_pool_id @@ -1666,13 +1687,13 @@ export function createPanelHandler(ctx) { scope.vm_pool_id = currentRec.vm_pool_id } const pools = vmPoolsFor(apiKeyStore.db) - if (scope.vm_pool_id) assertVmPoolExists(pools, scope.vm_pool_id) + if (scope.vm_pool_id) await assertVmPoolExists(pools, scope.vm_pool_id) else assertVmScope(keyScopeVms(req), scope) input.group_type = scope.group_type input.allowed_vms = scope.allowed_vms input.vm_pool_id = scope.vm_pool_id } - const rec = apiKeyStore.update(id, input) + const rec = await apiKeyStore.update(id, input) if (!rec) { return json(res, 404, { ok: false, error: { message: 'api key not found' } }) } @@ -1686,8 +1707,8 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/api-keys\/[^/]+\/reveal$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const out = apiKeyStore.reveal(id) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const out = await apiKeyStore.reveal(id) if (!out) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) if (!out.ok) { return json(res, 409, { @@ -1699,9 +1720,9 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/api-keys\/[^/]+\/rotate$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true try { - const rec = apiKeyStore.rotate(id) + const rec = await apiKeyStore.rotate(id) if (!rec) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) return json(res, 200, { ok: true, item: publicKeyView(rec, { reveal: true }) }) } catch (e) { @@ -1713,15 +1734,15 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/api-keys\/[^/]+\/reset-quota$/.test(p)) { const id = p.split('/')[4] - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const rec = apiKeyStore.update(id, { reset_quota: true }) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const rec = await apiKeyStore.update(id, { reset_quota: true }) if (!rec) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) return json(res, 200, { ok: true, item: publicKeyView(rec, { reveal: false }) }) } if (req.method === 'DELETE' && /^\/api\/panel\/api-keys\/[^/]+$/.test(p)) { const id = p.split('/').pop() - if (denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true - const ok = apiKeyStore.remove(id) + if (await denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId: id })) return true + const ok = await apiKeyStore.remove(id) if (!ok) return json(res, 404, { ok: false, error: { message: 'api key not found' } }) return json(res, 200, { ok: true, deleted: id }) } @@ -1729,14 +1750,14 @@ export function createPanelHandler(ctx) { if (req.method === 'GET' && p === '/api/panel/api-endpoints') { return json(res, 200, { ok: true, - items: apiEndpointStore.list({ reveal: false }), + items: await apiEndpointStore.list({ reveal: false }), presets: API_ENDPOINT_PRESETS, }) } if (req.method === 'POST' && p === '/api/panel/api-endpoints/fetch-models') { const body = await readBody(req, 8192).catch(() => ({})) try { - const saved = savedEndpointKey(apiEndpointStore, body?.key_id) + const saved = await savedEndpointKey(apiEndpointStore, body?.key_id) const proxyUrl = Object.prototype.hasOwnProperty.call(body || {}, 'proxy_id') ? resolveConsoleProxyUrl(proxyPool, body.proxy_id) : saved?.proxy_url || body?.proxy_url @@ -1769,8 +1790,8 @@ export function createPanelHandler(ctx) { if (body?.proxy_id != null) entry.proxy_id = String(body.proxy_id || '').trim() } } - const rec = apiEndpointStore.create(body || {}) - apiScheduler.reload(apiEndpointStore.listRaw()) + const rec = await apiEndpointStore.create(body || {}) + apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 201, { ok: true, item: publicEndpointView(rec, { reveal: true }) }) } catch (e) { return json(res, 400, { @@ -1784,24 +1805,24 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 1 << 20).catch(() => ({})) try { const proxyUrl = body?.proxy_id ? resolveConsoleProxyUrl(proxyPool, body.proxy_id) : '' - const rec = apiEndpointStore.update(id, body || {}) + const rec = await apiEndpointStore.update(id, body || {}) if (!rec) return json(res, 404, { ok: false, error: { message: 'endpoint not found' } }) const keyId = String(body?.key_id || '').trim() const apiKey = String(body?.api_key || '').trim() if (keyId) { - apiEndpointStore.updateKey(keyId, { + await apiEndpointStore.updateKey(keyId, { ...(apiKey ? { api_key: apiKey } : {}), ...(body?.proxy_id != null ? { proxy_url: proxyUrl, proxy_id: String(body.proxy_id || '').trim() } : {}), }) } else if (apiKey) { - apiEndpointStore.addKey(id, { + await apiEndpointStore.addKey(id, { api_key: apiKey, proxy_url: proxyUrl, proxy_id: String(body?.proxy_id || '').trim(), }) } - apiScheduler.reload(apiEndpointStore.listRaw()) - const saved = apiEndpointStore.getRaw(id) || rec + apiScheduler.reload(await apiEndpointStore.listRaw()) + const saved = (await apiEndpointStore.getRaw(id)) || rec return json(res, 200, { ok: true, item: publicEndpointView(saved) }) } catch (e) { return json(res, 400, { ok: false, error: { message: String(e.message || e), code: e.code } }) @@ -1809,15 +1830,15 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/api-endpoints\/[^/]+$/.test(p)) { const id = p.split('/').pop() - const ok = apiEndpointStore.remove(id) + const ok = await apiEndpointStore.remove(id) if (!ok) return json(res, 404, { ok: false, error: { message: 'endpoint not found' } }) - apiScheduler.reload(apiEndpointStore.listRaw()) + apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 200, { ok: true, deleted: id }) } if (req.method === 'POST' && /^\/api\/panel\/api-endpoints\/[^/]+\/fetch-models$/.test(p)) { const id = p.split('/')[4] const body = await readBody(req, 8192).catch(() => ({})) - const ep = apiEndpointStore.getRaw(id) + const ep = await apiEndpointStore.getRaw(id) if (!ep) return json(res, 404, { ok: false, error: { message: 'endpoint not found' } }) const key = (ep.api_key_entries || []).find((k) => !k.disabled && (body?.key_id ? k.id === body.key_id : true)) if (!key) @@ -1833,8 +1854,8 @@ export function createPanelHandler(ctx) { headers: ep.headers, }) const apply = body?.apply !== false - const rec = apply ? apiEndpointStore.mergeFetchedModels(id, result.models) : ep - if (apply) apiScheduler.reload(apiEndpointStore.listRaw()) + const rec = apply ? await apiEndpointStore.mergeFetchedModels(id, result.models) : ep + if (apply) apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 200, { ok: true, fetched: result.models.length, @@ -1852,9 +1873,9 @@ export function createPanelHandler(ctx) { const id = p.split('/')[4] const body = await readBody(req, 8192).catch(() => ({})) try { - const key = apiEndpointStore.addKey(id, body || {}) - apiScheduler.reload(apiEndpointStore.listRaw()) - const view = publicEndpointView(apiEndpointStore.getRaw(id), { reveal: true }) + const key = await apiEndpointStore.addKey(id, body || {}) + apiScheduler.reload(await apiEndpointStore.listRaw()) + const view = publicEndpointView(await apiEndpointStore.getRaw(id), { reveal: true }) return json(res, 201, { ok: true, item: view?.api_key_entries?.find((k) => k.id === key.id) || key }) } catch (e) { const status = e.code === 'endpoint_not_found' ? 404 : 400 @@ -1863,18 +1884,18 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/api-endpoints\/[^/]+\/keys\/[^/]+$/.test(p)) { const kid = p.split('/').pop() - const ok = apiEndpointStore.removeKey(kid) + const ok = await apiEndpointStore.removeKey(kid) if (!ok) return json(res, 404, { ok: false, error: { message: 'key not found' } }) - apiScheduler.reload(apiEndpointStore.listRaw()) + apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 200, { ok: true, deleted: kid }) } if (req.method === 'PATCH' && /^\/api\/panel\/api-endpoints\/[^/]+\/keys\/[^/]+$/.test(p)) { const kid = p.split('/').pop() const body = await readBody(req, 8192).catch(() => ({})) - const rec = apiEndpointStore.updateKey(kid, body || {}) + const rec = await apiEndpointStore.updateKey(kid, body || {}) if (!rec) return json(res, 404, { ok: false, error: { message: 'key not found' } }) - apiScheduler.reload(apiEndpointStore.listRaw()) - const ep = apiEndpointStore.getRaw(rec.endpoint_id) + apiScheduler.reload(await apiEndpointStore.listRaw()) + const ep = await apiEndpointStore.getRaw(rec.endpoint_id) const item = publicEndpointView(ep)?.api_key_entries?.find((k) => k.id === kid) return json(res, 200, { ok: true, item }) } @@ -1908,8 +1929,8 @@ export function createPanelHandler(ctx) { return json(res, 200, { ok: true, mode: 'debug', - config: requestLog.snapshot(), - items: requestLog.listDebug({ + config: await requestLog.snapshot(), + items: await requestLog.listDebug({ limit, error_class: u.searchParams.get('error_class') || null, exclude_error_class: u.searchParams.get('exclude_error_class') || null, @@ -1936,11 +1957,11 @@ export function createPanelHandler(ctx) { q: u.searchParams.get('q') || null, owner_user_id: panelIdentity(req).role === 'user' ? req.panelUserId : u.searchParams.get('user_id') || null, } - const { items, total } = requestLog.queryNormal(filters) + const { items, total } = await requestLog.queryNormal(filters) return json(res, 200, { ok: true, mode: 'normal', - config: requestLog.snapshot(), + config: await requestLog.snapshot(), items, total, limit: filters.limit, @@ -1964,7 +1985,7 @@ export function createPanelHandler(ctx) { status_code: listed.status_code || (/^\d+$/.test(legacyStatus || '') ? legacyStatus : null), errors_only: legacyStatus === 'error', } - const { items, total, capped } = new UsageLogsView(getDb(), { + const { items, total, capped } = await new UsageLogsView(getDb(), { mutedErrorClasses: requestLog?.mutedErrorClasses, }).exportRows(filters) const stamp = new Date().toISOString().slice(0, 19).replace(/[:T]/g, '') @@ -1998,17 +2019,17 @@ export function createPanelHandler(ctx) { bucket, since, until, - totals: tenant ? null : requestLog.totals(), - buckets: requestLog.aggregate({ since, until, bucket, owner_user_id }), - window: requestLog.windowStats({ since, until, owner_user_id }), - muted_error_classes: requestLog.snapshot().muted_error_classes, - config: tenant ? null : requestLog.snapshot(), + totals: tenant ? null : await requestLog.totals(), + buckets: await requestLog.aggregate({ since, until, bucket, owner_user_id }), + window: await requestLog.windowStats({ since, until, owner_user_id }), + muted_error_classes: (await requestLog.snapshot()).muted_error_classes, + config: tenant ? null : await requestLog.snapshot(), }) } if (req.method === 'GET' && /^\/api\/panel\/request-logs\/[^/]+\/attempts$/.test(p)) { const requestId = p.split('/')[4] const owner_user_id = panelIdentity(req).role === 'user' ? req.panelUserId : null - if (owner_user_id && !requestLog.getDebug(requestId, { owner_user_id })) { + if (owner_user_id && !(await requestLog.getDebug(requestId, { owner_user_id }))) { return json(res, 404, { ok: false, error: { message: 'debug log not found' } }) } return json( @@ -2017,16 +2038,16 @@ export function createPanelHandler(ctx) { panel.ok({ request_id: requestId, attempts: - (typeof ctx.getAttemptsRepo === 'function' ? ctx.getAttemptsRepo() : ctx.attemptsRepo)?.list?.( + (await (typeof ctx.getAttemptsRepo === 'function' ? ctx.getAttemptsRepo() : ctx.attemptsRepo)?.list?.( requestId, - ) || [], + )) || [], }), ) } if (req.method === 'GET' && /^\/api\/panel\/request-logs\/[^/]+$/.test(p)) { const id = p.split('/').pop() const owner_user_id = panelIdentity(req).role === 'user' ? req.panelUserId : null - const rec = requestLog.getDebug(id, { owner_user_id }) + const rec = await requestLog.getDebug(id, { owner_user_id }) if (!rec) return json(res, 404, { ok: false, error: { message: 'debug log not found' } }) return json(res, 200, { ok: true, item: rec }) } @@ -2038,31 +2059,39 @@ export function createPanelHandler(ctx) { const tenantId = () => (panelIdentity(req).role === 'user' ? req.panelUserId : null) if (req.method === 'GET' && p === '/api/panel/usage-logs') { const u = new URL(req.url, 'http://x') - return json(res, 200, panel.ok(usageLogsView().listBatch(usageLogFilters(req, u)))) + return json(res, 200, panel.ok(await usageLogsView().listBatch(usageLogFilters(req, u)))) } if (req.method === 'GET' && p === '/api/panel/usage-logs/summary') { const u = new URL(req.url, 'http://x') - return json(res, 200, panel.ok(usageLogsView().summary(usageLogFilters(req, u)))) + return json(res, 200, panel.ok(await usageLogsView().summary(usageLogFilters(req, u)))) } if (req.method === 'GET' && p === '/api/panel/usage-logs/filter-options') { - return json(res, 200, panel.ok(usageLogsView().filterOptions({ owner_user_id: tenantId() }))) + return json(res, 200, panel.ok(await usageLogsView().filterOptions({ owner_user_id: tenantId() }))) } if (req.method === 'GET' && p === '/api/panel/usage-logs/session-suggestions') { const u = new URL(req.url, 'http://x') const q = u.searchParams.get('q') || '' const limit = u.searchParams.get('limit') - return json(res, 200, panel.ok(usageLogsView().sessionSuggestions({ q, limit, owner_user_id: tenantId() }))) + return json( + res, + 200, + panel.ok(await usageLogsView().sessionSuggestions({ q, limit, owner_user_id: tenantId() })), + ) } if (req.method === 'GET' && p === '/api/panel/usage-logs/active-sessions') { const u = new URL(req.url, 'http://x') const minutes = u.searchParams.get('minutes') const limit = u.searchParams.get('limit') - return json(res, 200, panel.ok(usageLogsView().activeSessions({ minutes, limit, owner_user_id: tenantId() }))) + return json( + res, + 200, + panel.ok(await usageLogsView().activeSessions({ minutes, limit, owner_user_id: tenantId() })), + ) } if (req.method === 'GET' && p === '/api/panel/usage-logs/overview') { const u = new URL(req.url, 'http://x') const tz = u.searchParams.get('tz') || 'UTC' - return json(res, 200, panel.ok(usageLogsView().overview({ tz, owner_user_id: tenantId() }))) + return json(res, 200, panel.ok(await usageLogsView().overview({ tz, owner_user_id: tenantId() }))) } // ---- Statistics page ---- @@ -2071,7 +2100,11 @@ export function createPanelHandler(ctx) { const q = statsParams(req, u, 'dimension') if (q.error) return json(res, 400, { ok: false, error: { message: q.error } }) const { range, dimension, tz, owner_user_id } = q - return json(res, 200, panel.ok(new StatisticsRepo(getDb()).statistics({ range, dimension, tz, owner_user_id }))) + return json( + res, + 200, + panel.ok(await new StatisticsRepo(getDb()).statistics({ range, dimension, tz, owner_user_id })), + ) } if (req.method === 'GET' && p === '/api/panel/statistics/leaderboard') { const u = new URL(req.url, 'http://x') @@ -2082,7 +2115,7 @@ export function createPanelHandler(ctx) { return json( res, 200, - panel.ok(new StatisticsRepo(getDb()).leaderboard({ range, scope, limit, tz, owner_user_id })), + panel.ok(await new StatisticsRepo(getDb()).leaderboard({ range, scope, limit, tz, owner_user_id })), ) } @@ -2093,16 +2126,16 @@ export function createPanelHandler(ctx) { const groupBy = u.searchParams.get('group_by') === 'key' ? 'key' : 'vm' const since = u.searchParams.get('from') || u.searchParams.get('since') || null const until = u.searchParams.get('until') || null - return json(res, 200, panel.ok(requestLog.ownerBilling({ ownerUserId, since, until, groupBy }))) + return json(res, 200, panel.ok(await requestLog.ownerBilling({ ownerUserId, since, until, groupBy }))) } // ---- Backups (local auto/manual, download, restore) ---- if (req.method === 'GET' && p === '/api/panel/backups') { return json(res, 200, { ok: true, - items: backupService.list(), - config: backupService.getSchedule(), - next_auto_at: backupService.nextAutoAt(), + items: await backupService.list(), + config: await backupService.getSchedule(), + next_auto_at: await backupService.nextAutoAt(), restoring: backupService.isRestoring, }) } @@ -2121,22 +2154,22 @@ export function createPanelHandler(ctx) { if (req.method === 'GET' && p === '/api/panel/backups/config') { return json(res, 200, { ok: true, - config: backupService.getSchedule(), - next_auto_at: backupService.nextAutoAt(), + config: await backupService.getSchedule(), + next_auto_at: await backupService.nextAutoAt(), }) } if (req.method === 'PUT' && p === '/api/panel/backups/config') { const body = await readBody(req, 8192).catch(() => ({})) try { - const config = backupService.updateSchedule(body || {}) - return json(res, 200, { ok: true, config, next_auto_at: backupService.nextAutoAt() }) + const config = await backupService.updateSchedule(body || {}) + return json(res, 200, { ok: true, config, next_auto_at: await backupService.nextAutoAt() }) } catch (e) { return json(res, 400, { ok: false, error: { message: String(e.message || e), code: e.code } }) } } if (req.method === 'GET' && /^\/api\/panel\/backups\/[^/]+\/download$/.test(p)) { const id = p.split('/')[4] - const rec = backupService.get(id) + const rec = await backupService.get(id) if (!rec || !rec.file_path || !fs.existsSync(rec.file_path)) { return json(res, 404, { ok: false, error: { message: 'backup file not found' } }) } @@ -2181,7 +2214,8 @@ export function createPanelHandler(ctx) { } if (req.method === 'DELETE' && /^\/api\/panel\/backups\/[^/]+$/.test(p)) { const id = p.split('/').pop() - if (!backupService.remove(id)) return json(res, 404, { ok: false, error: { message: 'backup not found' } }) + if (!(await backupService.remove(id))) + return json(res, 404, { ok: false, error: { message: 'backup not found' } }) return json(res, 200, { ok: true, deleted: id }) } @@ -2304,7 +2338,7 @@ export function createPanelHandler(ctx) { const previous = structuredClone(ctx.routingConfig) try { if (targets.all) { - persistRoutingPatch({ + await persistRoutingPatch({ inference: { ...(ctx.routingConfig.inference || {}), dataplane: parsed.value, @@ -2637,7 +2671,7 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 8192).catch(() => ({})) const nextOwner = body.user_id === null || body.user_id === '' ? null : normalizeOwnerId(body.user_id || body.owner_user_id) - if (nextOwner && !panelUsers.getById(nextOwner)) { + if (nextOwner && !(await panelUsers.getById(nextOwner))) { return json(res, 404, { ok: false, error: { message: 'user not found' } }) } const currentOrigin = vm.origin || VM_ORIGIN.platform @@ -3332,12 +3366,12 @@ export function createPanelHandler(ctx) { fs.unlinkSync(vmPath) invalidateVmSlotReadCache(cfg.paths.project, id) try { - vmTestSchedule.remove(id) + await vmTestSchedule.remove(id) } catch {} if (active === id) { clearActiveVm(cfg.paths.project) try { - clearActiveVmFromDb() + await clearActiveVmFromDb() } catch {} try { reloadActiveVm(cfg) @@ -3345,7 +3379,7 @@ export function createPanelHandler(ctx) { } // drop the DB credential mirror row too try { - removeVmFromDb(id) + await removeVmFromDb(id) } catch {} // optional chat side files try { @@ -3512,8 +3546,8 @@ export function createPanelHandler(ctx) { } try { const repo = ctx.accountQuota?.repo - const row = repo && typeof repo.get === 'function' ? repo.get(id) : null - if (row && typeof repo.save === 'function') repo.save({ ...row, email: null, name: id }) + const row = repo && typeof repo.get === 'function' ? await repo.get(id) : null + if (row && typeof repo.save === 'function') await repo.save({ ...row, email: null, name: id }) } catch {} const saved = getVm(cfg.paths.project, id) || vm return json( @@ -3866,7 +3900,7 @@ export function createPanelHandler(ctx) { const ident = panelIdentity(req) if (parsed.ok && ident.role === 'user') { const ownerId = normalizeOwnerId(req.panelUserId) - const quota = clampVmCreateQuota(panelUsers.getById(ownerId)?.vm_create_quota, 0) + const quota = clampVmCreateQuota((await panelUsers.getById(ownerId))?.vm_create_quota, 0) const used = countUserCreatedVms(listVms(cfg.paths.project), ownerId) if (!ownerId || quota <= 0 || used >= quota) { return json(res, 403, { @@ -3930,7 +3964,7 @@ export function createPanelHandler(ctx) { const ident = panelIdentity(req) if (ident.role === 'user') { const ownerId = normalizeOwnerId(req.panelUserId) - const quota = clampVmCreateQuota(panelUsers.getById(ownerId)?.vm_create_quota, 0) + const quota = clampVmCreateQuota((await panelUsers.getById(ownerId))?.vm_create_quota, 0) const used = countUserCreatedVms(existing, ownerId) if (!ownerId || quota <= 0 || used >= quota) { return json(res, 403, { @@ -4994,7 +5028,7 @@ export function createPanelHandler(ctx) { } // GET /api/panel/usage if (req.method === 'GET' && p === '/api/panel/usage') { - return json(res, 200, panel.buildUsage({ accountQuota, cfg, requestLog })) + return json(res, 200, await panel.buildUsage({ accountQuota, cfg, requestLog })) } // GET /api/panel/models if (req.method === 'GET' && p === '/api/panel/models') { @@ -5014,7 +5048,7 @@ export function createPanelHandler(ctx) { 200, panel.ok({ policy: pol, - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), platform: 'anthropic', }), @@ -5024,18 +5058,18 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 2 * 1024 * 1024) const platform = String(body.platform || url.searchParams.get('platform') || '').toLowerCase() if (platform === 'openai' || platform === 'gpt' || platform === 'codex') { - const saved = saveGptModelPolicy(body.policy || body) + const saved = await saveGptModelPolicy(body.policy || body) const models = listGptPolicyModels() return json(res, 200, panel.ok({ policy: saved, models, effective: models, platform: 'openai' })) } - const saved = saveModelPolicy(body.policy || body) + const saved = await saveModelPolicy(body.policy || body) seedModelCatalog() return json( res, 200, panel.ok({ policy: saved, - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), platform: 'anthropic', }), @@ -5046,18 +5080,18 @@ export function createPanelHandler(ctx) { const body = await readBody(req, 8192).catch(() => ({})) const platform = String(body.platform || url.searchParams.get('platform') || '').toLowerCase() if (platform === 'openai' || platform === 'gpt' || platform === 'codex') { - const saved = resetGptModelPolicy() + const saved = await resetGptModelPolicy() const models = listGptPolicyModels() return json(res, 200, panel.ok({ policy: saved, models, effective: models, platform: 'openai' })) } - const saved = resetModelPolicy() + const saved = await resetModelPolicy() seedModelCatalog() return json( res, 200, panel.ok({ policy: saved, - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), platform: 'anthropic', }), @@ -5078,7 +5112,7 @@ export function createPanelHandler(ctx) { 200, panel.ok({ policy: getModelPolicy(), - models: listPolicyModels(), + models: await listPolicyModels(), effective: listOfficialModels(), synced: status.added.length, added: status.added, @@ -5088,12 +5122,12 @@ export function createPanelHandler(ctx) { } // GET /api/panel/pricing — effective price catalog for the 计费 tab if (req.method === 'GET' && p === '/api/panel/pricing') { - return json(res, 200, panel.ok(buildPricing({ requestLog }))) + return json(res, 200, panel.ok(await buildPricing({ requestLog }))) } // PUT /api/panel/pricing/overrides { key, price|null } if (req.method === 'PUT' && p === '/api/panel/pricing/overrides') { const body = await readBody(req, 256 * 1024) - const saved = setPriceOverride(String(body?.key || ''), body?.price ?? null) + const saved = await setPriceOverride(String(body?.key || ''), body?.price ?? null) if (!saved.ok) { return json(res, 400, { ok: false, error: { code: 'invalid_price', message: saved.error } }) } @@ -5114,7 +5148,7 @@ export function createPanelHandler(ctx) { error: { code: 'invalid_range', message: 'since/until must be ISO times with since < until' }, }) } - const result = requestLog.repo.repriceRange({ + const result = await requestLog.repo.repriceRange({ since: new Date(since).toISOString(), until: new Date(until).toISOString(), }) @@ -5240,9 +5274,9 @@ export function createPanelHandler(ctx) { const { applied: appliedDuringSwitch, ...publicEngineRuntime } = engineRuntime let applied try { - applied = appliedDuringSwitch ?? persistRoutingPatch(body) + applied = appliedDuringSwitch ?? (await persistRoutingPatch(body)) } catch (error) { - restoreRoutingRuntime(previousRoutingConfig) + await restoreRoutingRuntime(previousRoutingConfig) return json(res, 503, { ok: false, error: { code: 'routing_persist_failed', message: String(error?.message || error) }, @@ -5282,11 +5316,11 @@ export function createPanelHandler(ctx) { return json(res, 200, panel.ok(cfg.distill)) } if (req.method === 'GET' && p === '/api/panel/refusal-guards') { - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'PUT' && p === '/api/panel/refusal-guards') { const body = await readBody(req, 8 * 1024).catch(() => ({})) - const patched = applyRefusalGuardPatch(new SettingsRepo(), body || {}) + const patched = await applyRefusalGuardPatch(new SettingsRepo(), body || {}) if (!patched.ok) { return json(res, 400, { ok: false, @@ -5297,18 +5331,18 @@ export function createPanelHandler(ctx) { }, }) } - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'DELETE' && /^\/api\/panel\/refusal-guards\/[0-9a-f]{64}$/i.test(p)) { const fp = p.split('/').pop() - const removed = new RefusalGuardsRepo().remove(fp) + const removed = await new RefusalGuardsRepo().remove(fp) if (!removed) { return json(res, 404, { ok: false, error: { type: 'not_found_error', code: 'refusal_guard_not_found', message: '指纹不存在' }, }) } - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'DELETE' && p === '/api/panel/refusal-guards') { const body = await readBody(req, 8 * 1024).catch(() => ({})) @@ -5322,21 +5356,21 @@ export function createPanelHandler(ctx) { }, }) } - new RefusalGuardsRepo().clear() - return json(res, 200, panel.ok(refusalGuardSnapshot())) + await new RefusalGuardsRepo().clear() + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'DELETE' && p === '/api/panel/refusal-device-blocks') { const body = await readBody(req, 8 * 1024).catch(() => ({})) const repo = new RefusalDeviceBlocksRepo() const deviceId = String(body?.device_id || '').trim() if (deviceId) { - if (!repo.remove(deviceId)) { + if (!(await repo.remove(deviceId))) { return json(res, 404, { ok: false, error: { type: 'not_found_error', code: 'refusal_device_not_found', message: 'device 不存在' }, }) } - return json(res, 200, panel.ok(refusalGuardSnapshot())) + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (body?.confirm !== true) { return json(res, 400, { @@ -5348,18 +5382,18 @@ export function createPanelHandler(ctx) { }, }) } - repo.clear() - return json(res, 200, panel.ok(refusalGuardSnapshot())) + await repo.clear() + return json(res, 200, panel.ok(await refusalGuardSnapshot())) } if (req.method === 'GET' && p === '/api/panel/protocol-entry') { - return json(res, 200, panel.ok(requestLog.protocolEntryStats())) + return json(res, 200, panel.ok(await requestLog.protocolEntryStats())) } if (req.method === 'GET' && p === '/api/panel/jev-intercept') { return json(res, 200, panel.ok(jevInterceptSnapshot())) } if (req.method === 'PUT' && p === '/api/panel/jev-intercept') { const body = await readBody(req, 64 * 1024).catch(() => ({})) - const patched = applyJevPatch(new SettingsRepo(), body || {}) + const patched = await applyJevPatch(new SettingsRepo(), body || {}) if (!patched.ok) { return json(res, 400, { ok: false, @@ -5862,7 +5896,7 @@ export function createPanelHandler(ctx) { // ---- intercept rules admin ---- if (p === '/admin/intercept/rules') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return if (req.method === 'GET') { return json(res, 200, { rules: reloadRules() }) } @@ -5884,11 +5918,11 @@ export function createPanelHandler(ctx) { } if (req.method === 'GET' && p === '/admin/vm/oauth') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 200, await oauthStatusWithWorker()) } if (req.method === 'POST' && p === '/admin/vm/oauth/refresh') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const body = await readBody(req, 4096).catch(() => ({})) const id = String(body.vm_id || getActiveVmId(cfg.paths.project) || '').trim() const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) @@ -5907,13 +5941,13 @@ export function createPanelHandler(ctx) { // Claude CLI inference/update was removed; workers are built and deployed separately. if (req.method === 'GET' && p === '/admin/vm/claude-code/version') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 410, { error: { code: 'claude_cli_removed', message: 'Claude CLI runtime was replaced by the Go slot worker' }, }) } if (req.method === 'POST' && p === '/admin/vm/claude-code/update') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return return json(res, 410, { error: { code: 'claude_cli_removed', message: 'Build and roll out the Go slot worker instead' }, }) @@ -5921,6 +5955,33 @@ export function createPanelHandler(ctx) { return false } + + return async (req, res, url) => { + if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) return handlePanel(req, res, url) + try { + await flushPanelWrites() + } catch (error) { + return persistenceFailure(res, error) + } + const state = {} + let result + res._panelResponseState = state + try { + result = await handlePanel(req, res, url) + } finally { + delete res._panelResponseState + } + try { + await flushPanelWrites() + } catch (error) { + return persistenceFailure(res, error) + } + if (state.response) { + ctx.json(...state.response) + return true + } + return result + } } function readTelemetryUploadLog(projectRoot, id) { diff --git a/src/lib/admin/panel-tenant.mjs b/src/lib/admin/panel-tenant.mjs index c6a3865c4..726774af2 100644 --- a/src/lib/admin/panel-tenant.mjs +++ b/src/lib/admin/panel-tenant.mjs @@ -64,10 +64,10 @@ export function denyIfUserCannotDeleteVm(req, res, vm, json) { return true } -export function denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId }) { +export async function denyIfUserMissesKey(req, res, { apiKeyStore, json, keyId }) { const ident = panelIdentity(req) if (ident.role !== 'user') return false - const rec = apiKeyStore?.repo?.getById?.(keyId) || null + const rec = (await apiKeyStore?.repo?.getById?.(keyId)) || null const owner = normalizeOwnerId(rec?.user_id) if (!rec || owner !== normalizeOwnerId(req.panelUserId)) { json(res, 404, { ok: false, error: { message: 'api key not found' } }) diff --git a/src/lib/admin/price-catalog.mjs b/src/lib/admin/price-catalog.mjs index 90390c293..282fe4e84 100644 --- a/src/lib/admin/price-catalog.mjs +++ b/src/lib/admin/price-catalog.mjs @@ -2,7 +2,7 @@ * Three-layer price catalog: override > models.dev > builtin. * ID matching is exact / alias / Claude date-strip — no fuzzy family regex. */ -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { BUILTIN_ALIASES, BUILTIN_RATES, BUILTIN_SOURCE } from './builtin-rates.mjs' export const PRICE_FIELDS = ['input', 'output', 'cache_read', 'cache_5m', 'cache_1h'] @@ -27,6 +27,7 @@ let injectedFetch = null let backgroundRefresh = true let catalogMemo = null let refreshInflight = null +let refreshPaused = false export function setPriceCatalogTestHooks(hooks = null) { if (!hooks) { @@ -296,12 +297,12 @@ function readOverrides() { return raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {} } -function persistOverrides(next) { - settingsRepo().set(SETTINGS_OVERRIDES, next) +async function persistOverrides(next) { + await settingsRepo().set(SETTINGS_OVERRIDES, next) } -function persistSnapshot(next) { - settingsRepo().set(SETTINGS_SNAPSHOT, next) +async function persistSnapshot(next) { + await settingsRepo().set(SETTINGS_SNAPSHOT, next) } function statusOf(snap) { @@ -366,7 +367,7 @@ export async function syncModelsDev({ fetchImpl } = {}) { anthropic: parsed.anthropic, openai: parsed.openai, } - persistSnapshot(snap) + await persistSnapshot(snap) invalidatePriceCatalog() return statusOf(snap) } catch (e) { @@ -377,7 +378,7 @@ export async function syncModelsDev({ fetchImpl } = {}) { anthropic: prev.anthropic || {}, openai: prev.openai || {}, } - persistSnapshot(snap) + await persistSnapshot(snap) invalidatePriceCatalog() return statusOf(snap) } @@ -394,7 +395,7 @@ function canPersist() { } export function maybeRefreshModelsDev({ now } = {}) { - if (!backgroundRefresh) return + if (!backgroundRefresh || refreshPaused) return if (!canPersist()) return if (refreshInflight) return const snap = readSnapshot() @@ -402,13 +403,22 @@ export function maybeRefreshModelsDev({ now } = {}) { const t = now ?? Date.now() if (fetchedAt && Number.isFinite(fetchedAt) && t - fetchedAt < MODELS_DEV_STALE_MS) return refreshInflight = Promise.resolve() - .then(() => syncModelsDev()) + .then(async () => await syncModelsDev()) .catch(() => {}) .finally(() => { refreshInflight = null }) } +export async function pausePriceCatalog() { + refreshPaused = true + while (refreshInflight) await refreshInflight +} + +export function resumePriceCatalog() { + refreshPaused = false +} + function storedPrice(entry, platform) { if (!entry || typeof entry !== 'object') return null const { updated_at: _updated, ...rest } = entry @@ -584,7 +594,7 @@ export function listPriceOverrides() { return clone(readOverrides()) } -export function setPriceOverride(key, price) { +export async function setPriceOverride(key, price) { const k = String(key || '').trim() if (!OVERRIDE_KEY_RE.test(k) || k === '*/*') { return { ok: false, error: 'invalid override key' } @@ -592,7 +602,7 @@ export function setPriceOverride(key, price) { const overrides = { ...readOverrides() } if (price == null) { delete overrides[k] - persistOverrides(overrides) + await persistOverrides(overrides) invalidatePriceCatalog() return { ok: true } } @@ -600,7 +610,7 @@ export function setPriceOverride(key, price) { const v = validatePrice(price, { platform: plat }) if (!v.ok) return { ok: false, error: v.error } overrides[k] = { ...v.price, updated_at: new Date().toISOString() } - persistOverrides(overrides) + await persistOverrides(overrides) invalidatePriceCatalog() return { ok: true } } diff --git a/src/lib/admin/pricing-view.mjs b/src/lib/admin/pricing-view.mjs index a7a1539a9..444b994a6 100644 --- a/src/lib/admin/pricing-view.mjs +++ b/src/lib/admin/pricing-view.mjs @@ -14,12 +14,12 @@ function policyPrices() { return out } -export function buildPricing({ requestLog = null } = {}) { +export async function buildPricing({ requestLog = null } = {}) { const policy = policyPrices() const inPolicy = new Set(Object.values(policy).flatMap((row) => (row ? [row.id] : []))) let unpriced = [] try { - unpriced = requestLog?.repo?.unpricedModels?.() || [] + unpriced = (await requestLog?.repo?.unpricedModels?.()) || [] } catch { unpriced = [] } diff --git a/src/lib/admin/resource-owner.mjs b/src/lib/admin/resource-owner.mjs index b03bf1156..29144f45b 100644 --- a/src/lib/admin/resource-owner.mjs +++ b/src/lib/admin/resource-owner.mjs @@ -49,13 +49,13 @@ export function ownerScopeFromApiKey(record, { apiKeyKind = null, ownerRole = nu return { type: 'user', userId } } -export function ownerScopeFromRequest(req, usersRepo = null) { +export async function ownerScopeFromRequest(req, usersRepo = null) { const rec = req?.apiKeyRecord || null let ownerRole = null const userId = normalizeOwnerId(rec?.user_id) if (userId && usersRepo?.getById) { try { - ownerRole = usersRepo.getById(userId)?.role || null + ownerRole = (await usersRepo.getById(userId))?.role || null } catch { ownerRole = null } diff --git a/src/lib/admin/routing-runtime.mjs b/src/lib/admin/routing-runtime.mjs index d37618333..88f1cd5a5 100644 --- a/src/lib/admin/routing-runtime.mjs +++ b/src/lib/admin/routing-runtime.mjs @@ -26,7 +26,7 @@ import { PoolScheduler, normalizePoolRouting } from '../pool/pool-scheduler.mjs' import { FailoverRunner } from '../pool/failover-runner.mjs' import { unitCircuit } from '../pool/unit-circuit.mjs' import { RateLimitService } from '../pool/rate-limit-service.mjs' -import { AccountRuntimeRepo } from '../db/repos/account-runtime-repo.mjs' +import { SchedulingRuntime } from '../pool/scheduling-runtime.mjs' import { RequestAttemptsRepo } from '../db/repos/request-attempts-repo.mjs' import { normalizeOfficialCcConfig } from '../oauth/official-cc-bootstrap.mjs' import { normalizeHealthProbeConfig } from './health-probe.mjs' @@ -429,7 +429,7 @@ export function createRoutingRuntime(ctx) { return applied } - function persistRoutingPatch(body = {}) { + async function persistRoutingPatch(body = {}) { const hasCodexQuota = body?.codex && Object.prototype.hasOwnProperty.call(body.codex, 'quota') if (hasCodexQuota) validateOpenAIQuotaPatch(body.codex.quota) validateInferenceWebsocketPatch(body) @@ -508,7 +508,7 @@ export function createRoutingRuntime(ctx) { syncTierDefaultsIntoRouting() if (body.logging) { routingConfig.logging = normalizeLoggingConfig({ ...(routingConfig.logging || {}), ...body.logging }) - ctx.requestLog.setConfig({ + await ctx.requestLog.setConfig({ mode: routingConfig.logging.mode, retainDays: routingConfig.logging.retain_days, debugRetainDays: routingConfig.logging.debug_retain_days, @@ -682,15 +682,15 @@ export function createRoutingRuntime(ctx) { }) } - function initPoolRuntime() { + async function initPoolRuntime() { const routingConfig = getRouting() - const runtimeRepo = new AccountRuntimeRepo() + const runtimeRepo = await new SchedulingRuntime().init() const attemptsRepo = new RequestAttemptsRepo() setRuntimeRepo(runtimeRepo) setAttemptsRepo(attemptsRepo) ctx.accountQuota.attachRuntimeRepo(runtimeRepo) try { - attemptsRepo.cleanup(routingConfig?.logging?.retain_days || 7) + await attemptsRepo.cleanup(routingConfig?.logging?.retain_days || 7) } catch {} const poolScheduler = new PoolScheduler({ projectRoot: ctx.cfg.paths.project, diff --git a/src/lib/admin/usage-profile.mjs b/src/lib/admin/usage-profile.mjs index 140e65ff4..dab6d408b 100644 --- a/src/lib/admin/usage-profile.mjs +++ b/src/lib/admin/usage-profile.mjs @@ -88,24 +88,24 @@ export function riskPatchForProfile(mode) { /** * @param {{ mode: string, persistRouting: (patch: object) => unknown, settings: { get: Function, set: Function }, distillFile: string, setDistill?: (rules: object) => void }} input */ -export function applyUsageProfile({ mode, persistRouting, settings, distillFile, setDistill } = {}) { +export async function applyUsageProfile({ mode, persistRouting, settings, distillFile, setDistill } = {}) { const profile = normalizeUsageProfile(mode) if (!profile) { const err = new Error('mode 必须是 personal 或 relay') err.code = 'invalid_profile' throw err } - const routing = persistRouting(routingPatchForProfile(profile)) + const routing = await persistRouting(routingPatchForProfile(profile)) const risk = riskPatchForProfile(profile) const distill = saveDistillRules(distillFile, risk.distill) if (typeof setDistill === 'function') setDistill(distill) - const jev = applyJevPatch(settings, risk.jev) + const jev = await applyJevPatch(settings, risk.jev) if (!jev.ok) { const err = new Error(jev.problems.join('；')) err.code = 'invalid_profile' throw err } - const refusal = applyRefusalGuardPatch(settings, risk.refusal) + const refusal = await applyRefusalGuardPatch(settings, risk.refusal) if (!refusal.ok) { const err = new Error(refusal.problems.join('；')) err.code = 'invalid_profile' diff --git a/src/lib/admin/vm-pools.mjs b/src/lib/admin/vm-pools.mjs index 9307f6cb7..131b9acf8 100644 --- a/src/lib/admin/vm-pools.mjs +++ b/src/lib/admin/vm-pools.mjs @@ -33,43 +33,43 @@ export function normalizePoolMembers(value, vms) { } return ids } -export function createVmPool(repo, { name, vm_ids, enabled = true } = {}, vms = []) { +export async function createVmPool(repo, { name, vm_ids, enabled = true } = {}, vms = []) { const ids = vm_ids == null ? null : normalizePoolMembers(vm_ids, vms) - const pool = repo.insert({ name: normalizePoolName(name), enabled: enabled !== false }) + const pool = await repo.insert({ name: normalizePoolName(name), enabled: enabled !== false }) if (ids == null) return pool try { - return repo.setMembers(pool.id, ids) + return await repo.setMembers(pool.id, ids) } catch (error) { - repo.remove(pool.id) + await repo.remove(pool.id) throw error } } -export function updateVmPool(repo, id, patch = {}, vms = []) { - let pool = repo.get(id) +export async function updateVmPool(repo, id, patch = {}, vms = []) { + let pool = await repo.get(id) if (!pool) return null if (patch.name != null) { - pool = repo.rename(id, normalizePoolName(patch.name)) + pool = await repo.rename(id, normalizePoolName(patch.name)) } if (patch.enabled != null) { if (typeof patch.enabled !== 'boolean') { throw Object.assign(new Error('enabled 必须是布尔值'), { code: 'invalid_vm_pool_enabled' }) } - pool = repo.setEnabled(id, patch.enabled) + pool = await repo.setEnabled(id, patch.enabled) } if (Object.prototype.hasOwnProperty.call(patch, 'vm_ids')) { - pool = repo.setMembers(id, normalizePoolMembers(patch.vm_ids, vms)) + pool = await repo.setMembers(id, normalizePoolMembers(patch.vm_ids, vms)) } return pool } -export function deleteVmPool(repo, id) { - if (!repo.get(id)) return { ok: false, error: 'vm_pool_not_found' } - const keys = repo.countKeys(id) +export async function deleteVmPool(repo, id) { + if (!(await repo.get(id))) return { ok: false, error: 'vm_pool_not_found' } + const keys = await repo.countKeys(id) if (keys > 0) { return { ok: false, error: 'vm_pool_in_use', keys } } - repo.remove(id) + await repo.remove(id) return { ok: true } } @@ -86,9 +86,9 @@ export function vmPoolHttpStatus(code) { return 400 } -export function assertVmPoolExists(repo, id) { +export async function assertVmPoolExists(repo, id) { if (!id) return - if (!repo.get(id)) { + if (!(await repo.get(id))) { throw Object.assign(new Error('账号池不存在'), { code: 'vm_pool_unknown' }) } } diff --git a/src/lib/admin/vm-test-chat.mjs b/src/lib/admin/vm-test-chat.mjs index cba5689c7..089f4aa37 100644 --- a/src/lib/admin/vm-test-chat.mjs +++ b/src/lib/admin/vm-test-chat.mjs @@ -76,7 +76,7 @@ function gptRank(id) { return mini * 1000 - major * 10 - minor } -export function listTestableModels(platform) { +export async function listTestableModels(platform) { const kind = normalizeTestPlatform(platform) const rank = (id) => { if (kind === 'openai' || isGptSeriesId(id)) return gptRank(id) @@ -100,17 +100,15 @@ export function listTestableModels(platform) { const effective = listOfficialModels().map((m) => map(m, 'effective')) const list = effective.length ? effective - : listPolicyModels() - .filter((m) => m.enabled !== false) - .map((m) => map(m, 'policy')) + : (await listPolicyModels()).filter((m) => m.enabled !== false).map((m) => map(m, 'policy')) const filtered = list.filter((m) => !isGptTestModel(m)) return filtered.sort((a, b) => rank(a.id) - rank(b.id) || String(a.id).localeCompare(String(b.id))) } -export function testModelsView(platform) { +export async function testModelsView(platform) { const kind = normalizeTestPlatform(platform) return { - items: listTestableModels(kind), + items: await listTestableModels(kind), platform: kind, protocol: kind === 'openai' ? 'openai.responses' : kind === 'anthropic' ? 'anthropic.messages' : null, inbound_path: kind === 'openai' ? '/v1/responses' : kind === 'anthropic' ? '/v1/messages' : null, @@ -207,8 +205,8 @@ export async function syncCodexCatalog({ let access = account.access_token || target.codex?.access_token const accountId = account.chatgpt_account_id || target.codex?.chatgpt_account_id const refreshToken = account.refresh_token || target.codex?.refresh_token - const fetchOnce = () => - fetchChatgptModelCatalog({ + const fetchOnce = async () => + await fetchChatgptModelCatalog({ accessToken: access, accountId, proxyUrl, @@ -273,7 +271,7 @@ export async function syncCodexCatalog({ } } const models = [...merged.values()] - const { added } = syncGptIdsIntoPolicy(models, { displayNameOf: (id) => modelsDevMeta('openai', id)?.name }) + const { added } = await syncGptIdsIntoPolicy(models, { displayNameOf: (id) => modelsDevMeta('openai', id)?.name }) return { ok: true, vm_id: usedVm, @@ -321,7 +319,7 @@ export async function resolveTestModels({ }) if (live.ok) source = 'chatgpt' } - return { ...testModelsView(kind), source } + return { ...(await testModelsView(kind)), source } } function extractText(body) { @@ -851,11 +849,11 @@ async function postV1Loopback({ path, baseUrl, apiKey, inbound, headers, vmId, t } async function postV1Messages(opts) { - return postV1Loopback({ ...opts, path: '/v1/messages' }) + return await postV1Loopback({ ...opts, path: '/v1/messages' }) } async function postV1Responses(opts) { - return postV1Loopback({ ...opts, path: '/v1/responses' }) + return await postV1Loopback({ ...opts, path: '/v1/responses' }) } /** @@ -968,7 +966,7 @@ export async function runVmTestChat(opts = {}) { let model = String(opts.model || '').trim() if (!model) { - const models = listTestableModels(codex ? 'openai' : 'anthropic') + const models = await listTestableModels(codex ? 'openai' : 'anthropic') model = models[0]?.id || (codex ? 'gpt-5.4' : 'claude-haiku-4-5-20251001') } push('info', `模型 ${model}`) diff --git a/src/lib/admin/vm-test-schedule.mjs b/src/lib/admin/vm-test-schedule.mjs index 99f622c6c..4134a886d 100644 --- a/src/lib/admin/vm-test-schedule.mjs +++ b/src/lib/admin/vm-test-schedule.mjs @@ -166,8 +166,8 @@ function assertCron(cron, timezone) { } } -function defaultDialogParams(vm) { - const models = listTestableModels(isCodexVm(vm) ? 'openai' : 'anthropic') +async function defaultDialogParams(vm) { + const models = await listTestableModels(isCodexVm(vm) ? 'openai' : 'anthropic') const model = models[0]?.id || '' if (!model) { throw new VmTestScheduleError('no test model available for this vm', 'test_model_missing', 400) @@ -353,6 +353,7 @@ export class VmTestScheduleRunner { this.now = now this.jobs = new Map() this.running = new Set() + this.writeFailure = null this.started = false } @@ -404,15 +405,15 @@ export class VmTestScheduleRunner { } } - saveDefaultTemplate(input) { + async saveDefaultTemplate(input) { const structure = normalizeScheduleStructure(input) compileStructuredSchedule(structure) - this._settings().set(VM_TEST_SCHEDULE_DEFAULT_KEY, structure) + await this._settings().set(VM_TEST_SCHEDULE_DEFAULT_KEY, structure) return structure } - clearDefaultTemplate() { - this._settings().remove?.(VM_TEST_SCHEDULE_DEFAULT_KEY) + async clearDefaultTemplate() { + await this._settings().remove?.(VM_TEST_SCHEDULE_DEFAULT_KEY) return null } @@ -472,18 +473,18 @@ export class VmTestScheduleRunner { return null } - save(vmId, input) { + async save(vmId, input) { if (!isValidVmId(vmId)) throw new VmTestScheduleError('invalid vm id', 'invalid_vm_id') const vm = this._vm(vmId) if (!vm) throw new VmTestScheduleError('vm not found', 'vm_not_found', 404) const previous = this._settings().get(keyFor(vmId)) || {} const config = normalizeVmTestSchedule(input, vm) - this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) + await this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) if (this.started) this._syncJob(vmId, config) return this.get(vmId) } - applyScheduleStructure(vmId, structure, { preserveDialog = true } = {}) { + async applyScheduleStructure(vmId, structure, { preserveDialog = true } = {}) { if (!isValidVmId(vmId)) throw new VmTestScheduleError('invalid vm id', 'invalid_vm_id') const vm = this._vm(vmId) if (!vm) throw new VmTestScheduleError('vm not found', 'vm_not_found', 404) @@ -500,7 +501,7 @@ export class VmTestScheduleRunner { reasoning_effort: previous.reasoning_effort ?? null, } } else { - dialog = defaultDialogParams(vm) + dialog = await defaultDialogParams(vm) } const enabled = previous.enabled === true const config = normalizeVmTestSchedule( @@ -511,7 +512,7 @@ export class VmTestScheduleRunner { }, vm, ) - this._settings().set(keyFor(vmId), { + await this._settings().set(keyFor(vmId), { ...config, enabled, ...savedResult(previous), @@ -526,7 +527,7 @@ export class VmTestScheduleRunner { * A slot of the other kind (Claude vs Codex) keeps its own dialog params, since * a model of one kind cannot run on the other; switch and timing still apply. */ - syncSchedules({ scope, vmIds = [], schedule, test = null }) { + async syncSchedules({ scope, vmIds = [], schedule, test = null }) { const structure = normalizeScheduleStructure(schedule) const full = test == null ? null : normalizeSyncTest(test, structure) let targets = [] @@ -547,16 +548,16 @@ export class VmTestScheduleRunner { continue } if (full) { - this._applyFullTest(vmId, structure, full) + await this._applyFullTest(vmId, structure, full) updated.push(vmId) continue } const hadPlan = !!this._settings().get(keyFor(vmId)) - this.applyScheduleStructure(vmId, structure, { preserveDialog: hadPlan }) + await this.applyScheduleStructure(vmId, structure, { preserveDialog: hadPlan }) if (!hadPlan) { const stored = this._settings().get(keyFor(vmId)) if (stored) { - this._settings().set(keyFor(vmId), { ...stored, enabled: false }) + await this._settings().set(keyFor(vmId), { ...stored, enabled: false }) this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) } @@ -566,7 +567,7 @@ export class VmTestScheduleRunner { return { updated, missing, schedule: structure } } - _applyFullTest(vmId, structure, full) { + async _applyFullTest(vmId, structure, full) { const vm = this._vm(vmId) const previous = this._settings().get(keyFor(vmId)) || {} let dialog @@ -578,22 +579,27 @@ export class VmTestScheduleRunner { max_tokens: previous.max_tokens ?? null, reasoning_effort: previous.reasoning_effort ?? null, } - } else dialog = defaultDialogParams(vm) + } else dialog = await defaultDialogParams(vm) const config = normalizeVmTestSchedule({ enabled: full.enabled, ...structure, ...dialog }, vm) - this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) + await this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) if (this.started) this._syncJob(vmId, config) } - remove(vmId) { + async remove(vmId) { this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) - this._settings().remove?.(keyFor(vmId)) + await this._settings().remove?.(keyFor(vmId)) return null } - start() { + async start() { this.started = true - this.reload() + await this.reload() + } + + async flush() { + while (this.running.size) await new Promise((resolve) => setTimeout(resolve, 10)) + if (this.writeFailure) throw this.writeFailure } stop() { @@ -602,7 +608,7 @@ export class VmTestScheduleRunner { this.jobs.clear() } - reload() { + async reload() { for (const job of this.jobs.values()) job.stop() this.jobs.clear() const all = this._settings().all?.() || {} @@ -611,7 +617,7 @@ export class VmTestScheduleRunner { const vmId = key.slice(VM_TEST_SCHEDULE_PREFIX.length) const vm = this._vm(vmId) if (!vm) { - this._settings().remove?.(key) + await this._settings().remove?.(key) continue } const schedule = this._stored(vmId) @@ -623,8 +629,10 @@ export class VmTestScheduleRunner { this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) if (!this.started || !schedule.enabled || !this._vm(vmId)) return - const job = new Cron(schedule.cron, { timezone: schedule.timezone, mode: '5-part', catch: true, unref: true }, () => - this._run(vmId), + const job = new Cron( + schedule.cron, + { timezone: schedule.timezone, mode: '5-part', catch: true, unref: true }, + async () => await this._run(vmId), ) this.jobs.set(vmId, job) } @@ -632,40 +640,47 @@ export class VmTestScheduleRunner { async _run(vmId) { if (!this.started || this.running.has(vmId)) return if (!this._vm(vmId)) { - this.remove(vmId) + await this.remove(vmId) return } const schedule = this._stored(vmId) if (!schedule?.enabled) return this.running.add(vmId) const startedAt = this.now() - let result try { - result = await this.runTest(vmId, { - model: schedule.model, - prompt: schedule.prompt, - max_tokens: schedule.max_tokens, - reasoning_effort: schedule.reasoning_effort, - }) - } catch (error) { - result = { ok: false, error } + let result + try { + result = await this.runTest(vmId, { + model: schedule.model, + prompt: schedule.prompt, + max_tokens: schedule.max_tokens, + reasoning_effort: schedule.reasoning_effort, + }) + } catch (error) { + result = { ok: false, error } + } + if (!this.started) return + + const current = this._settings().get(keyFor(vmId)) + if (!current || String(current.revision || '') !== schedule.revision || !this._vm(vmId)) return + const duration = Number(result?.duration_ms) + const lastDuration = + Number.isSafeInteger(duration) && duration >= 0 ? duration : Math.max(0, this.now() - startedAt) + const lastOk = result?.ok === true + try { + await this._settings().set(keyFor(vmId), { + ...current, + last_run_at: new Date(startedAt).toISOString(), + last_ok: lastOk, + last_duration_ms: lastDuration, + last_error: lastOk ? null : compactError(result?.error), + }) + } catch (error) { + this.writeFailure ||= error + throw error + } } finally { this.running.delete(vmId) } - if (!this.started) return - - const current = this._settings().get(keyFor(vmId)) - if (!current || String(current.revision || '') !== schedule.revision || !this._vm(vmId)) return - const duration = Number(result?.duration_ms) - const lastDuration = - Number.isSafeInteger(duration) && duration >= 0 ? duration : Math.max(0, this.now() - startedAt) - const lastOk = result?.ok === true - this._settings().set(keyFor(vmId), { - ...current, - last_run_at: new Date(startedAt).toISOString(), - last_ok: lastOk, - last_duration_ms: lastDuration, - last_error: lastOk ? null : compactError(result?.error), - }) } } diff --git a/src/lib/cluster/cluster-manager.mjs b/src/lib/cluster/cluster-manager.mjs index 90338f7dd..4fe95bc48 100644 --- a/src/lib/cluster/cluster-manager.mjs +++ b/src/lib/cluster/cluster-manager.mjs @@ -5,6 +5,7 @@ */ import crypto from 'node:crypto' +import { withTransaction } from '../db/database.mjs' import fs from 'node:fs' import net from 'node:net' import path from 'node:path' @@ -137,6 +138,9 @@ export class ClusterManager { this._localAt = 0 this._localInflight = null this.repo = repo + this.nodes = new Map() + this.platforms = new Map() + this.directLinksState = new Map() // Unix sockets need a native filesystem; WSL drvfs (/mnt/) refuses them. this.socketDir = socketDir || path.join(dataDir, 'cluster') this.vmsOnNode = vmsOnNode @@ -153,8 +157,27 @@ export class ClusterManager { this._healthTimer = null } - start() { - for (const node of this.repo.list()) this._attach(node) + async reload() { + const nodes = await this.repo.list() + const directLinks = this.repo.directLinks ? await this.repo.directLinks() : new Map() + const platforms = await Promise.all( + nodes.map(async (node) => [node.id, this.repo.platform ? await this.repo.platform(node.id) : 'linux']), + ) + this.nodes = new Map(nodes.map((node) => [node.id, node])) + this.directLinksState = directLinks + this.platforms = new Map(platforms) + } + + async rebind(db) { + await this.stop() + this.repo = new this.repo.constructor(db) + await this.start() + await this.restoreSlotRelays() + } + + async start() { + await this.reload() + for (const node of this.nodes.values()) this._attach(node) this._healthTimer = setInterval(() => { for (const id of this.links.keys()) this._checkHealth(id) }, HEALTH_INTERVAL_MS) @@ -216,8 +239,8 @@ export class ClusterManager { } async _targetFor(id) { - const node = this.repo.get(id) - const secrets = this.repo.getSecrets(id) + const node = this.nodes.get(id) + const secrets = await this.repo.getSecrets(id) if (!node || !secrets) throw new ClusterError(404, 'node_not_found', '节点不存在') return { host: node.host, @@ -283,7 +306,7 @@ export class ClusterManager { }) } - _view(node, directLinks = this.repo.directLinks()) { + _view(node, directLinks = this.directLinksState) { const link = this.links.get(node.id) return { ...node, @@ -297,12 +320,12 @@ export class ClusterManager { } list() { - const directLinks = this.repo.directLinks() - return this.repo.list().map((node) => this._view(node, directLinks)) + const directLinks = this.directLinksState + return [...this.nodes.values()].map((node) => this._view(node, directLinks)) } get(id) { - const node = this.repo.get(id) + const node = this.nodes.get(id) if (!node) throw new ClusterError(404, 'node_not_found', '节点不存在') return this._view(node) } @@ -328,7 +351,7 @@ export class ClusterManager { /** Ping the gate; a good answer lifts the SSH fallback early, a bad one starts it. */ async _probeDirect(id) { - const cfg = this.repo.getDirectLink(id) + const cfg = await this.repo.getDirectLink(id) if (!cfg?.certSha256) return const checked_at = new Date().toISOString() try { @@ -351,7 +374,7 @@ export class ClusterManager { * the backoff too; a missing slot socket does not, it is not a link fault. */ async _dialDirect(nodeId, vmId, kind) { - const cfg = this.repo.getDirectLink(nodeId) + const cfg = await this.repo.getDirectLink(nodeId) if (!cfg?.certSha256) return null if ((this.directState.get(nodeId)?.downUntil || 0) > Date.now()) return null const down = (err) => { @@ -372,7 +395,7 @@ export class ClusterManager { async _ensureGate(id, { strict = false } = {}) { if (this.nodePlatform(id) === 'darwin') return - const cfg = this.repo.getDirectLink(id) + const cfg = await this.repo.getDirectLink(id) const link = this.links.get(id) if (!cfg || link?.state !== 'ready' || !link.client) return const client = link.client @@ -380,7 +403,7 @@ export class ClusterManager { const host = await this.remoteHost(id) const result = await ensureNodeGate({ client, host, docker: sshDocker(client) }, cfg, this.gateBin) if (result.certSha256 !== cfg.certSha256) { - this.repo.setDirectLinkCert(id, result.certSha256) + await this.repo.setDirectLinkCert(id, result.certSha256) this.logger.log(`[cluster] slot gate on ${id}: pinned certificate ${result.certSha256.slice(0, 16)}…`) } this._directPatch(id, { gate_error: null }) @@ -394,11 +417,12 @@ export class ClusterManager { /** Enable or move the direct link. Token and certificate pin are kept across edits and never returned. */ async setDirectLink(id, body) { - if (!this.repo.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') + if (!this.nodes.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') this.assertLinuxNode(id, '直连通道') const { address, port } = parseDirectLinkInput(body) - const token = this.repo.getDirectLink(id)?.token || newGateToken() - this.repo.setDirectLink(id, { address, port, token }) + const token = (await this.repo.getDirectLink(id))?.token || newGateToken() + await this.repo.setDirectLink(id, { address, port, token }) + this.directLinksState = await this.repo.directLinks() this.directState.delete(id) await this._ensureGate(id).catch(() => {}) // The gate needs a moment to bind after a (re)create. @@ -411,8 +435,9 @@ export class ClusterManager { } async clearDirectLink(id) { - if (!this.repo.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') - this.repo.clearDirectLink(id) + if (!this.nodes.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') + await this.repo.clearDirectLink(id) + this.directLinksState.delete(id) this.directState.delete(id) const link = this.links.get(id) if (link?.state === 'ready' && link.client) { @@ -453,33 +478,35 @@ export class ClusterManager { docker, uname: unameText, platform: nodePlatformOf(unameText?.split(/\s+/)[0]), - existing_id: this.repo.findByEndpoint(spec.host, spec.port, spec.username), + existing_id: await this.repo.findByEndpoint(spec.host, spec.port, spec.username), } } finally { client.end() } } - add(input) { + async add(input) { const spec = parseNodeInput(input, { requireHostKey: true }) - if (spec.jump_node_id && !this.repo.get(spec.jump_node_id)) { + if (spec.jump_node_id && !this.nodes.get(spec.jump_node_id)) { throw new ClusterError(400, 'invalid_jump', '跳板节点不存在') } - if (this.repo.findByEndpoint(spec.host, spec.port, spec.username)) { + if (await this.repo.findByEndpoint(spec.host, spec.port, spec.username)) { throw new ClusterError(409, 'node_exists', '该主机与用户已接入') } - const node = this.repo.insert({ + const node = await this.repo.insert({ ...spec, id: `node-${crypto.randomBytes(4).toString('hex')}`, host_key_alg: spec.host_key_alg || 'unknown', }) + this.nodes.set(node.id, node) + this.platforms.set(node.id, 'linux') this._attach(node) this._nodesChanged() return this._view(node) } - _prepareEdit(id, input) { - const node = this.repo.get(id) + async _prepareEdit(id, input) { + const node = this.nodes.get(id) if (!node) throw new ClusterError(404, 'node_not_found', '节点不存在') validateNodeEditPatch(node, input) const labelOnly = Object.keys(input).every((key) => key === 'label' || key === 'expected_updated_at') @@ -488,7 +515,7 @@ export class ClusterManager { if (label.length > 64) throw new ClusterError(400, 'invalid_label', '备注过长') return { id, node, spec: { label }, connectionEdit: false, labelOnly: true, vms: this.vmsOnNode(id) } } - const next = mergeNodeEdit(node, this.repo.getSecrets(id), input) + const next = mergeNodeEdit(node, await this.repo.getSecrets(id), input) const spec = parseNodeInput(next, { requireHostKey: true }) const link = this.links.get(id) const vms = this.vmsOnNode(id) @@ -504,12 +531,12 @@ export class ClusterManager { vms, placementVersion: nodePlacementVersion(vms), } - this._assertEditCurrent(edit) + await this._assertEditCurrent(edit) return edit } - _assertEditCurrent(edit) { - const current = this.repo.get(edit.id) + async _assertEditCurrent(edit) { + const current = await this.repo.get(edit.id) if (!current) throw new ClusterError(404, 'node_not_found', '节点不存在') if (current.updated_at !== edit.node.updated_at) throw nodeEditConflict() if (!edit.connectionEdit) return @@ -524,10 +551,10 @@ export class ClusterManager { if (edit.vms.length && edit.spec.username !== edit.node.username) { throw new ClusterError(409, 'node_username_in_use', '节点上仍有虚拟机，不能更换 SSH 用户；请先迁移或删除虚拟机') } - if (this.repo.dependentsOf(edit.id).length) { + if ((await this.repo.dependentsOf(edit.id)).length) { throw new ClusterError(409, 'node_in_use', '仍有节点经由它跳转，暂不支持修改该节点的连接配置') } - const duplicate = this.repo.findByEndpoint(edit.spec.host, edit.spec.port, edit.spec.username) + const duplicate = await this.repo.findByEndpoint(edit.spec.host, edit.spec.port, edit.spec.username) if (duplicate && duplicate !== edit.id) throw new ClusterError(409, 'node_exists', '该主机与用户已接入') if (this.installs.get(edit.id)?.status === 'running') throw new ClusterError(409, 'node_busy', '节点正在安装 Docker，请稍后重试') @@ -537,7 +564,7 @@ export class ClusterManager { async _probeEdit(edit) { let previousHost = null if (edit.vms.length && edit.client && edit.link.state === 'ready') previousHost = await this.remoteHost(edit.id) - this._assertEditCurrent(edit) + await this._assertEditCurrent(edit) const sock = await this._jumpSock(edit.spec.jump_node_id, edit.spec.host, edit.spec.port) const result = await connectSsh({ host: edit.spec.host, @@ -581,7 +608,7 @@ export class ClusterManager { } else { host = await readNodeHost(result.client).catch(() => null) } - this._assertEditCurrent(edit) + await this._assertEditCurrent(edit) if (!candidate.live()) throw nodeEditConflict() return { ...candidate, host, docker } } catch (err) { @@ -592,14 +619,14 @@ export class ClusterManager { } async probeUpdate(id, input) { - const edit = this._prepareEdit(id, input) + const edit = await this._prepareEdit(id, input) if (!edit.connectionEdit) { return { host_key: { alg: edit.node.host_key_alg, sha256: edit.node.host_key_sha256 }, docker: this.health.get(id)?.docker || { ok: false, error: '未探测' }, platform: this.nodePlatform(id), vm_count: edit.vms.length, - dependent_count: this.repo.dependentsOf(id).length, + dependent_count: (await this.repo.dependentsOf(id)).length, } } const candidate = await this._probeEdit(edit) @@ -618,38 +645,60 @@ export class ClusterManager { } async update(id, input) { - const edit = this._prepareEdit(id, input) + const edit = await this._prepareEdit(id, input) const candidate = edit.connectionEdit ? await this._probeEdit(edit) : null let adopted = false + let committed = false + let adoptedGeneration try { - this._assertEditCurrent(edit) - if (candidate && !candidate.live()) throw nodeEditConflict() - let saved - try { - saved = edit.labelOnly - ? this.repo.updateLabel(id, edit.spec.label, edit.node.updated_at) - : this.repo.update(id, edit.spec, edit.node.updated_at) - } catch (err) { - if (String(err.message).includes('UNIQUE constraint failed: cluster_nodes.')) { - throw new ClusterError(409, 'node_exists', '该主机与用户已接入') + const saved = await withTransaction(this.repo.db, async () => { + await this._assertEditCurrent(edit) + const saved = edit.labelOnly + ? await this.repo.updateLabel(id, edit.spec.label, edit.node.updated_at) + : await this.repo.update(id, edit.spec, edit.node.updated_at) + if (!saved) throw nodeEditConflict() + if (candidate) { + // Reconnect and placement may change while PostgreSQL executes the CAS. + // Keep this final check and adoption synchronous; a rejection rolls back the CAS. + if ( + !candidate.live() || + this.links.get(id) !== edit.link || + edit.link.generation !== edit.generation || + edit.link.client !== edit.client || + nodePlacementVersion(this.vmsOnNode(id)) !== edit.placementVersion + ) + throw nodeEditConflict() + if (!edit.link.adopt(candidate, edit.generation)) throw nodeEditConflict() + candidate.cleanup() + adopted = true + adoptedGeneration = edit.link.generation } - throw err - } - if (!saved) throw nodeEditConflict() + return saved + }) + committed = true + this.nodes.set(id, saved) if (candidate) { - // No await between the final version check, SQLite CAS and adoption. Existing relay closures retain this link. - candidate.cleanup() - edit.link.adopt(candidate, edit.generation) - adopted = true - if (candidate.host) { + if (edit.link.generation !== adoptedGeneration || edit.link.client !== candidate.client) { + // A reconnect during COMMIT must resolve its target from the committed record. + edit.link.reconnect() + } else if (candidate.host) { this.hosts.set(candidate.client, Promise.resolve(candidate.host)) - this._learnHost(id, candidate.host) + await this._learnHost(id, candidate.host) } for (const [ticket, entry] of this.tickets) if (entry.nodeId === id) this.tickets.delete(ticket) } return this._view(saved) + } catch (error) { + if (adopted && !committed) { + const persisted = await this.repo.get(id).catch(() => null) + if (persisted) this.nodes.set(id, persisted) + else this.nodes.delete(id) + edit.link.reconnect() + } + if (error.code === '23505') throw new ClusterError(409, 'node_exists', '该主机与用户已接入') + throw error } finally { - if (candidate && !adopted) { + if (candidate && (!adopted || !committed)) { candidate.cleanup() candidate.client.end() } @@ -673,8 +722,8 @@ export class ClusterManager { } async remove(id) { - if (!this.repo.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') - const dependents = this.repo.dependentsOf(id) + if (!this.nodes.get(id)) throw new ClusterError(404, 'node_not_found', '节点不存在') + const dependents = await this.repo.dependentsOf(id) if (dependents.length) { throw new ClusterError(409, 'node_in_use', `仍有节点经由它跳转：${dependents.join(', ')}`) } @@ -682,6 +731,7 @@ export class ClusterManager { if (placed.length) { throw new ClusterError(409, 'node_has_vms', `节点上仍有虚拟机：${placed.join(', ')}，先删除它们`) } + await this.repo.remove(id) this.links.get(id)?.stop() this.links.delete(id) await this.bridges.get(id)?.stop() @@ -690,7 +740,9 @@ export class ClusterManager { this.installs.delete(id) this.directState.delete(id) this.hostFacts.delete(id) - this.repo.remove(id) + this.nodes.delete(id) + this.platforms.delete(id) + this.directLinksState.delete(id) this._nodesChanged() } @@ -731,20 +783,25 @@ export class ClusterManager { const cached = this.hosts.get(client) if (cached) return cached const pending = readNodeHost(client) - pending.then( - (host) => this._learnHost(id, host), - () => this.hosts.delete(client), - ) + .then(async (host) => { + await this._learnHost(id, host) + return host + }) + .catch((error) => { + this.hosts.delete(client) + throw error + }) this.hosts.set(client, pending) return pending } - _learnHost(id, host) { + async _learnHost(id, host) { try { - if (!this.repo.get(id)) return + if (!this.nodes.get(id)) return this.hostFacts.set(id, host) if (host.platform && this.nodePlatform(id) !== host.platform) { - this.repo.setPlatform(id, host.platform) + await this.repo.setPlatform(id, host.platform) + this.platforms.set(id, host.platform) this.logger.log(JSON.stringify({ event: 'cluster-node-platform', node: id, platform: host.platform })) } } catch (err) { @@ -755,7 +812,7 @@ export class ClusterManager { /** 'linux' | 'darwin'. Unknown or unreadable (older schema) is Linux: the Docker runtime is the default. */ nodePlatform(id) { try { - return this.repo.platform(id) === 'darwin' ? 'darwin' : 'linux' + return this.platforms.get(id) === 'darwin' ? 'darwin' : 'linux' } catch { return 'linux' } @@ -889,7 +946,7 @@ export class ClusterManager { /** Boot: relays exist before any link is ready so the scheduler's first health dial finds a socket. */ async restoreSlotRelays() { - for (const node of this.repo.list()) { + for (const node of [...this.nodes.values()]) { for (const vm of this.vmsOnNode(node.id)) { await this.ensureSlotRelays(node.id, vm.id).catch((err) => this.logger.warn(`[cluster] slot relay ${vm.id} on ${node.id}: ${err.message}`), @@ -925,7 +982,7 @@ export class ClusterManager { /** A jumped node sees the jump host as SSH_CLIENT, so only direct ready nodes can observe our public address. */ _natObserver() { - for (const node of this.repo.list()) { + for (const node of [...this.nodes.values()]) { if (node.jump_node_id) continue const link = this.links.get(node.id) if (link?.state === 'ready' && link.client) return { node, client: link.client } diff --git a/src/lib/cluster/cluster-routes.mjs b/src/lib/cluster/cluster-routes.mjs index 4542ff2cd..aa9961ba0 100644 --- a/src/lib/cluster/cluster-routes.mjs +++ b/src/lib/cluster/cluster-routes.mjs @@ -35,27 +35,27 @@ function clampInt(v, lo, hi, dflt) { } export function createClusterRoutes({ manager, json, readBody, ok }) { - function fail(res, err) { + function fail(res, err, respond = json) { const status = err instanceof ClusterError ? err.status : 500 const code = err instanceof ClusterError ? err.code : 'cluster_error' - return json(res, status, { ok: false, error: { type: 'cluster', code, message: err.message || String(err) } }) + return respond(res, status, { ok: false, error: { type: 'cluster', code, message: err.message || String(err) } }) } - async function route(req, res, url) { + async function route(req, res, url, respond = json) { const p = url.pathname const m = req.method - if (p === `${PREFIX}/nodes` && m === 'GET') return json(res, 200, ok({ items: manager.list() })) + if (p === `${PREFIX}/nodes` && m === 'GET') return respond(res, 200, ok({ items: manager.list() })) if (p === `${PREFIX}/local` && m === 'GET') { - return json(res, 200, ok(await manager.localStatus({ refresh: url.searchParams.get('refresh') === '1' }))) + return respond(res, 200, ok(await manager.localStatus({ refresh: url.searchParams.get('refresh') === '1' }))) } if (p === `${PREFIX}/nodes` && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 201, ok(manager.add(body))) + return respond(res, 201, ok(await manager.add(body))) } if (p === `${PREFIX}/nodes/probe` && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.probe(body))) + return respond(res, 200, ok(await manager.probe(body))) } const nm = NODE_RE.exec(p) @@ -63,31 +63,31 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { const id = nm[1] const sub = nm[2] || '' - if (sub === '' && m === 'GET') return json(res, 200, ok(manager.get(id))) + if (sub === '' && m === 'GET') return respond(res, 200, ok(manager.get(id))) if (sub === '' && m === 'PATCH') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.update(id, body))) + return respond(res, 200, ok(await manager.update(id, body))) } if (sub === '/probe' && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.probeUpdate(id, body))) + return respond(res, 200, ok(await manager.probeUpdate(id, body))) } if (sub === '' && m === 'DELETE') { await manager.remove(id) - return json(res, 200, ok({ id })) + return respond(res, 200, ok({ id })) } - if (sub === '/reconnect' && m === 'POST') return json(res, 200, ok(manager.reconnect(id))) + if (sub === '/reconnect' && m === 'POST') return respond(res, 200, ok(manager.reconnect(id))) if (sub === '/direct-link' && m === 'PUT') { const body = await readBody(req, MAX_BODY) - return json(res, 200, ok(await manager.setDirectLink(id, body))) + return respond(res, 200, ok(await manager.setDirectLink(id, body))) } - if (sub === '/direct-link' && m === 'DELETE') return json(res, 200, ok(await manager.clearDirectLink(id))) - if (sub === '/shell-ticket' && m === 'POST') return json(res, 200, ok(manager.issueShellTicket(id))) + if (sub === '/direct-link' && m === 'DELETE') return respond(res, 200, ok(await manager.clearDirectLink(id))) + if (sub === '/shell-ticket' && m === 'POST') return respond(res, 200, ok(manager.issueShellTicket(id))) - if (sub === '/docker/info' && m === 'GET') return json(res, 200, ok(await dockerInfo(manager.docker(id)))) + if (sub === '/docker/info' && m === 'GET') return respond(res, 200, ok(await dockerInfo(manager.docker(id)))) if (sub === '/preflight' && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json( + return respond( res, 200, ok( @@ -101,7 +101,7 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { if (sub === '/slot-image' && m === 'POST') { const body = await readBody(req, MAX_BODY) manager.get(id) - return json( + return respond( res, 202, ok( @@ -111,20 +111,20 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { } if (sub === '/slot-image' && m === 'GET') { manager.get(id) - return json( + return respond( res, 200, ok(slotImageJob(id, kernelOf(url.searchParams.get('kernel')), url.searchParams.get('runtime_type'))), ) } - if (sub === '/docker/install' && m === 'POST') return json(res, 202, ok(manager.startDockerInstall(id))) - if (sub === '/docker/install' && m === 'GET') return json(res, 200, ok(manager.get(id).install)) + if (sub === '/docker/install' && m === 'POST') return respond(res, 202, ok(manager.startDockerInstall(id))) + if (sub === '/docker/install' && m === 'GET') return respond(res, 200, ok(manager.get(id).install)) if (sub === '/docker/containers' && m === 'GET') { - return json(res, 200, ok({ items: await listContainers(manager.docker(id)) })) + return respond(res, 200, ok({ items: await listContainers(manager.docker(id)) })) } if (sub === '/docker/containers' && m === 'POST') { const body = await readBody(req, MAX_BODY) - return json(res, 201, ok(await createAndStartContainer(manager.docker(id), body))) + return respond(res, 201, ok(await createAndStartContainer(manager.docker(id), body))) } const cm = /^\/docker\/containers\/([^/]+)(?:\/(start|stop|restart|logs))?$/.exec(sub) @@ -133,35 +133,35 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { const action = cm[2] if (!action && m === 'DELETE') { await removeContainer(manager.docker(id), cid) - return json(res, 200, ok({ id: cid })) + return respond(res, 200, ok({ id: cid })) } if (action === 'logs' && m === 'GET') { const tail = clampInt(url.searchParams.get('tail'), 1, 5000, 200) - return json(res, 200, ok(await containerLogs(manager.docker(id), cid, { tail }))) + return respond(res, 200, ok(await containerLogs(manager.docker(id), cid, { tail }))) } if (action && action !== 'logs' && m === 'POST') { await containerAction(manager.docker(id), cid, action) - return json(res, 200, ok({ id: cid, action })) + return respond(res, 200, ok({ id: cid, action })) } } return false } - async function handle(req, res, url) { + async function handle(req, res, url, respond = json) { if (!url.pathname.startsWith(`${PREFIX}/`)) return false try { - const handled = await route(req, res, url) + const handled = await route(req, res, url, respond) if (handled === false) - return json(res, 404, { ok: false, error: { type: 'cluster', code: 'not_found', message: 'not found' } }) + return respond(res, 404, { ok: false, error: { type: 'cluster', code: 'not_found', message: 'not found' } }) return true } catch (err) { if (!(err instanceof ClusterError) && err?.status) { - return json(res, err.status, { + return respond(res, err.status, { ok: false, error: { type: 'cluster', code: 'bad_request', message: err.message }, }) } - return fail(res, err) + return fail(res, err, respond) } } diff --git a/src/lib/core/refusal-guard.mjs b/src/lib/core/refusal-guard.mjs index 6ece998ec..5c8092d39 100644 --- a/src/lib/core/refusal-guard.mjs +++ b/src/lib/core/refusal-guard.mjs @@ -186,7 +186,7 @@ export function normalizeRefusalSimilarity(value) { } /** Apply a panel PUT. Unknown similarity values are rejected, not coerced. */ -export function applyRefusalGuardPatch(settings, body = {}) { +export async function applyRefusalGuardPatch(settings, body = {}) { const problems = [] if (Object.hasOwn(body, 'enabled') && typeof body.enabled !== 'boolean') problems.push('enabled 必须是布尔') if (Object.hasOwn(body, 'similarity_enabled') && typeof body.similarity_enabled !== 'boolean') { @@ -199,13 +199,14 @@ export function applyRefusalGuardPatch(settings, body = {}) { problems.push('similarity 只能是 80、85、90、95') } if (problems.length) return { ok: false, problems } - if (typeof body.enabled === 'boolean') settings.set(REFUSAL_GUARD_SETTING, body.enabled) - if (typeof body.similarity_enabled === 'boolean') settings.set(REFUSAL_SIMILARITY_SETTING, body.similarity_enabled) + if (typeof body.enabled === 'boolean') await settings.set(REFUSAL_GUARD_SETTING, body.enabled) + if (typeof body.similarity_enabled === 'boolean') + await settings.set(REFUSAL_SIMILARITY_SETTING, body.similarity_enabled) if (typeof body.device_block_enabled === 'boolean') { - settings.set(REFUSAL_DEVICE_BLOCK_SETTING, body.device_block_enabled) + await settings.set(REFUSAL_DEVICE_BLOCK_SETTING, body.device_block_enabled) } if (Object.hasOwn(body, 'similarity')) { - settings.set(REFUSAL_SIMILARITY_VALUE_SETTING, normalizeRefusalSimilarity(body.similarity)) + await settings.set(REFUSAL_SIMILARITY_VALUE_SETTING, normalizeRefusalSimilarity(body.similarity)) } return { ok: true } } @@ -214,7 +215,7 @@ export function applyRefusalGuardPatch(settings, body = {}) { * Pre-hop decision. Device ban wins, then the exact fingerprint, then a * user-text signature at or above the configured similarity. Does not write. */ -export function matchStoredRefusal({ +export async function matchStoredRefusal({ inbound, body = inbound, headers, @@ -226,17 +227,17 @@ export function matchStoredRefusal({ } = {}) { const deviceId = inboundRefusalDeviceId({ inbound, body, headers }) if (deviceBlockEnabled && deviceId && typeof devices?.get === 'function') { - const banned = devices.get(deviceId) + const banned = await devices.get(deviceId) if (banned) return { kind: 'device', deviceId, row: banned } } if (!repo || typeof repo.get !== 'function') return null const fingerprint = refusalFingerprint(body, inbound) - const exact = repo.get(fingerprint) + const exact = await repo.get(fingerprint) if (exact) return { kind: 'exact', deviceId, fingerprint, row: exact } if (!similarityEnabled || typeof repo.nearest !== 'function') return null const signature = refusalSignature(refusalUserDocument(inbound, body)) if (!signature) return null - const near = repo.nearest(signature, similarity) + const near = await repo.nearest(signature, similarity) if (!near) return null return { kind: 'similar', deviceId, fingerprint: near.fingerprint, score: near.score, row: near } } diff --git a/src/lib/core/security.mjs b/src/lib/core/security.mjs index f9a368d5b..97d92d2d0 100644 --- a/src/lib/core/security.mjs +++ b/src/lib/core/security.mjs @@ -142,13 +142,13 @@ function saveSessions() { loadSessions() -export function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra = {}) { +export async function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra = {}) { if (ttlMs && typeof ttlMs === 'object') { extra = ttlMs ttlMs = extra.ttlMs || SESSION_TTL_MS } const token = 'kin-panel-' + crypto.randomBytes(24).toString('hex') - const role = extra.role || (_lookupRole && _lookupRole(username)) || 'admin' + const role = extra.role || (_lookupRole && (await _lookupRole(username))) || 'admin' panelSessions.set(panelSessionKey(token), { user: username, role, @@ -158,7 +158,7 @@ export function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra = {}) return token } -export function verifyPanelSession(token) { +export async function verifyPanelSession(token) { if (!token || !token.startsWith('kin-panel-')) return null const key = panelSessionKey(token) let s = panelSessions.get(key) @@ -174,7 +174,7 @@ export function verifyPanelSession(token) { return null } if (!s.role && _lookupRole) { - s.role = _lookupRole(s.user) || null + s.role = (await _lookupRole(s.user)) || null } if (!s.role) { const admin = getPanelAdmin() @@ -213,12 +213,12 @@ export function revokePanelSessionsForUser(username, { exceptToken } = {}) { /** * @returns {{ username: string, role: string, id?: string } | null} */ -export function verifyPanelLogin(username, password) { +export async function verifyPanelLogin(username, password) { if (_authenticateUser) { - const rec = _authenticateUser(username, password) + const rec = await _authenticateUser(username, password) if (rec) return { username: rec.username, role: rec.role || 'user', id: rec.id } // Existing DB user (including disabled) must not fall through to env password. - if (_lookupRole && _lookupRole(username)) return null + if (_lookupRole && (await _lookupRole(username))) return null } const admin = getPanelAdmin() if (!admin.password) return null diff --git a/src/lib/db/pending-writes.mjs b/src/lib/db/pending-writes.mjs new file mode 100644 index 000000000..37d9a3af5 --- /dev/null +++ b/src/lib/db/pending-writes.mjs @@ -0,0 +1,20 @@ +/** Response-finish writes are tracked so maintenance and shutdown can drain them. */ +const pending = new Set() +let failure = null + +export function trackPendingWrite(operation) { + const tracked = Promise.resolve(operation) + pending.add(tracked) + tracked + .catch((error) => { + failure ||= error + console.warn(`[db] response persistence failed: ${error.code || error.name}`) + }) + .finally(() => pending.delete(tracked)) + return tracked +} + +export async function flushPendingWrites() { + while (pending.size) await Promise.allSettled([...pending]) + if (failure) throw failure +} diff --git a/src/lib/db/settings-state.mjs b/src/lib/db/settings-state.mjs new file mode 100644 index 000000000..da4771201 --- /dev/null +++ b/src/lib/db/settings-state.mjs @@ -0,0 +1,40 @@ +/** Explicitly loaded settings for synchronous policy callbacks. Writes remain asynchronous. */ +import { getDb } from './database.mjs' +import { SettingsRepo } from './repos/settings-repo.mjs' + +export async function loadSettingsState(db = getDb()) { + const settings = await new SettingsRepo(db).all() + db.settingsState = new Map(Object.entries(settings)) + return db.settingsState +} + +export class RuntimeSettings { + constructor(db = getDb()) { + this.db = db + this.repo = new SettingsRepo(db) + } + + get(key, fallback = null) { + if (!this.db.settingsState) throw new Error('settings not loaded; await loadSettingsState() before startup') + return this.db.settingsState.has(key) ? structuredClone(this.db.settingsState.get(key)) : fallback + } + + has(key) { + if (!this.db.settingsState) throw new Error('settings not loaded; await loadSettingsState() before startup') + return this.db.settingsState.has(key) + } + + async set(key, value) { + const next = structuredClone(value) + return this.repo.set(key, next) + } + + async remove(key) { + await this.repo.remove(key) + } + + all() { + if (!this.db.settingsState) throw new Error('settings not loaded; await loadSettingsState() before startup') + return Object.fromEntries([...this.db.settingsState].map(([key, value]) => [key, structuredClone(value)])) + } +} diff --git a/src/lib/oauth/credential-refresh-monitor.mjs b/src/lib/oauth/credential-refresh-monitor.mjs index b15b1b8d4..652a236dc 100644 --- a/src/lib/oauth/credential-refresh-monitor.mjs +++ b/src/lib/oauth/credential-refresh-monitor.mjs @@ -179,6 +179,9 @@ export function createCredentialRefreshMonitor(opts = {}) { getSnapshot: () => lastRun, runOnce, start, + flush: async () => { + while (inflight) await inflight + }, stop, } } diff --git a/src/lib/oauth/usage-probe-monitor.mjs b/src/lib/oauth/usage-probe-monitor.mjs index dddcff5a6..20b457629 100644 --- a/src/lib/oauth/usage-probe-monitor.mjs +++ b/src/lib/oauth/usage-probe-monitor.mjs @@ -366,6 +366,9 @@ export function createUsageProbeMonitor(opts = {}) { runOnce, probeNow, start, + flush: async () => { + while (inflight || probing.size) await Promise.allSettled([...(inflight ? [inflight] : []), ...probing.values()]) + }, stop, } } diff --git a/src/lib/pool/account-quota.mjs b/src/lib/pool/account-quota.mjs index 4f300be1d..e685e5e31 100644 --- a/src/lib/pool/account-quota.mjs +++ b/src/lib/pool/account-quota.mjs @@ -1,5 +1,5 @@ /** - * Per-account Claude usage quota tracker (SQLite-backed). + * Per-account Claude usage quota tracker backed by PostgreSQL. * * Scheduling and list Extra come from Messages response headers. * Official GET /api/oauth/usage is a manual calibration hop that writes @@ -10,7 +10,9 @@ */ import { resolveStoreDb } from '../db/database.mjs' +import { addInteger } from '../db/integers.mjs' import { AccountsRepo } from '../db/repos/accounts-repo.mjs' +import { WriteQueue } from './write-queue.mjs' import { computeWeeklySplit, weeklySplitConfig } from './weekly-split.mjs' import { applyEffectiveWindows, @@ -60,7 +62,12 @@ export { estimateRequestRisk } export class AccountQuota { constructor({ dataDir, db, config, accounts }) { this.db = resolveStoreDb({ db, dataDir }) - this.repo = new AccountsRepo(this.db) + this.persistence = new AccountsRepo(this.db) + this.writes = this.db.schedulingWriteQueue ||= new WriteQueue(this.db) + this.rows = new Map() + this.allocations = new Map() + this.seedAccounts = accounts || [] + this.repo = this._workingSet() this.config = config?.quota || { safety_ratio: 0.85, weekly_safety_ratio: 0.8, @@ -78,17 +85,68 @@ export class AccountQuota { this.sessions = config?.sessions instanceof SessionLimitRegistry ? config.sessions : new SessionLimitRegistry() // vm_id → normalized `vm.policy.quota`; vm.json is the source of truth, this is the hot-path copy. this.vmQuota = new Map() - // seed accounts - for (const a of accounts || []) this.ensure(a) } - /** Kept for API compat + post-restore hook (no in-memory cache to refresh). */ - reload() {} + /** Load persisted counters and allocations; restore reload omits startup seed accounts. */ + async init({ seed = true } = {}) { + for (const queue of this.db.pendingWriteQueues || []) await queue.flush() + for (const row of await this.persistence.list()) { + this.rows.set(row.account_id, row) + this.allocations.set(row.account_id, await this.persistence.recentAllocations(row.account_id, 50)) + } + if (seed) for (const account of this.seedAccounts) this.ensure(account) + await this.flush() + return this + } + + async reload() { + await this.flush() + this.rows.clear() + this.allocations.clear() + return await this.init({ seed: false }) + } + + flush() { + return this.writes.flush() + } + + _workingSet() { + const save = (account, insert = false) => { + const next = structuredClone(account) + this.rows.set(next.account_id, next) + this.writes.enqueue(async () => { + const existing = insert ? await this.persistence.get(next.account_id) : true + return this.persistence[existing ? 'save' : 'insert'](next) + }) + return structuredClone(next) + } + return { + get: (id) => structuredClone(this.rows.get(id) || null), + list: () => [...this.rows.values()].map((row) => structuredClone(row)), + insert: (row) => save(row, true), + save: (row) => save(row), + remove: (id) => { + const removed = this.rows.delete(id) + this.allocations.delete(id) + this.writes.enqueue(() => this.persistence.remove(id)) + return removed + }, + addAllocation: (id, input, { max = 50 } = {}) => { + const allocation = { ...input, at: input.at || new Date().toISOString() } + this.allocations.set(id, [...(this.allocations.get(id) || []), structuredClone(allocation)].slice(-max)) + this.writes.enqueue(() => this.persistence.addAllocation(id, allocation, { max })) + }, + recentAllocations: (id, limit = 5) => structuredClone((this.allocations.get(id) || []).slice(-limit)), + allocationCount: (id) => (this.allocations.get(id) || []).length, + } + } /** Re-bind to a fresh DB connection (after backup restore). */ - rebind(db) { + async rebind(db) { + await this.flush() this.db = db - this.repo = new AccountsRepo(db) + this.persistence = new AccountsRepo(db) + return await this.reload() } /** @@ -237,12 +295,12 @@ export class AccountQuota { usage.cache_read_tokens != null || usage.cache_creation_tokens != null) ) { - acc.tokens_in += Number(usage.input_tokens) || 0 - acc.tokens_out += Number(usage.output_tokens) || 0 - acc.cache_read_tokens = (acc.cache_read_tokens || 0) + (Number(usage.cache_read_tokens) || 0) - acc.cache_creation_tokens = (acc.cache_creation_tokens || 0) + (Number(usage.cache_creation_tokens) || 0) + acc.tokens_in = addInteger(acc.tokens_in, usage.input_tokens || 0) + acc.tokens_out = addInteger(acc.tokens_out, usage.output_tokens || 0) + acc.cache_read_tokens = addInteger(acc.cache_read_tokens, usage.cache_read_tokens || 0) + acc.cache_creation_tokens = addInteger(acc.cache_creation_tokens, usage.cache_creation_tokens || 0) } - if (countRequest) acc.requests += 1 + if (countRequest) acc.requests = addInteger(acc.requests, 1) if (exhausted) applyHeaderExhausted(acc, h) @@ -486,10 +544,10 @@ export class AccountQuota { const shouldCount = countRequest == null ? usageBody?.input_tokens != null : !!countRequest if (usageBody?.input_tokens != null) { - acc.tokens_in += Number(usageBody.input_tokens) || 0 - acc.tokens_out += Number(usageBody.output_tokens) || 0 + acc.tokens_in = addInteger(acc.tokens_in, usageBody.input_tokens || 0) + acc.tokens_out = addInteger(acc.tokens_out, usageBody.output_tokens || 0) } - if (shouldCount) acc.requests += 1 + if (shouldCount) acc.requests = addInteger(acc.requests, 1) this.repo.addAllocation(accountId, { at: new Date().toISOString(), diff --git a/src/lib/pool/api-protocol.mjs b/src/lib/pool/api-protocol.mjs index 1f4ca9a8a..047a7235c 100644 --- a/src/lib/pool/api-protocol.mjs +++ b/src/lib/pool/api-protocol.mjs @@ -148,9 +148,9 @@ export async function runApiInference({ const retry = Number(upstream.headers['retry-after'] || 30) const until = new Date(Date.now() + Math.max(1, retry) * 1000).toISOString() try { - store.setKeyCooldown(picked.key.id, until) + await store.setKeyCooldown(picked.key.id, until) } catch {} - scheduler.reload(store.listRaw()) + scheduler.reload(await store.listRaw()) } const resultBase = { diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 4984830ec..df2068a86 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -537,7 +537,7 @@ export class FailoverRunner { async run(args = {}) { const sessionKey = String(args.stickyKey || '') - if (!sessionKey) return this.runOnce(args) + if (!sessionKey) return await this.runOnce(args) const previous = this.sessionTails.get(sessionKey) || Promise.resolve() let releaseTurn const turn = new Promise((resolve) => { @@ -789,7 +789,7 @@ export class FailoverRunner { if (selected.seatMoved) seatMovedTo = selected.vmId bindAll({ accountId: selected.accountId, vmId: selected.vmId, slotIndex: pinnedSlot }, { countHit: false }) const attemptStarted = Date.now() - this.attemptsRepo?.begin?.({ + await this.attemptsRepo?.begin?.({ requestId, attemptNo, vmId: selected.vmId, @@ -858,7 +858,7 @@ export class FailoverRunner { noteHop(result) if (result) result.committed = result.committed || committed if (signal?.aborted || isClientCancelledResult(result)) { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: result?.status ?? null, errorScope: 'request', action: 'stop', @@ -908,7 +908,7 @@ export class FailoverRunner { } catch {} } const terminalState = result?.terminalState || (result?.ok ? 'unknown' : 'error') - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: result?.status ?? null, errorScope: policy.scope, action: policy.action, @@ -988,7 +988,7 @@ export class FailoverRunner { const invalidCache = error?.status === 400 && error.body?.error?.code === 'invalid_cache_control' noteHop(invalidCache ? { upstreamExecutions: 0 } : null) if (error?.code === 'response_sent') { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 403, errorScope: 'request', action: 'stop', @@ -1008,7 +1008,7 @@ export class FailoverRunner { } } if (invalidCache) { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, errorScope: 'request', action: 'stop', @@ -1026,7 +1026,7 @@ export class FailoverRunner { } } if (signal?.aborted || error?.code === 'selection_cancelled' || error?.code === 'request_cancelled') { - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, errorScope: 'request', action: 'stop', @@ -1066,7 +1066,7 @@ export class FailoverRunner { lastPolicy = policy this.noteUnitHealth(selected, policy, result) notifyProxyFailure(this.onProxyFailure, selected, policy) - this.attemptsRepo?.complete?.(requestId, attemptNo, { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, errorScope: policy.scope, action: policy.action, diff --git a/src/lib/pool/scheduling-runtime.mjs b/src/lib/pool/scheduling-runtime.mjs new file mode 100644 index 000000000..db233fac5 --- /dev/null +++ b/src/lib/pool/scheduling-runtime.mjs @@ -0,0 +1,208 @@ +/** Loaded scheduling state keeps eligibility checks and reservations synchronous. */ +import { AccountRuntimeRepo } from '../db/repos/account-runtime-repo.mjs' +import { WriteQueue } from './write-queue.mjs' + +export class SchedulingRuntime { + constructor(db) { + this.persistence = new AccountRuntimeRepo(db) + this.rows = new Map() + this.writes = this.persistence.db.schedulingWriteQueue ||= new WriteQueue(this.persistence.db) + } + + async init() { + await this.flush() + this.rows = new Map((await this.persistence.list()).map((row) => [row.account_id, row])) + return this + } + + flush() { + return this.writes.flush() + } + + get(id) { + return structuredClone(this.rows.get(id) || null) + } + + list() { + return [...this.rows.values()].map((row) => structuredClone(row)) + } + + upsert(input = {}) { + if (!input.account_id || !input.vm_id) return null + const current = this.get(input.account_id) || {} + const next = { + cooldown_until: null, + cooldown_reason: null, + last_used_at: null, + credential_generation: 0, + refresh_status: null, + worker_heartbeat_at: null, + worker_status: null, + rate_limited_at: null, + rate_limit_reset_at: null, + overload_until: null, + session_window_start: null, + session_window_end: null, + session_window_status: null, + ...current, + ...structuredClone(input), + status: input.status ?? current.status ?? 'unknown', + priority: Number(input.priority ?? current.priority ?? 0), + weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), + model_states: structuredClone(input.model_states ?? current.model_states ?? {}), + updated_at: new Date().toISOString(), + } + this.rows.set(next.account_id, next) + this.writes.enqueue(() => this.persistence.upsert(next)) + return structuredClone(next) + } + + remove(id) { + const removed = this.rows.delete(id) + this.writes.enqueue(() => this.persistence.remove(id)) + return removed + } + + updateWindow( + accountId, + { + vmId = null, + rateLimitedAt, + rateLimitResetAt, + overloadUntil, + sessionWindowStart, + sessionWindowEnd, + sessionWindowStatus, + } = {}, + ) { + const current = this.get(accountId) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, + } + if (!current.vm_id && vmId) current.vm_id = vmId + if (!current.vm_id) return null + if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt + if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt + if (overloadUntil !== undefined) current.overload_until = overloadUntil + if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart + if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd + if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus + return this.upsert(current) + } + + markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { + const current = this.get(accountId) || { + account_id: accountId, + vm_id: vmId, + model_states: {}, + } + if (model) { + current.model_states = { ...(current.model_states || {}) } + current.model_states[model] = { + status, + cooldown_until: Number(until) || null, + reason: reason || null, + updated_at: new Date().toISOString(), + } + } else { + current.status = status + current.cooldown_until = Number(until) || null + current.cooldown_reason = reason || null + // Structured mirror of the protocol-level limit (sub2api account columns). + // overload_until is only the 529 hard column (RateLimitService.handle529). + // A 502 failover also uses reason provider_overloaded for 15s; writing it + // here paints every hopped VM as 过载冷却 and removes them from the pool. + if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { + current.rate_limited_at = Date.now() + current.rate_limit_reset_at = Number(until) || null + } + } + return this.upsert(current) + } + + static AUTH_COOLDOWN = /authentication_failed_after_refresh|permission_denied/i + + static GRANT_REVOKE_COOLDOWN = + /oauth_revoked|oauth_invalid_grant|invalid_grant|token has been revoked|oauth_no_refresh/i + + isAuthCooldown(state) { + return SchedulingRuntime.AUTH_COOLDOWN.test(String(state?.cooldown_reason || '')) + } + + isGrantRevokeCooldown(state) { + return ( + SchedulingRuntime.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || '')) || + (String(state?.status || '') === 'disabled' && + SchedulingRuntime.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || state?.status || ''))) + ) + } + + clearAuthCooldown(accountId, { vmId = null } = {}) { + const current = this.get(accountId) + if (!current || !this.isAuthCooldown(current)) return false + return !!this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + }) + } + + clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { + const current = this.get(accountId) + if (!current || !this.isGrantRevokeCooldown(current)) return false + return !!this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + }) + } + + clearAccountCooldown(accountId, { vmId = null } = {}) { + const current = this.get(accountId) + if (!current) return false + return !!this.upsert({ + ...current, + vm_id: current.vm_id || vmId, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + rate_limit_reset_at: null, + overload_until: null, + model_states: {}, + }) + } + + clearExpired(now = Date.now()) { + for (const state of this.list()) { + if (!state.vm_id) continue + let changed = false + if (state.cooldown_until && state.cooldown_until <= now) { + state.status = 'ready' + state.cooldown_until = null + state.cooldown_reason = null + changed = true + } + if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { + state.rate_limit_reset_at = null + changed = true + } + if (state.overload_until && state.overload_until <= now) { + state.overload_until = null + changed = true + } + const models = { ...(state.model_states || {}) } + for (const [model, modelState] of Object.entries(models)) { + if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { + delete models[model] + changed = true + } + } + if (changed) this.upsert({ ...state, model_states: models }) + } + } +} diff --git a/src/lib/pool/sticky-router.mjs b/src/lib/pool/sticky-router.mjs index ed3dcbd5b..b4e5309a7 100644 --- a/src/lib/pool/sticky-router.mjs +++ b/src/lib/pool/sticky-router.mjs @@ -7,6 +7,7 @@ import crypto from 'node:crypto' import { ENVELOPE_NEEDLES, extractPrompt } from '../core/distill-detect.mjs' import { resolveStoreDb } from '../db/database.mjs' import { StickyRepo } from '../db/repos/sticky-repo.mjs' +import { WriteQueue } from './write-queue.mjs' import { claudeCodeAgentRootSession, extractCallerSession, @@ -204,18 +205,70 @@ export function childDeclaredWithoutParent(body = {}, headers = {}) { export class StickyRouter { constructor({ dataDir, db, config }) { this.db = resolveStoreDb({ db, dataDir }) - this.repo = new StickyRepo(this.db) + this.persistence = new StickyRepo(this.db) + this.writes = new WriteQueue(this.db) + this.sessions = {} this.config = mergeStickyConfig(config) - this.repo.dropAliasKeys() + this.repo = { + get: (key) => structuredClone(this.sessions[key] || null), + all: () => structuredClone(this.sessions), + upsert: (key, entry) => { + const next = { ...entry, device_id: entry.device_id || this.sessions[key]?.device_id || null } + this.sessions[key] = structuredClone(next) + this.writes.enqueue(() => this.persistence.upsert(key, next)) + }, + remove: (key) => { + delete this.sessions[key] + this.writes.enqueue(() => this.persistence.remove(key)) + }, + removeByAccount: ({ accountId = null, vmId = null } = {}) => { + let removed = 0 + for (const [key, entry] of Object.entries(this.sessions)) { + if ((accountId && entry.account_id === accountId) || (vmId && entry.vm_id === vmId)) { + delete this.sessions[key] + removed += 1 + } + } + if (removed) this.writes.enqueue(() => this.persistence.removeByAccount({ accountId, vmId })) + return removed + }, + purgeExpired: (now) => { + let removed = 0 + for (const [key, entry] of Object.entries(this.sessions)) { + if (entry.expires_at != null && entry.expires_at < now) { + delete this.sessions[key] + removed += 1 + } + } + if (removed) this.writes.enqueue(() => this.persistence.purgeExpired(now)) + return removed + }, + } } /** Kept for API compat + post-restore hook (state lives in DB). */ - reload() {} + async init() { + for (const queue of this.db.pendingWriteQueues || []) await queue.flush() + await this.persistence.dropAliasKeys() + this.sessions = await this.persistence.all() + return this + } + + async reload() { + await this.flush() + return await this.init() + } + + flush() { + return this.writes.flush() + } /** Re-bind to a fresh DB connection (after backup restore). */ - rebind(db) { + async rebind(db) { + await this.flush() this.db = db - this.repo = new StickyRepo(db) + this.persistence = new StickyRepo(db) + return await this.init() } isolateKey(raw, req = null) { diff --git a/src/lib/pool/thinking-owner.mjs b/src/lib/pool/thinking-owner.mjs index de6485fd9..4f9454945 100644 --- a/src/lib/pool/thinking-owner.mjs +++ b/src/lib/pool/thinking-owner.mjs @@ -9,8 +9,7 @@ * block with its thinking text, and is dropped when that text is empty. * * Key: sha256 of the signature's length, first 100 and last 100 characters - * (`data` for redacted_thinking). The table lives outside the versioned - * migrations so this local addition cannot take a migration number later. + * (`data` for redacted_thinking). The table is created by the PostgreSQL baseline migration. */ import crypto from 'node:crypto' import { getDb } from '../db/database.mjs' @@ -48,23 +47,15 @@ export class ThinkingOwnerStore { return null } if (this.boundDb === db) return this.sql - db.exec(`CREATE TABLE IF NOT EXISTS thinking_signature_owners ( - key TEXT PRIMARY KEY, - account_id TEXT NOT NULL, - created_at INTEGER NOT NULL - )`) - db.exec('CREATE INDEX IF NOT EXISTS idx_thinking_owner_created ON thinking_signature_owners(created_at)') this.sql = { - owners: db.prepare( - 'SELECT key, account_id FROM thinking_signature_owners WHERE key IN (SELECT value FROM json_each(?))', - ), + owners: db.prepare('SELECT key, account_id FROM thinking_signature_owners WHERE key = ANY($1::text[])'), refresh: db.prepare( - 'UPDATE thinking_signature_owners SET created_at = ? WHERE key IN (SELECT value FROM json_each(?)) AND created_at < ?', + 'UPDATE thinking_signature_owners SET created_at = $1 WHERE key = ANY($2::text[]) AND created_at < $3', ), insert: db.prepare( - 'INSERT OR IGNORE INTO thinking_signature_owners (key, account_id, created_at) VALUES (?, ?, ?)', + 'INSERT INTO thinking_signature_owners (key, account_id, created_at) VALUES ($1, $2, $3) ON CONFLICT(key) DO NOTHING', ), - purge: db.prepare('DELETE FROM thinking_signature_owners WHERE created_at < ?'), + purge: db.prepare('DELETE FROM thinking_signature_owners WHERE created_at < $1'), } this.boundDb = db return this.sql @@ -74,26 +65,26 @@ export class ThinkingOwnerStore { * key → account for the keys that have a record; null when no database is * reachable, so the caller can leave history alone instead of guessing. */ - ownersOf(keys, now = Date.now()) { + async ownersOf(keys, now = Date.now()) { const sql = this.statements() if (!sql) return null const found = new Map() if (!keys.length) return found - const list = JSON.stringify(keys) - for (const row of sql.owners.all(list)) found.set(row.key, row.account_id) - if (found.size) sql.refresh.run(now, list, now - REFRESH_AFTER_MS) + const list = keys + for (const row of await sql.owners.all(list)) found.set(row.key, row.account_id) + if (found.size) await sql.refresh.run(now, list, now - REFRESH_AFTER_MS) return found } /** First writer wins: a signature belongs to the account that returned it. */ - record(keys, accountId, now = Date.now()) { + async record(keys, accountId, now = Date.now()) { const account = String(accountId || '') const sql = keys.length && account ? this.statements() : null if (!sql) return - for (const key of keys) sql.insert.run(key, account, now) + for (const key of keys) await sql.insert.run(key, account, now) if (now - this.lastPurge >= PURGE_EVERY_MS) { this.lastPurge = now - sql.purge.run(now - this.ttlMs) + await sql.purge.run(now - this.ttlMs) } } } @@ -118,14 +109,14 @@ function historyKeys(body) { * database the body goes unchanged. * @returns {{ body: object, kept: number, converted: number, dropped: number, unknown: number }} */ -export function filterThinkingForAccount(body, accountId, store = thinkingOwners) { +export async function filterThinkingForAccount(body, accountId, store = thinkingOwners) { const account = String(accountId || '') const counts = { kept: 0, converted: 0, dropped: 0, unknown: 0 } const keys = account ? historyKeys(body) : [] if (!keys.length) return { body, ...counts } let owners = null try { - owners = store.ownersOf([...new Set(keys)]) + owners = await store.ownersOf([...new Set(keys)]) } catch {} if (!owners) return { body, ...counts, kept: keys.length } const out = mapThinkingBlocks(body, (block, message) => { @@ -150,14 +141,14 @@ export function filterThinkingForAccount(body, accountId, store = thinkingOwners } /** Bind the thinking blocks of a served response to the account that produced them. */ -export function recordThinkingOwners(message, accountId, store = thinkingOwners) { +export async function recordThinkingOwners(message, accountId, store = thinkingOwners) { const keys = [] for (const block of Array.isArray(message?.content) ? message.content : []) { const key = thinkingSignatureKey(block) if (key) keys.push(key) } try { - store.record(keys, accountId) + await store.record(keys, accountId) } catch {} return keys.length } diff --git a/src/lib/pool/write-queue.mjs b/src/lib/pool/write-queue.mjs new file mode 100644 index 000000000..a48176ebe --- /dev/null +++ b/src/lib/pool/write-queue.mjs @@ -0,0 +1,31 @@ +/** Ordered persistence for synchronous scheduling state. Flush surfaces any failed write. */ +export class WriteQueue { + constructor(db = null) { + if (db) { + db.pendingWriteQueues ||= new Set() + db.pendingWriteQueues.add(this) + } + this.pending = Promise.resolve() + this.failure = null + } + + enqueue(write) { + const operation = this.pending.then(() => { + if (this.failure) throw this.failure + return write() + }) + this.pending = operation.catch((error) => { + this.failure ||= error + }) + return this.pending + } + + async flush() { + let pending + do { + pending = this.pending + await pending + } while (pending !== this.pending) + if (this.failure) throw this.failure + } +} diff --git a/src/lib/protocol/gpt-model-policy.mjs b/src/lib/protocol/gpt-model-policy.mjs index 7e2c7450e..ac25de0c1 100644 --- a/src/lib/protocol/gpt-model-policy.mjs +++ b/src/lib/protocol/gpt-model-policy.mjs @@ -2,7 +2,7 @@ * GPT catalog. Separate settings key from Claude model_policy. * Live ids from ChatGPT sync; seed is empty-catalog fallback. */ -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { isGptSeriesId, isSyncableGptCatalogId } from './gpt-ids.mjs' const SETTINGS_KEY = 'gpt_model_policy' @@ -56,15 +56,15 @@ export function seedGptPolicy() { let policy = seedGptPolicy() let loaded = false -function persist() { +async function persist() { try { - new SettingsRepo().set(SETTINGS_KEY, policy) + await new SettingsRepo().set(SETTINGS_KEY, policy) } catch (e) { console.warn('[gpt-model-policy] persist failed', e?.message || e) } } -export function loadGptModelPolicy({ force = false } = {}) { +export async function loadGptModelPolicy({ force = false } = {}) { if (loaded && !force) return policy try { const stored = new SettingsRepo().get(SETTINGS_KEY, null) @@ -77,7 +77,7 @@ export function loadGptModelPolicy({ force = false } = {}) { policy.source = policy.source || 'settings' } else { policy = seedGptPolicy() - persist() + await persist() } } catch { policy = seedGptPolicy() @@ -87,11 +87,10 @@ export function loadGptModelPolicy({ force = false } = {}) { } export function getGptModelPolicy() { - if (!loaded) loadGptModelPolicy() return policy } -export function saveGptModelPolicy(next) { +export async function saveGptModelPolicy(next) { const seed = seedGptPolicy() const models = { ...(seed.models || {}) } if (next?.models && typeof next.models === 'object') { @@ -108,11 +107,10 @@ export function saveGptModelPolicy(next) { updated_at: new Date().toISOString(), source: 'panel', } - persist() + await persist() return policy } export function listGptPolicyModels() { - if (!loaded) loadGptModelPolicy() return Object.entries(policy.models || {}) .filter(([id, cfg]) => isGptSeriesId(id) && cfg?.enabled !== false) .map(([id, cfg]) => ({ id, ...cfg, family: 'codex' })) @@ -120,7 +118,6 @@ export function listGptPolicyModels() { } export function getGptPolicyCatalogIds({ enabledOnly = false } = {}) { - if (!loaded) loadGptModelPolicy() return Object.entries(policy.models || {}) .filter(([id, cfg]) => isGptSeriesId(id) && (!enabledOnly || cfg?.enabled !== false)) .map(([id]) => id) @@ -128,17 +125,16 @@ export function getGptPolicyCatalogIds({ enabledOnly = false } = {}) { export function isGptModelEnabled(id) { if (!isGptSeriesId(id)) return false - if (!loaded) loadGptModelPolicy() const row = policy.models?.[id] if (row) return row.enabled !== false if (policy.source === 'codex' && Object.keys(policy.models || {}).length) return false return true } -export function resetGptModelPolicy() { +export async function resetGptModelPolicy() { policy = seedGptPolicy() loaded = true - persist() + await persist() return policy } @@ -147,8 +143,7 @@ export function resetGptModelPolicy() { * a GPT id the policy has never seen starts disabled until the admin enables it * (GPT has no worker_only mode). */ -export function syncGptIdsIntoPolicy(ids = [], { displayNameOf = null } = {}) { - if (!loaded) loadGptModelPolicy() +export async function syncGptIdsIntoPolicy(ids = [], { displayNameOf = null } = {}) { const live = [] const seen = new Set() for (const item of ids || []) { @@ -182,7 +177,7 @@ export function syncGptIdsIntoPolicy(ids = [], { displayNameOf = null } = {}) { updated_at: new Date().toISOString(), source: 'codex', } - persist() + await persist() return { ok: true, synced: live.length, diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index 772964936..9b604d25c 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -204,7 +204,7 @@ function pinnedVmId(req) { * A request that continues a stored response (`previous_response_id`) can * only run where that response lives. */ -export function pickCodexCandidates( +export async function pickCodexCandidates( projectRoot, req, { stickyRouter = null, sessions = null, body = null, excluded = null, routing = {} } = {}, @@ -212,7 +212,7 @@ export function pickCodexCandidates( const pin = pinnedVmId(req) const model = body?.model || null const routingPolicy = normalizeCodexRouting(routing.codex || routing).quota - const scope = keyScopeFromRequest(req) + const scope = await keyScopeFromRequest(req) const poolDenial = vmPoolDenial(scope) if (poolDenial) return { error: 'vm_pool_unavailable', message: poolDenial.message, ids: [], candidates: [] } if (scope.group_type === 'anthropic') return { error: 'key_group_mismatch', ids: [], candidates: [] } @@ -402,7 +402,7 @@ function clientGoneSignal(req, res) { async function admitCodexCandidate(projectRoot, req, opts, { deadline, signal }) { let woken = false for (;;) { - const picked = pickCodexCandidates(projectRoot, req, opts) + const picked = await pickCodexCandidates(projectRoot, req, opts) const livePolicy = normalizeCodexRouting(opts.routing?.codex || opts.routing).quota if (picked.error && picked.error !== 'capacity_unavailable') return { picked } for (const candidate of picked.candidates || []) { @@ -739,8 +739,8 @@ export async function handleCodexProtocol({ if (captureOutbound) logBag.outbound_body = outboundBody logBag.outbound_headers = redactHeaders(outboundHeaders) let hopEmitted = false - const runHop = () => - runCodexKernelHop({ + const runHop = async () => + await runCodexKernelHop({ hop, args: { exec: execFor(projectRoot, vm), @@ -1034,7 +1034,7 @@ function searchCredentials(projectRoot, vm) { * refresh token on its own, so first adopt a token it wrote meanwhile; otherwise refresh * once per slot, shared by concurrent callers. */ -function refreshSearchAccess(projectRoot, vm, usedAccess, { proxyUrl, fetchImpl, refresh }) { +async function refreshSearchAccess(projectRoot, vm, usedAccess, { proxyUrl, fetchImpl, refresh }) { const now = searchCredentials(projectRoot, vm) if (now.access && now.access !== usedAccess) return now.access if (!now.refresh) return '' diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 05da003e3..04978346e 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -1,3 +1,4 @@ +import { trackPendingWrite } from '../db/pending-writes.mjs' /** * /v1 protocol handler. URLs, auth, and response envelopes stay with the * server wiring; this factory owns convert → pool → Go/Rust hop → client. @@ -16,7 +17,7 @@ import { } from '../core/refusal-guard.mjs' import { RefusalGuardsRepo } from '../db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../db/repos/refusal-device-blocks-repo.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { findAccount } from '../admin/panel-api.mjs' import { jevScopeWaits, readJevConfig, scopeAccountFromVm } from './jev-intercept.mjs' import { runProtocolIntercept } from './intercept-gate.mjs' @@ -171,7 +172,7 @@ export function createHandleProtocol(deps) { const json = (...args) => deps.json(...args) const writeSSEHeaders = (...args) => deps.writeSSEHeaders(...args) const readBody = (...args) => deps.readBody(...args) - const requireAuth = (...args) => deps.requireAuth(...args) + const requireAuth = async (...args) => await deps.requireAuth(...args) const cfg = deps.cfg const requestLog = deps.requestLog const stickyRouter = deps.stickyRouter @@ -198,13 +199,13 @@ export function createHandleProtocol(deps) { return disjointOpenaiUsage(next) } - function attachPricing(logBag, { managedKey = null, rateMultiplier = 1, bill = false } = {}) { + async function attachPricing(logBag, { managedKey = null, rateMultiplier = 1, bill = false } = {}) { const model = logBag.upstream_model || logBag.model const priced = costColumnsFromUsage(logBag.usage || {}, model) Object.assign(logBag, priced) if (!bill || !managedKey) return try { - apiKeyStore.recordUsage(managedKey, logBag.usage || {}, { + await apiKeyStore.recordUsage(managedKey, logBag.usage || {}, { model, rateMultiplier, officialCost: priced.total_cost, @@ -370,10 +371,10 @@ export function createHandleProtocol(deps) { } } - function banRefusalDevice(deviceId, { requestId, fingerprint, reason }) { + async function banRefusalDevice(deviceId, { requestId, fingerprint, reason }) { if (!deviceId) return try { - deviceRepo()?.block?.({ deviceId, requestId, fingerprint, reason }) + await deviceRepo()?.block?.({ deviceId, requestId, fingerprint, reason }) } catch { /* the prompt block still stands */ } @@ -388,7 +389,7 @@ export function createHandleProtocol(deps) { } } - function rememberRefusal({ inbound, body, headers, result, logBag, requestId }) { + async function rememberRefusal({ inbound, body, headers, result, logBag, requestId }) { if (!refusalEnabled()) return const contentRefusal = isUpstreamRefusal(result, logBag) const timed = result?.policy?.rememberRefusal === true @@ -398,7 +399,7 @@ export function createHandleProtocol(deps) { const fingerprint = refusalFingerprint(body, inbound) try { const ttl = Number(result?.policy?.refusalTtlMs) || 0 - repo.remember({ + await repo.remember({ fingerprint, model: body?.model || inbound?.model || '', requestId, @@ -411,7 +412,7 @@ export function createHandleProtocol(deps) { return } if (!contentRefusal || !refusalPolicy().device_block_enabled) return - banRefusalDevice(inboundRefusalDeviceId({ inbound, body, headers }), { + await banRefusalDevice(inboundRefusalDeviceId({ inbound, body, headers }), { requestId, fingerprint, reason: 'refusal_guard', @@ -451,7 +452,7 @@ export function createHandleProtocol(deps) { requestContext, slotWaitMs: candidate.slotWaitMs, noGoFallback, - ensureCredential: (exec) => ensureWorkerCredential(exec), + ensureCredential: async (exec) => await ensureWorkerCredential(exec), onEvent: async (line) => { if (/response_headers/.test(String(line))) return applyClaudeSSELineToMessage(restoreToolNamesInSSELine(line, toolNames), assembler) @@ -500,21 +501,27 @@ export function createHandleProtocol(deps) { reasoning_effort: null, outbound_session_id: null, } - res.on('finish', () => { - try { - const groupId = req.apiKeyRecord?.group_id ?? 1 - requestLog.finish(logCtx, { - status: res.statusCode || 0, - api_key_kind: req.apiKeyKind || null, - api_key_id: req.apiKeyRecord?.id || null, - user_id: req.apiKeyRecord?.user_id ?? null, - group_id: groupId, - rate_multiplier: deps.groupsRepo.rateMultiplier(groupId), - ...logBag, - }) - } catch {} - }) - if (!requireAuth(req, res)) { + res.on('finish', () => + trackPendingWrite( + (async () => { + try { + const groupId = req.apiKeyRecord?.group_id ?? 1 + await requestLog.finish(logCtx, { + status: res.statusCode || 0, + api_key_kind: req.apiKeyKind || null, + api_key_id: req.apiKeyRecord?.id || null, + user_id: req.apiKeyRecord?.user_id ?? null, + group_id: groupId, + rate_multiplier: await deps.groupsRepo.rateMultiplier(groupId), + ...logBag, + }) + } catch (error) { + throw error + } + })(), + ), + ) + if (!(await requireAuth(req, res))) { logBag.error_code = req.authError?.code || ErrorCode.INVALID_API_KEY logBag.error_message = req.authError?.message || 'Invalid credentials' logBag.api_key_presented = presentedApiKeyForLog(req.presentedApiKey) @@ -676,7 +683,7 @@ export function createHandleProtocol(deps) { logBag.error_message = errorResult.body?.error?.message || null return json(res, errorResult.status, errorResult.body) } - const keyScope = keyScopeFromRequest(req) + const keyScope = await keyScopeFromRequest(req) const poolDenial = vmPoolDenial(keyScope) if (poolDenial) { stats.errors++ @@ -740,8 +747,8 @@ export function createHandleProtocol(deps) { } // Account scope needs the slot, so the Codex hop runs the same gate again // once a candidate is claimed. Without a scope the pre-hop verdict stands. - const accountGate = (account) => - applyProtocolIntercept({ + const accountGate = async (account) => + await applyProtocolIntercept({ req, inbound, body: ctx.body, @@ -753,7 +760,7 @@ export function createHandleProtocol(deps) { }) if (protocol === 'anthropic.messages') { const codex = normalizeCodexRouting(routing.codex) - return handleCodexProtocol({ + return await handleCodexProtocol({ req, res, protocol, @@ -781,7 +788,7 @@ export function createHandleProtocol(deps) { ...codexSticky, }) } - return handleCodexProtocol({ + return await handleCodexProtocol({ req, res, protocol, @@ -1023,7 +1030,7 @@ export function createHandleProtocol(deps) { if (inferenceBackend === 'api') { const managedKey = req.apiKeyRecord || null if (managedKey) { - const gate = apiKeyStore.acquire(managedKey) + const gate = await apiKeyStore.acquire(managedKey) if (!gate.ok) { stats.errors++ return json( @@ -1099,9 +1106,9 @@ export function createHandleProtocol(deps) { ? Math.max(0, result.firstTokenAt - logCtx.t0) : (result?.ttftMs ?? null) logBag.final_state = result?.terminalState || null - attachPricing(logBag, { + await attachPricing(logBag, { managedKey, - rateMultiplier: deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), + rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), bill: !!(result?.ok && managedKey), }) if (clientStream) { @@ -1140,7 +1147,7 @@ export function createHandleProtocol(deps) { const streamIdleTimeoutMs = resolveStreamIdleTimeoutMs(getRouting(), cfg) const managedKey = req.apiKeyRecord || null if (managedKey) { - const gate = apiKeyStore.acquire(managedKey) + const gate = await apiKeyStore.acquire(managedKey) if (!gate.ok) { stats.errors++ return json( @@ -1193,7 +1200,7 @@ export function createHandleProtocol(deps) { const pinVmRaw = String(req.headers['x-kin-vm'] || '').trim() const pinVmId = req.apiKeyKind === 'master' && isValidVmId(pinVmRaw) ? pinVmRaw : null // Pin is panel test-chat / diagnostics (manage). Unpinned /v1 is dispatch. - const ownerScope = pinVmId ? { type: 'any' } : ownerScopeFromRequest(req, apiKeyStore?.users) + const ownerScope = pinVmId ? { type: 'any' } : await ownerScopeFromRequest(req, apiKeyStore?.users) // Cache bypass is a client hint; only authenticated master-key probes may skip accounting. const healthReal = req.apiKeyKind === 'master' && isHealthRealBypass(req.headers) // Cache lives per account; a failover to another account starts cold by design. @@ -1235,7 +1242,7 @@ export function createHandleProtocol(deps) { applyAttempt: async (body, selected, extra = {}) => { // Only this account's own thinking replays as thinking. Another // account's signature, or one with no record, goes out as text. - const owned = filterThinkingForAccount(body, selected.accountId, thinkingOwnerStore) + const owned = await filterThinkingForAccount(body, selected.accountId, thinkingOwnerStore) body = owned.body if (owned.kept || owned.converted || owned.dropped) { const { kept, converted, dropped, unknown } = owned @@ -1455,7 +1462,7 @@ export function createHandleProtocol(deps) { }, callAttempt: async ({ candidate, body, attemptMeta, deliveryMode: attemptDelivery, signal, onCommit }) => { if (!clientStream) { - return streamAndAssembleClaudeMessage({ + return await streamAndAssembleClaudeMessage({ candidate, body, reqHeaders: req.headers, @@ -1516,7 +1523,7 @@ export function createHandleProtocol(deps) { routing: getRouting(), slotWaitMs: candidate.slotWaitMs, noGoFallback: !!pinVmId, - ensureCredential: (exec) => ensureWorkerCredential(exec), + ensureCredential: async (exec) => await ensureWorkerCredential(exec), onCommit: () => { if (protocol === 'anthropic.messages' && !res.headersSent) writeSSEHeaders(res) onCommit() @@ -1580,7 +1587,7 @@ export function createHandleProtocol(deps) { logBag.final_account_id = result?.accountId || null if (result?.threadBetas) logBag.hop_meta = { ...logBag.hop_meta, thread_betas: result.threadBetas } if (result?.accountId && (result.ok || result.committed)) - recordThinkingOwners(result.body, result.accountId, thinkingOwnerStore) + await recordThinkingOwners(result.body, result.accountId, thinkingOwnerStore) // Zero executions is a real 0, not unknown; the VM is the last one that ran. logBag.attempt_count = result?.attemptCount ?? 0 logBag.final_state = result?.finalState || result?.terminalState || null @@ -1600,7 +1607,14 @@ export function createHandleProtocol(deps) { logBag.error_code = logBag.error_code || 'content_filter_refusal' logBag.error_message = logBag.error_message || 'upstream stop_reason=refusal' } - rememberRefusal({ inbound, body: ctx.body, headers: req.headers, result, logBag, requestId: logCtx.request_id }) + await rememberRefusal({ + inbound, + body: ctx.body, + headers: req.headers, + result, + logBag, + requestId: logCtx.request_id, + }) if (result?.accountId) { try { @@ -1633,9 +1647,9 @@ export function createHandleProtocol(deps) { if (vm) pool.syncQuotaSchedule(vm) } } catch {} - attachPricing(logBag, { + await attachPricing(logBag, { managedKey, - rateMultiplier: deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), + rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), bill: !!managedKey, }) } @@ -1668,7 +1682,14 @@ export function createHandleProtocol(deps) { res.write(buildAnthropicSseError(mapped.body?.error || {})) } } - rememberRefusal({ inbound, body: ctx.body, headers: req.headers, result, logBag, requestId: logCtx.request_id }) + await rememberRefusal({ + inbound, + body: ctx.body, + headers: req.headers, + result, + logBag, + requestId: logCtx.request_id, + }) return res.end() } @@ -1724,21 +1745,27 @@ export function createHandleProtocol(deps) { attempt_count: null, final_state: null, } - res.on('finish', () => { - try { - const groupId = req.apiKeyRecord?.group_id ?? 1 - requestLog.finish(logCtx, { - status: res.statusCode || 0, - api_key_kind: req.apiKeyKind || null, - api_key_id: req.apiKeyRecord?.id || null, - user_id: req.apiKeyRecord?.user_id ?? null, - group_id: groupId, - rate_multiplier: deps.groupsRepo.rateMultiplier(groupId), - ...logBag, - }) - } catch {} - }) - if (!requireAuth(req, res)) { + res.on('finish', () => + trackPendingWrite( + (async () => { + try { + const groupId = req.apiKeyRecord?.group_id ?? 1 + await requestLog.finish(logCtx, { + status: res.statusCode || 0, + api_key_kind: req.apiKeyKind || null, + api_key_id: req.apiKeyRecord?.id || null, + user_id: req.apiKeyRecord?.user_id ?? null, + group_id: groupId, + rate_multiplier: await deps.groupsRepo.rateMultiplier(groupId), + ...logBag, + }) + } catch (error) { + throw error + } + })(), + ), + ) + if (!(await requireAuth(req, res))) { logBag.error_code = req.authError?.code || ErrorCode.INVALID_API_KEY logBag.error_message = req.authError?.message || 'Invalid credentials' logBag.api_key_presented = presentedApiKeyForLog(req.presentedApiKey) @@ -1765,7 +1792,7 @@ export function createHandleProtocol(deps) { } logBag.model = typeof body?.model === 'string' ? body.model : null logBag.inbound_summary = summarizeBody(body) - return handleCodexSearch({ + return await handleCodexSearch({ req, res, body, diff --git a/src/lib/protocol/intercept-gate.mjs b/src/lib/protocol/intercept-gate.mjs index f4eb7e304..b99fa023b 100644 --- a/src/lib/protocol/intercept-gate.mjs +++ b/src/lib/protocol/intercept-gate.mjs @@ -62,11 +62,11 @@ function similarityRatio(value) { return n > 1 ? n / 100 : n } -export function commitProtocolBlock(decision, { repo, devices, policy, requestId } = {}) { +export async function commitProtocolBlock(decision, { repo, devices, policy, requestId } = {}) { if (!decision || decision.action !== 'block') return if (decision.kind === 'device') { try { - devices?.hit?.(decision.deviceId) + await devices?.hit?.(decision.deviceId) } catch { /* counter is best-effort */ } @@ -74,14 +74,14 @@ export function commitProtocolBlock(decision, { repo, devices, policy, requestId } if ((decision.kind === 'exact' || decision.kind === 'similar') && decision.fingerprint) { try { - repo?.hit?.(decision.fingerprint) + await repo?.hit?.(decision.fingerprint) } catch { /* counter is best-effort */ } } if (decision.remember && policy?.enabled && repo && decision.fingerprint) { try { - repo.remember?.({ + await repo.remember?.({ fingerprint: decision.fingerprint, model: decision.model || '', requestId: requestId || null, @@ -96,7 +96,7 @@ export function commitProtocolBlock(decision, { repo, devices, policy, requestId } if (!decision.banDevice || !policy?.enabled || !policy?.device_block_enabled || !decision.deviceId) return try { - deviceStore(devices)?.block?.({ + await deviceStore(devices)?.block?.({ deviceId: decision.deviceId, requestId: requestId || null, fingerprint: decision.fingerprint || null, @@ -233,7 +233,7 @@ export async function evaluateProtocolIntercept({ } if (policy?.enabled && repo && typeof repo.get === 'function') { - let match = matchStoredRefusal({ + let match = await matchStoredRefusal({ inbound, body, headers, @@ -317,6 +317,6 @@ export async function evaluateProtocolIntercept({ export async function runProtocolIntercept(ctx = {}) { const decision = await evaluateProtocolIntercept(ctx) if (decision.action !== 'block') return decision - commitProtocolBlock(decision, ctx) + await commitProtocolBlock(decision, ctx) return decision } diff --git a/src/lib/protocol/jev-intercept.mjs b/src/lib/protocol/jev-intercept.mjs index fa6365dde..1a014666f 100644 --- a/src/lib/protocol/jev-intercept.mjs +++ b/src/lib/protocol/jev-intercept.mjs @@ -921,10 +921,10 @@ export function validateJevPatch(body = {}) { } /** Apply a panel PUT. Omitted keys stay. An empty api_key, or api_keys: [], clears them. */ -export function applyJevPatch(settings, body = {}) { +export async function applyJevPatch(settings, body = {}) { const problems = validateJevPatch(body) if (problems.length) return { ok: false, problems } - const current = normalizeJevConfig(settings.get(JEV_SETTING, null)) + const current = normalizeJevConfig(await settings.get(JEV_SETTING, null)) const next = normalizeJevConfig({ ...current, ...body, @@ -938,7 +938,7 @@ export function applyJevPatch(settings, body = {}) { patterns: Object.hasOwn(body, 'patterns') ? body.patterns : current.patterns, rules: Object.hasOwn(body, 'rules') ? body.rules : current.rules, }) - settings.set(JEV_SETTING, next) + await settings.set(JEV_SETTING, next) if (Object.hasOwn(body, 'api_keys') || Object.hasOwn(body, 'api_key')) rearmJevKeys(next) return { ok: true, config: publicJevConfig(next) } } diff --git a/src/lib/protocol/model-policy.mjs b/src/lib/protocol/model-policy.mjs index ca0287809..72f8c5cb4 100644 --- a/src/lib/protocol/model-policy.mjs +++ b/src/lib/protocol/model-policy.mjs @@ -8,7 +8,7 @@ * worker_only — ignore policy enabled flags for listing */ -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { CONTEXT_1M_BETA, DEFAULT_CONTEXT_1M_WHITELIST, @@ -537,18 +537,18 @@ function heuristicCapabilities(modelId = '') { return { ...CAP_46 } } -export function loadModelPolicy({ force = false } = {}) { +export async function loadModelPolicy({ force = false } = {}) { if (loaded && !force) return policy try { const repo = new SettingsRepo() const stored = repo.get(SETTINGS_KEY, null) if (stored) { - const persisted = persistHaiku55Model(repo, persistOpus55Model(repo, stored)) + const persisted = await persistHaiku55Model(repo, await persistOpus55Model(repo, stored)) policy = normalizePolicy(persisted) policy.source = policy.source || 'settings' } else { policy = seedDefaultPolicy() - repo.set(SETTINGS_KEY, policy) + await repo.set(SETTINGS_KEY, policy) } } catch { policy = seedDefaultPolicy() @@ -558,7 +558,7 @@ export function loadModelPolicy({ force = false } = {}) { } /** Append Opus 5.5 onto an existing settings row. Does not reset other models. */ -function persistOpus55Model(repo, stored) { +async function persistOpus55Model(repo, stored) { if (!stored || typeof stored !== 'object') return stored const seed = seedDefaultPolicy() const models = stored.models && typeof stored.models === 'object' ? { ...stored.models } : {} @@ -580,7 +580,7 @@ function persistOpus55Model(repo, stored) { if (!dirty) return stored const next = { ...stored, models, aliases } try { - repo.set(SETTINGS_KEY, next) + await repo.set(SETTINGS_KEY, next) } catch { return next } @@ -588,7 +588,7 @@ function persistOpus55Model(repo, stored) { } /** Append or migrate Haiku 5.5 onto an existing settings row. Does not reset other models. */ -function persistHaiku55Model(repo, stored) { +async function persistHaiku55Model(repo, stored) { if (!stored || typeof stored !== 'object') return stored const seed = seedDefaultPolicy() const desired = seed.models[HAIKU_55_ID] @@ -617,7 +617,7 @@ function persistHaiku55Model(repo, stored) { if (!dirty) return stored const next = { ...stored, models } try { - repo.set(SETTINGS_KEY, next) + await repo.set(SETTINGS_KEY, next) } catch { return next } @@ -625,17 +625,16 @@ function persistHaiku55Model(repo, stored) { } export function getModelPolicy() { - if (!loaded) loadModelPolicy() return policy } -export function saveModelPolicy(next) { +export async function saveModelPolicy(next) { policy = normalizePolicy(next) policy.updated_at = new Date().toISOString() policy.source = 'panel' try { const repo = new SettingsRepo() - repo.set(SETTINGS_KEY, policy) + await repo.set(SETTINGS_KEY, policy) } catch (e) { console.warn('[model-policy] persist failed', e?.message || e) } @@ -643,18 +642,17 @@ export function saveModelPolicy(next) { return policy } -export function resetModelPolicy() { +export async function resetModelPolicy() { policy = seedDefaultPolicy() try { const repo = new SettingsRepo() - repo.set(SETTINGS_KEY, policy) + await repo.set(SETTINGS_KEY, policy) } catch {} loaded = true return policy } export function resolvePolicyModelId(raw = '') { - if (!loaded) loadModelPolicy() const m = String(raw || '').trim() if (!m) return null const bare = m.split('/').filter(Boolean).pop() || m @@ -678,7 +676,6 @@ export function resolvePolicyModelId(raw = '') { } export function getModelEntry(modelId = '') { - if (!loaded) loadModelPolicy() const key = resolvePolicyModelId(modelId) if (key && policy.models[key]) return { id: key, ...policy.models[key] } const id = key || String(modelId || '') @@ -720,7 +717,6 @@ export function getBetaPolicy(modelId = '') { } export function getContext1mWhitelist() { - if (!loaded) loadModelPolicy() return normalizeContext1mWhitelist(policy.defaults?.context_1m_whitelist, DEFAULT_CONTEXT_1M_WHITELIST) } @@ -738,7 +734,6 @@ export function resolveContext1mPass( } export function shouldPassContext1mByPolicy(modelId = '') { - if (!loaded) loadModelPolicy() return resolveContext1mPass(modelId, { entry: getModelEntry(modelId), whitelist: getContext1mWhitelist(), @@ -775,7 +770,6 @@ function adaptiveOnlyThinking(thinking) { } export function normalizeThinkingByPolicy(body = {}) { - if (!loaded) loadModelPolicy() if (!body || typeof body !== 'object') return body if (policy.defaults.normalize_thinking === false) return body const thinking = body.thinking @@ -912,7 +906,6 @@ export function applyBetaPolicyToHeader( modelId = '', { isOfficial = false, whitelist, want1m = false } = {}, ) { - if (!loaded) loadModelPolicy() const pass = whitelist !== undefined ? resolveContext1mPass(modelId, { @@ -946,14 +939,12 @@ export function unofficialMimicryBetaHeader(modelId = '') { /** All policy model ids that look like Claude catalog ids (disabled included). */ export function getPolicyCatalogIds({ enabledOnly = false } = {}) { - if (!loaded) loadModelPolicy() return Object.entries(policy.models || {}) .filter(([id, cfg]) => isCatalogModelId(id) && (!enabledOnly || cfg.enabled !== false)) .map(([id]) => id) } export function filterPublicModelIds(workerIds = []) { - if (!loaded) loadModelPolicy() const mode = policy.catalog_mode || 'policy_only' const ids = [ ...new Set( @@ -972,7 +963,6 @@ export function filterPublicModelIds(workerIds = []) { } export function listPolicyModels() { - if (!loaded) loadModelPolicy() return Object.entries(policy.models) .map(([id, cfg]) => ({ id, ...cfg })) .sort((a, b) => (a.sort ?? 100) - (b.sort ?? 100) || a.id.localeCompare(b.id)) @@ -983,8 +973,7 @@ export function listPolicyModels() { * public only under worker_only; other modes add it disabled for the admin to enable. * `displayNameOf(id)` may supply a display name (models.dev) for new entries. */ -export function syncWorkerModelsIntoPolicy(workerIds = [], { displayNameOf = null } = {}) { - if (!loaded) loadModelPolicy() +export async function syncWorkerModelsIntoPolicy(workerIds = [], { displayNameOf = null } = {}) { const added = [] const enabled = policy.catalog_mode === 'worker_only' for (const workerId of workerIds || []) { @@ -1021,7 +1010,7 @@ export function syncWorkerModelsIntoPolicy(workerIds = [], { displayNameOf = nul policy.updated_at = new Date().toISOString() policy.source = 'worker_merge' try { - new SettingsRepo().set(SETTINGS_KEY, policy) + await new SettingsRepo().set(SETTINGS_KEY, policy) } catch {} } return { policy, added } @@ -1032,7 +1021,6 @@ export const FALLBACK_MAX_TOKENS = 4096 /** Official per-model default for a missing caller max_tokens. Never overwrites inbound. */ export function defaultMaxTokensForModel(modelId = '') { - if (!loaded) loadModelPolicy() const params = getModelParams(modelId) const n = Number(params.max_tokens_default) const cap = Number(params.max_tokens_cap) diff --git a/src/lib/protocol/models.mjs b/src/lib/protocol/models.mjs index 5dd98c189..cf1e8b7d2 100644 --- a/src/lib/protocol/models.mjs +++ b/src/lib/protocol/models.mjs @@ -10,7 +10,6 @@ import { getModelEntry, getModelPolicy, getPolicyCatalogIds, - loadModelPolicy, seedDefaultPolicy, } from './model-policy.mjs' import { hasClaudeCode1mSuffix, stripClaudeCode1mSuffix } from './context-1m.mjs' @@ -35,7 +34,6 @@ export function seedModelCatalog() { if (catalogIsTestPinned()) return cache let ids = [...SEED_MODEL_IDS] try { - loadModelPolicy() const fromPolicy = getPolicyCatalogIds() if (fromPolicy.length) ids = fromPolicy } catch {} @@ -65,9 +63,6 @@ function listGptPublicModels() { */ export function gatewayModelCatalog({ groupType = 'all' } = {}) { seedModelCatalog() - try { - loadModelPolicy() - } catch {} const gpt = groupType === 'anthropic' ? [] : listGptPublicModels() const claude = groupType === 'openai' ? [] : listOfficialModels() return { @@ -142,7 +137,6 @@ export function latestIdForUndatedAlias(raw, ids) { function resolvePolicyAliasToCatalog(raw, ids) { try { - loadModelPolicy() const policy = getModelPolicy() const lower = String(raw || '').toLowerCase() const candidates = [] @@ -234,9 +228,6 @@ export function clearModelsCache() { } export function listOfficialModels() { - try { - loadModelPolicy() - } catch {} const ids = filterPublicModelIds(cache.ids || []) return ids.map((id) => { let display = id diff --git a/src/lib/protocol/responses-ws.mjs b/src/lib/protocol/responses-ws.mjs index 24d046140..f0b44a2ce 100644 --- a/src/lib/protocol/responses-ws.mjs +++ b/src/lib/protocol/responses-ws.mjs @@ -637,7 +637,7 @@ export function createResponsesWsHandler({ } } - ws.on('message', (data, isBinary) => { + ws.on('message', async (data, isBinary) => { if (closed) return if (isBinary) { closed = true @@ -724,7 +724,7 @@ export function createResponsesWsHandler({ }) } - function handleUpgrade(req, socket, head) { + async function handleUpgrade(req, socket, head) { let pathname = '/v1/responses' try { pathname = new URL(req.url || '/', 'http://local').pathname @@ -735,7 +735,7 @@ export function createResponsesWsHandler({ const routing = typeof getRouting === 'function' ? getRouting() || {} : {} if (rejectNewWs(socket, routing)) return true const res = handshakeRes(socket) - if (typeof requireAuth === 'function' && !requireAuth(req, res)) return true + if (typeof requireAuth === 'function' && !(await requireAuth(req, res))) return true wss.handleUpgrade(req, socket, head, (ws) => attach(ws, req)) return true } diff --git a/src/lib/protocol/user-count-tokens.mjs b/src/lib/protocol/user-count-tokens.mjs index 51e25dad7..516d946f2 100644 --- a/src/lib/protocol/user-count-tokens.mjs +++ b/src/lib/protocol/user-count-tokens.mjs @@ -6,7 +6,7 @@ import { makeError, mapUpstreamError, rewritePoolErrorForClient, ErrorType, Erro import { refusalGuardPolicy } from '../core/refusal-guard.mjs' import { RefusalGuardsRepo } from '../db/repos/refusal-guards-repo.mjs' import { RefusalDeviceBlocksRepo } from '../db/repos/refusal-device-blocks-repo.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { findAccount } from '../admin/panel-api.mjs' import { readJevConfig, scopeAccountFromVm } from './jev-intercept.mjs' import { runProtocolIntercept } from './intercept-gate.mjs' @@ -115,7 +115,7 @@ export async function peekCurrentAccount({ if (!poolScheduler?.peekAccount) { return { ok: false, code: 'no_eligible_accounts' } } - const keyScope = keyScopeFromRequest(req) + const keyScope = await keyScopeFromRequest(req) const poolDenial = vmPoolDenial(keyScope) if (poolDenial) return { ok: false, code: ErrorCode.VM_POOL_UNAVAILABLE, message: poolDenial.message } // Same gate as Messages: a platform-scoped key must not hop to the other platform's VMs. @@ -126,7 +126,7 @@ export async function peekCurrentAccount({ model, stickyKey, signal, - ownerScope: ownerScopeFromRequest(req, usersRepo), + ownerScope: await ownerScopeFromRequest(req, usersRepo), keyScope, }) } @@ -245,7 +245,7 @@ export async function blockCountTokensBeforeHop(req, inbound, deps = {}) { export async function handleUserCountTokens(req, res, deps) { const json = (...args) => deps.json(...args) - if (!deps.requireAuth(req, res)) return + if (!(await deps.requireAuth(req, res))) return let inbound try { inbound = await deps.readBody(req, inboundBodyLimit(deps.getRouting?.(), deps.cfg.limits.max_body_bytes)) @@ -372,7 +372,7 @@ export async function resolveUsageWindows({ export async function handleUserUsage(req, res, deps) { const json = (...args) => deps.json(...args) - if (!deps.requireAuth(req, res)) return + if (!(await deps.requireAuth(req, res))) return const peeked = await peekCurrentAccount({ poolScheduler: typeof deps.getPoolScheduler === 'function' ? deps.getPoolScheduler() : deps.poolScheduler, stickyRouter: deps.stickyRouter, diff --git a/src/lib/transport/kernel-watchdog.mjs b/src/lib/transport/kernel-watchdog.mjs index d5d38a5be..6fe2888bc 100644 --- a/src/lib/transport/kernel-watchdog.mjs +++ b/src/lib/transport/kernel-watchdog.mjs @@ -332,6 +332,9 @@ export function createKernelWatchdog({ timer = setInterval(() => void tick(), config.interval_sec * 1000) timer.unref?.() }, + async flush() { + while (inTick) await new Promise((resolve) => setTimeout(resolve, 10)) + }, stop() { running = false clearInterval(timer) diff --git a/src/lib/vm/cli-node-guard.mjs b/src/lib/vm/cli-node-guard.mjs index 2dc1371e1..d747773fb 100644 --- a/src/lib/vm/cli-node-guard.mjs +++ b/src/lib/vm/cli-node-guard.mjs @@ -113,6 +113,9 @@ export function createCliNodeGuard({ timer = setInterval(() => void tick(), intervalMs) timer.unref?.() }, + async flush() { + while (inTick) await new Promise((resolve) => setTimeout(resolve, 10)) + }, stop() { if (timer) clearInterval(timer) timer = null diff --git a/src/lib/vm/egress.mjs b/src/lib/vm/egress.mjs index 9334c88a3..546b15a31 100644 --- a/src/lib/vm/egress.mjs +++ b/src/lib/vm/egress.mjs @@ -9,7 +9,7 @@ import { execFileSync, spawn } from 'node:child_process' import fs from 'node:fs' import path from 'node:path' import { getDb, isDbOpen } from '../db/database.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { socksProxyUrl } from './socks-address.mjs' import { assertProxyAllowed, proxyBlockedReason } from './proxy-policy.mjs' @@ -556,7 +556,7 @@ async function stopOwnedEgress(pid) { } catch (error) { if (error.code !== 'ESRCH') return { ok: false, error: `egress_stop_failed: ${error.code || error.message}` } } - return waitEgressExit(pid) + return await waitEgressExit(pid) } export function inspectEgressProcess(projectRoot, proxyId) { @@ -653,7 +653,7 @@ export async function waitEgressExit( try { process.kill(pid, 'SIGKILL') } catch (error) { - if (error.code === 'ESRCH') return waitEgressReap(pid, killGraceMs) + if (error.code === 'ESRCH') return await waitEgressReap(pid, killGraceMs) return { ok: false, error: `egress_kill_failed: ${error.code || error.message}`, pid } } const reaped = await waitEgressReap(pid, killGraceMs) @@ -783,7 +783,11 @@ export async function runExclusiveProxyEgress(projectRoot, proxyId, fn) { export async function ensureProxyEgress(projectRoot, proxy, opts = {}) { if (isLocalEgressProxy(proxy)) return ensureLocalProxyEgress(proxy, { runDocker: opts.runDocker }) - return runExclusiveProxyEgress(projectRoot, proxy?.id, () => startProxyEgress(projectRoot, proxy, opts)) + return await runExclusiveProxyEgress( + projectRoot, + proxy?.id, + async () => await startProxyEgress(projectRoot, proxy, opts), + ) } export async function startProxyEgress( diff --git a/src/lib/vm/proxy-policy.mjs b/src/lib/vm/proxy-policy.mjs index 850b690e6..7a6aaba98 100644 --- a/src/lib/vm/proxy-policy.mjs +++ b/src/lib/vm/proxy-policy.mjs @@ -1,5 +1,5 @@ import { getDb, isDbOpen } from '../db/database.mjs' -import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { socksProxyFamily } from './socks-address.mjs' export function configuredIpv6Enabled() { diff --git a/src/lib/vm/proxy-pool.mjs b/src/lib/vm/proxy-pool.mjs index 003d3f076..a3c465113 100644 --- a/src/lib/vm/proxy-pool.mjs +++ b/src/lib/vm/proxy-pool.mjs @@ -14,6 +14,7 @@ import net from 'node:net' import crypto from 'node:crypto' import { resolveStoreDb } from '../db/database.mjs' import { ProxiesRepo } from '../db/repos/proxies-repo.mjs' +import { WriteQueue } from '../pool/write-queue.mjs' import { canBindProxyToVm, normalizeOwnerId, proxyOwnerId, vmOwnerId } from '../admin/resource-owner.mjs' import { validTimezone } from '../core/timezone.mjs' import { lookupProxyGeo, lookupProxyGeoV6 } from './proxy-geo.mjs' @@ -341,43 +342,53 @@ export class ProxyPool extends EventEmitter { this._probeSockets = new Map() // Node exits are not pool rows: px-local's geo is the control plane's, a node slot leaves elsewhere. this._exitGeo = new Map() - this.load() + this.writes = new WriteQueue(this.db) + this.db.pendingWriteQueues.add(this) } /** Re-read working set from DB (startup + post-restore). */ - load() { - try { - this.state.config = { ...DEFAULT_CONFIG, ...(this.repo.getConfig({}) || {}) } - this.state.proxies = this.repo.loadAll().map(hydrateProxy) - } catch { - this.state = { config: { ...DEFAULT_CONFIG }, proxies: [] } - } + async load() { + for (const queue of this.db.pendingWriteQueues || []) await queue.flush() + await this.flush() + this.state.config = { ...DEFAULT_CONFIG, ...((await this.repo.getConfig({})) || {}) } + this.state.proxies = (await this.repo.loadAll()).map(hydrateProxy) + return this + } + + async init() { + return await this.load() } - reload() { - this.load() + async reload() { + return await this.load() } - /** Re-bind to a fresh DB connection (after backup restore) and re-read. */ - rebind(db) { + async rebind(db) { + await this.flush() this.db = db this.repo = new ProxiesRepo(db) - this.load() + return await this.load() + } + + async flush() { + while (this._probing) await new Promise((resolve) => setTimeout(resolve, 10)) + await this.writes.flush() } save() { - // Batched rewrites (local-row migration) persist once at the end. if (this._saveDeferred) { this._savePending = true return } - this.repo.setConfig(this.state.config) - this.repo.replaceAll( - this.state.proxies.map((p) => ({ - ...p, - bound_vm_id: encodeBoundVmIds(boundVmIdsOf(p)), - })), - ) + const config = structuredClone(this.state.config) + const proxies = this.state.proxies.map((p) => ({ + ...structuredClone(p), + bound_vm_id: encodeBoundVmIds(boundVmIdsOf(p)), + })) + this.writes.enqueue(async () => { + await this.repo.setConfig(config) + await this.repo.replaceAll(proxies) + }) this.emit('change') } @@ -1382,7 +1393,7 @@ export class ProxyPool extends EventEmitter { const min = this.state.config.probe_interval_min || 10 const ms = min * 60 * 1000 void this.probeAll({ onlyEnabled: true }).catch(() => {}) - this._timer = setInterval(() => { + this._timer = setInterval(async () => { this.probeAll({ onlyEnabled: true }).catch(() => {}) }, ms) this._timer.unref?.() diff --git a/src/lib/vm/vm-db-sync.mjs b/src/lib/vm/vm-db-sync.mjs index 694a3e2a1..484715d7c 100644 --- a/src/lib/vm/vm-db-sync.mjs +++ b/src/lib/vm/vm-db-sync.mjs @@ -1,7 +1,7 @@ /** - * VM/credential DB mirror — write-through + reconcile + fs.watch. + * VM metadata DB mirror — write-through + reconcile + fs.watch. * - * Policy (user requirement: credentials must live in the database): + * Policy (VmsRepo omits active credentials and retains their presence flags): * - vms/*.json stay the runtime single-writer surface (persistOauthToVm, * CLI seed, execution context all read files — no behavior drift). * - Every vm.json write is mirrored into the `vms` table: @@ -21,6 +21,7 @@ import fs from 'node:fs' import path from 'node:path' import { isDbOpen, getDb } from '../db/database.mjs' import { VmsRepo } from '../db/repos/vms-repo.mjs' +import { WriteQueue } from '../pool/write-queue.mjs' import { setVmWriteHook, atomicWriteJson, isVmRecordFile } from './vm-file.mjs' import { reclaimAssignedVms, repairSlotSmbios, repairVmProxyCliEnabled } from './vm-registry.mjs' @@ -28,6 +29,9 @@ let _repo = null let _watcher = null let _watchTimer = null let _projectRoot = null +let _writes = new WriteQueue() +let _paused = false +const _deleted = new Set() function repo() { if (!isDbOpen()) return null @@ -42,103 +46,148 @@ export function isVmJsonPath(filePath) { return isVmRecordFile(base) || base === 'active.json' } -/** Mirror one vms/*.json file into the DB (safe no-op without DB). */ +/** Mirror a captured file snapshot in write order. Flush reports persistence errors. */ export function syncVmFile(filePath) { const r = repo() - if (!r) return false + if (!r || _paused) return Promise.resolve(false) + let value + let mtimeMs try { - const base = path.basename(filePath) - if (base === 'active.json') { - const a = JSON.parse(fs.readFileSync(filePath, 'utf8')) - if (a?.active_vm) r.setActiveVmId(a.active_vm) - return true - } - const vm = JSON.parse(fs.readFileSync(filePath, 'utf8')) - if (!vm?.id) return false - const st = fs.statSync(filePath) - r.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) - return true - } catch { - return false + value = JSON.parse(fs.readFileSync(filePath, 'utf8')) + mtimeMs = fs.statSync(filePath).mtimeMs + } catch (error) { + if (error.code === 'ENOENT') return Promise.resolve(false) + _writes.enqueue(() => { + throw error + }) + return _writes.flush().then(() => false) } + const active = path.basename(filePath) === 'active.json' + if (!active && value?.id) _deleted.delete(value.id) + if (active ? !value?.active_vm : !value?.id) return Promise.resolve(false) + _writes.enqueue(() => (active ? r.setActiveVmId(value.active_vm) : r.upsertFromVmJson(value, { mtimeMs }))) + return _writes.flush().then(() => true) } -/** Remove a VM row (called when the vm.json file is deleted). */ -export function removeVmFromDb(id) { +/** Deletes follow every previously captured upsert. */ +export async function removeVmFromDb(id) { const r = repo() if (!r || !id) return false - return r.remove(id) + _deleted.add(id) + let removed = false + _writes.enqueue(async () => { + removed = await r.remove(id) + }) + await _writes.flush() + return removed } -/** Forget settings.active_vm so startup reconcile does not rewrite active.json. */ -export function clearActiveVmFromDb() { +export async function clearActiveVmFromDb() { const r = repo() if (!r) return false - r.clearActiveVmId() + _writes.enqueue(() => r.clearActiveVmId()) + await _writes.flush() return true } +export function flushVmDbSync() { + return _writes.flush() +} + +export async function pauseVmDbSync() { + _paused = true + stopVmWatch() + await flushVmDbSync() +} + +export function resumeVmDbSync() { + _paused = false + if (_projectRoot) startVmWatch(_projectRoot) +} + /** * Startup reconcile: * - file newer than DB row (or row missing) → upsert from file * - DB row exists but file missing → rebuild file from vm_json * @returns {{ upserted: number, rebuilt: number, active: string|null }} */ -export function reconcileVms(projectRoot) { +export async function reconcileVms(projectRoot) { const r = repo() if (!r) return { upserted: 0, rebuilt: 0, active: null } const dir = path.join(projectRoot, 'vms') const out = { upserted: 0, rebuilt: 0, active: null } const seen = new Set() + // The file snapshot and its conditional database write share the same queue + // as write hooks. A newer hook always executes after this snapshot. if (fs.existsSync(dir)) { - for (const f of fs.readdirSync(dir)) { - if (!isVmRecordFile(f)) continue - const file = path.join(dir, f) - try { - const vm = JSON.parse(fs.readFileSync(file, 'utf8')) - if (!vm?.id) continue + for (const name of fs.readdirSync(dir)) { + if (!isVmRecordFile(name)) continue + _writes.enqueue(async () => { + const file = path.join(dir, name) + let vm + let stat + try { + vm = JSON.parse(fs.readFileSync(file, 'utf8')) + stat = fs.statSync(file) + } catch (error) { + if (error.code === 'ENOENT' || error instanceof SyntaxError) return + throw error + } + if (!vm?.id || _deleted.has(vm.id)) return seen.add(vm.id) - const st = fs.statSync(file) - const row = r.get(vm.id) - if (!row || (row.file_mtime_ms || 0) < Math.floor(st.mtimeMs)) { - r.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) + const row = await r.get(vm.id) + if (!row || (row.file_mtime_ms || 0) < Math.floor(stat.mtimeMs)) { + await r.upsertFromVmJson(vm, { mtimeMs: stat.mtimeMs }) out.upserted += 1 } - } catch {} + }) } } + await flushVmDbSync() - // DB rows without files → rebuild files (restore / disaster pull-up) - for (const row of r.list()) { - if (seen.has(row.id) || !row.vm) continue - try { - fs.mkdirSync(dir, { recursive: true }) + let rows = [] + _writes.enqueue(async () => { + rows = await r.list() + }) + await flushVmDbSync() + for (const row of rows) { + if (seen.has(row.id)) continue + _writes.enqueue(async () => { + const current = await r.get(row.id) const file = path.join(dir, `${row.id}.json`) - atomicWriteJson(file, row.vm, { mode: 0o600 }) + if (!current?.vm || _deleted.has(row.id) || fs.existsSync(file)) return + fs.mkdirSync(dir, { recursive: true }) + atomicWriteJson(file, current.vm, { mode: 0o600 }) out.rebuilt += 1 - } catch {} + }) } + await flushVmDbSync() - // active.json ↔ settings.active_vm - const activeFile = path.join(dir, 'active.json') - try { + _writes.enqueue(async () => { + const activeFile = path.join(dir, 'active.json') if (fs.existsSync(activeFile)) { - const a = JSON.parse(fs.readFileSync(activeFile, 'utf8')) - if (a?.active_vm) { - r.setActiveVmId(a.active_vm) - out.active = a.active_vm + let active + try { + active = JSON.parse(fs.readFileSync(activeFile, 'utf8')) + } catch (error) { + if (error instanceof SyntaxError) return + throw error + } + if (active?.active_vm && !_deleted.has(active.active_vm)) { + await r.setActiveVmId(active.active_vm) + out.active = active.active_vm } } else { - const dbActive = r.getActiveVmId() - if (dbActive) { + const active = await r.getActiveVmId() + if (active && !_deleted.has(active)) { fs.mkdirSync(dir, { recursive: true }) - atomicWriteJson(activeFile, { active_vm: dbActive, updated_at: new Date().toISOString() }) - out.active = dbActive + atomicWriteJson(activeFile, { active_vm: active, updated_at: new Date().toISOString() }) + out.active = active } } - } catch {} - + }) + await flushVmDbSync() return out } @@ -146,21 +195,20 @@ export function reconcileVms(projectRoot) { * Install the write-through hook + directory watcher. * Call once at server startup, after openDatabase(). */ -export function initVmDbSync(projectRoot, { watch = true } = {}) { +export async function initVmDbSync(projectRoot, { watch = true } = {}) { _projectRoot = projectRoot - setVmWriteHook((filePath) => { - if (isVmJsonPath(filePath)) syncVmFile(filePath) + await flushVmDbSync() + _writes = new WriteQueue(isDbOpen() ? getDb() : null) + _paused = false + _deleted.clear() + setVmWriteHook(async (filePath) => { + if (isVmJsonPath(filePath)) syncVmFile(filePath).catch(() => {}) }) - const result = reconcileVms(projectRoot) + const result = await reconcileVms(projectRoot) // Older releases soft-deleted tenants without releasing their assigned slots. // Only confirmed deletions qualify; disabled/missing users and private slots stay isolated. const deletedOwners = isDbOpen() - ? new Set( - getDb() - .prepare('SELECT id FROM users WHERE deleted_at IS NOT NULL') - .all() - .map((row) => row.id), - ) + ? new Set((await getDb().prepare('SELECT id FROM users WHERE deleted_at IS NOT NULL').all()).map((row) => row.id)) : new Set() result.reclaimed_vm_ids = reclaimAssignedVms(projectRoot, deletedOwners, { onError: (error, name) => { @@ -177,6 +225,7 @@ export function initVmDbSync(projectRoot, { watch = true } = {}) { } catch { result.smbios_repaired = 0 } + await flushVmDbSync() if (watch) startVmWatch(projectRoot) return result } @@ -189,10 +238,12 @@ export function startVmWatch(projectRoot) { try { _watcher = fs.watch(dir, () => { if (_watchTimer) clearTimeout(_watchTimer) - _watchTimer = setTimeout(() => { + _watchTimer = setTimeout(async () => { _watchTimer = null try { - reconcileDeletionsAndChanges(projectRoot) + reconcileDeletionsAndChanges(projectRoot).catch((error) => { + console.warn(`[vm] database mirror failed: ${error.code || error.name}`) + }) } catch {} }, 500) if (typeof _watchTimer.unref === 'function') _watchTimer.unref() @@ -205,37 +256,21 @@ export function startVmWatch(projectRoot) { } } -function reconcileDeletionsAndChanges(projectRoot) { +async function reconcileDeletionsAndChanges(projectRoot) { const r = repo() if (!r) return - reconcileVmsChangesOnly(projectRoot, r) + if (!_paused) await reconcileVmsChangesOnly(projectRoot, r) } /** Watch-triggered pass: upsert changed files; DO NOT rebuild deleted files * (a panel-initiated VM delete must stay deleted — server removes the row). */ -function reconcileVmsChangesOnly(projectRoot, r) { +async function reconcileVmsChangesOnly(projectRoot) { const dir = path.join(projectRoot, 'vms') if (!fs.existsSync(dir)) return - for (const f of fs.readdirSync(dir)) { - if (!isVmRecordFile(f)) continue - const file = path.join(dir, f) - try { - const st = fs.statSync(file) - const vm = JSON.parse(fs.readFileSync(file, 'utf8')) - if (!vm?.id) continue - const row = r.get(vm.id) - if (!row || (row.file_mtime_ms || 0) < Math.floor(st.mtimeMs)) { - r.upsertFromVmJson(vm, { mtimeMs: st.mtimeMs }) - } - } catch {} + for (const file of fs.readdirSync(dir)) { + if (_paused) return + if (isVmRecordFile(file) || file === 'active.json') await syncVmFile(path.join(dir, file)) } - const activeFile = path.join(dir, 'active.json') - try { - if (fs.existsSync(activeFile)) { - const a = JSON.parse(fs.readFileSync(activeFile, 'utf8')) - if (a?.active_vm && a.active_vm !== r.getActiveVmId()) r.setActiveVmId(a.active_vm) - } - } catch {} } export function stopVmWatch() { diff --git a/src/server.mjs b/src/server.mjs index a4a97d8fb..fb1490e7a 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -34,6 +34,7 @@ import { ensureTelemetrySidecar } from './lib/vm/slot-process-status.mjs' import { createSlotBridgeHealer, powerOnSlot, reviveDownSlots } from './lib/vm/slot-recovery.mjs' import { createUsageProbeMonitor, normalizeUsageProbeConfig } from './lib/oauth/usage-probe-monitor.mjs' import { createCatalogMonitor } from './lib/admin/catalog-monitor.mjs' +import { pausePriceCatalog, resumePriceCatalog } from './lib/admin/price-catalog.mjs' import { normalizeOfficialCcConfig } from './lib/oauth/official-cc-bootstrap.mjs' import { invalidateLiveCredentialCache } from './lib/admin/panel-live-credentials.mjs' import { normalizeHealthProbeConfig, createHealthProbeMonitor, HEALTH_REAL_HEADER } from './lib/admin/health-probe.mjs' @@ -47,7 +48,7 @@ import { normalizeTiers } from './lib/pool/quota-tiers.mjs' import { ApiKeyStore } from './lib/admin/api-keys.mjs' import { keyScopeFromRequest } from './lib/admin/key-scope.mjs' import { GroupsRepo } from './lib/db/repos/groups-repo.mjs' -import { SettingsRepo } from './lib/db/repos/settings-repo.mjs' +import { RuntimeSettings as SettingsRepo, loadSettingsState } from './lib/db/settings-state.mjs' import { ApiEndpointStore } from './lib/admin/api-endpoints.mjs' import { ApiScheduler } from './lib/pool/api-scheduler.mjs' import { resolveInferenceBackend } from './lib/pool/api-protocol.mjs' @@ -72,7 +73,9 @@ import { GATEWAY_CAPABILITIES } from './lib/vm/execution-context.mjs' import { isTelemetryPath, telemetryInterceptResponse } from './lib/identity/telemetry-rewrite.mjs' import { openDatabase, closeDatabase } from './lib/db/database.mjs' import { runLegacyImport } from './lib/db/legacy-import.mjs' -import { initVmDbSync, stopVmWatch } from './lib/vm/vm-db-sync.mjs' +import { initVmDbSync, stopVmWatch, flushVmDbSync, pauseVmDbSync, resumeVmDbSync } from './lib/vm/vm-db-sync.mjs' +import { flushPendingWrites } from './lib/db/pending-writes.mjs' +import { loadGptModelPolicy } from './lib/protocol/gpt-model-policy.mjs' import { BackupService } from './lib/admin/backup-service.mjs' import { ClusterNodesRepo } from './lib/db/repos/cluster-nodes-repo.mjs' import { ClusterManager } from './lib/cluster/cluster-manager.mjs' @@ -139,21 +142,20 @@ const allowRate = createRateLimiter({ refillPerSec: cfg.limits.rate_refill, }) -// --- Persistent store (SQLite, sub2api-inspired) --- +// --- Persistent PostgreSQL store --- +const activeRequests = new Set() +let _shuttingDown = false const dataDir = cfg.paths.data || path.join(cfg.paths.root, 'data') -openDatabase({ dataDir }) -try { - loadModelPolicy() -} catch (e) { - console.warn('[model-policy] boot load failed', e?.message || e) -} -// one-time migration of legacy JSON files (data/*.json + request-logs) into the DB -const legacyImport = runLegacyImport({ dataDir, projectRoot: cfg.paths.project }) +await openDatabase({ dataDir }) +const legacyImport = await runLegacyImport({ dataDir, projectRoot: cfg.paths.project }) if (legacyImport?.imported) { console.log('[db] legacy JSON import done:', JSON.stringify(legacyImport.counts || {})) } +await loadSettingsState() +await loadModelPolicy() +await loadGptModelPolicy() // VM/credential mirror: write-through hook + startup reconcile + fs.watch -const vmSync = initVmDbSync(cfg.paths.project) +const vmSync = await initVmDbSync(cfg.paths.project) if (vmSync.upserted || vmSync.rebuilt) { console.log(`[db] vm mirror reconciled: upserted=${vmSync.upserted} rebuilt=${vmSync.rebuilt}`) } @@ -287,7 +289,7 @@ routingConfig.usage_probe = normalizeUsageProbeConfig(routingConfig.usage_probe) routingConfig.notify = normalizeNotifyConfig(routingConfig.notify) routingConfig.tiers = normalizeTiers(routingConfig.tiers, routingConfig.quota, routingConfig.concurrency) syncTierDefaultsIntoRouting() -stickyRouter = new StickyRouter({ dataDir, config: routingConfig }) +stickyRouter = await new StickyRouter({ dataDir, config: routingConfig }).init() accountQuota = new AccountQuota({ dataDir, config: routingConfig, @@ -300,29 +302,30 @@ accountQuota = new AccountQuota({ max_rpm: v.policy?.maxRpm ?? routingConfig?.concurrency?.default_max_rpm ?? 0, })), }) +await accountQuota.init() accountQuota.loadVmQuotaOverrides(listVms(cfg.paths.project)) const apiKeyStore = new ApiKeyStore({ dataDir: cfg.paths.data }) groupsRepo = new GroupsRepo() const apiEndpointStore = new ApiEndpointStore({ dataDir: cfg.paths.data }) const apiScheduler = new ApiScheduler() -apiScheduler.reload(apiEndpointStore.listRaw()) +apiScheduler.reload(await apiEndpointStore.listRaw()) const panelUsers = new PanelUserStore({ dataDir: cfg.paths.data, projectRoot: cfg.paths.project }) try { - panelUsers.bootstrapFromEnv(getPanelAdmin()) + await panelUsers.bootstrapFromEnv(getPanelAdmin()) } catch (e) { console.warn('[panel-users] bootstrap failed', e?.message || e) } attachPanelAuth({ - authenticate: (username, password) => panelUsers.authenticate(username, password), - lookupRole: (username) => panelUsers.getByUsername(username)?.role || null, + authenticate: async (username, password) => await panelUsers.authenticate(username, password), + lookupRole: async (username) => (await panelUsers.getByUsername(username))?.role || null, }) requestLog = new RequestLogStore({ dataDir: cfg.paths.data, mode: process.env.KIN_REQUEST_LOG_MODE || routingConfig?.logging?.mode || 'normal', }) if (routingConfig?.logging) { - requestLog.setConfig({ + await requestLog.setConfig({ // Env wins at boot so e2e / ops can force off|debug. Panel PUT still hot-updates. mode: process.env.KIN_REQUEST_LOG_MODE || routingConfig.logging.mode, retainDays: routingConfig.logging.retain_days, @@ -332,7 +335,7 @@ if (routingConfig?.logging) { }) } try { - requestLog.cleanup() + await requestLog.cleanup() } catch {} requestLog.startCleanupScheduler() @@ -369,8 +372,9 @@ proxyPool = new ProxyPool({ setVmSchedulable(cfg.paths.project, vmId, true) }, egressCheck: (proxy) => proxyEgressReady(proxy, cfg.paths.project), - repairEgress: (proxy) => ensureProxyEgress(cfg.paths.project, proxy), + repairEgress: async (proxy) => await ensureProxyEgress(cfg.paths.project, proxy), }) +await proxyPool.init() // An upgrade replaces bin/ files; slots mounting data/run/slot-guest-bin must see the new ones. try { refreshSlotGuestBinDir(cfg.paths.project) @@ -384,7 +388,7 @@ try { if (!proxyPool.snapshot().proxies?.length) { const seeded = proxyPool.ensureLocal() if (seeded.created) { - void ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(seeded.proxy.id)) + void (await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(seeded.proxy.id))) console.log('[bootstrap] seeded local egress px-local') } } @@ -392,7 +396,7 @@ try { console.warn('[bootstrap] local egress seed failed', e?.message || e) } -initPoolRuntime() +await initPoolRuntime() try { applyRoutingTierConcurrency(routingConfig.tiers) } catch (e) { @@ -446,8 +450,8 @@ function attachSlotRuntime(vm) { healthMonitor = createHealthProbeMonitor({ config: routingConfig.health_probe, listTargets: () => listVms(cfg.paths.project).map(attachSlotRuntime), - runChat: (vm, real) => - runVmTestChat({ + runChat: async (vm, real) => + await runVmTestChat({ projectRoot: cfg.paths.project, vmId: vm.id, model: real.model, @@ -465,8 +469,8 @@ const vmTestSchedule = new VmTestScheduleRunner({ projectRoot: cfg.paths.project, settings: () => new SettingsRepo(), list: () => listVms(cfg.paths.project), - runTest: (vmId, params) => - runVmTestChat({ + runTest: async (vmId, params) => + await runVmTestChat({ projectRoot: cfg.paths.project, vmId, ...params, @@ -486,8 +490,8 @@ credentialRefreshMonitor = createCredentialRefreshMonitor({ } catch {} return { ...vm, last_probe: lastProbe } }), - refreshOne: (vm) => - refreshWorkerCredentialForVm({ + refreshOne: async (vm) => + await refreshWorkerCredentialForVm({ vmId: vm.id, vmPath: path.join(cfg.paths.project, 'vms', `${vm.id}.json`), homeDir: path.join(cfg.paths.project, 'vms', vm.id, 'cli-home'), @@ -495,20 +499,20 @@ credentialRefreshMonitor = createCredentialRefreshMonitor({ force: false, }), }) -function bootSlot(id) { - return powerOnSlot({ projectRoot: cfg.paths.project, id, proxyPool, routing: routingConfig }) +async function bootSlot(id) { + return await powerOnSlot({ projectRoot: cfg.paths.project, id, proxyPool, routing: routingConfig }) } kernelWatchdog = createKernelWatchdog({ config: routingConfig.kernel_watchdog, listTargets: () => listVms(cfg.paths.project), getRouting: () => routingConfig, homeDirFor: (vm) => path.join(cfg.paths.project, 'vms', vm.id, 'cli-home'), - ensureTelemetry: (vm) => ensureTelemetrySidecar({ projectRoot: cfg.paths.project, vm }), - revive: (vm) => bootSlot(vm.id), + ensureTelemetry: async (vm) => await ensureTelemetrySidecar({ projectRoot: cfg.paths.project, vm }), + revive: async (vm) => await bootSlot(vm.id), healBridges: createSlotBridgeHealer({ projectRoot: cfg.paths.project }), - onFault: (vm, reason) => { + onFault: async (vm, reason) => { const title = `槽内核故障 ${vm.id}` - dispatchNotify(routingConfig.notify, { + await dispatchNotify(routingConfig.notify, { title, text: `【KIN】${title}\n\n${reason}\n已停止调度该槽；内核恢复健康后自动恢复调度。`, subject: `KIN · ${title}`, @@ -538,7 +542,7 @@ usageProbeMonitor = createUsageProbeMonitor({ poolScheduler.syncQuotaSchedule(vm, account) } catch {} }, - probeOne: (vm) => panel.buildProbeOne({ cfg, accountQuota, id: vm.id, routingConfig }), + probeOne: async (vm) => await panel.buildProbeOne({ cfg, accountQuota, id: vm.id, routingConfig }), }) catalogMonitor = createCatalogMonitor({ projectRoot: cfg.paths.project, @@ -548,8 +552,8 @@ catalogMonitor = createCatalogMonitor({ notifyMonitor = createNotifyMonitor({ config: routingConfig.notify, baseUrl: cfg.base_url, - snapshot: () => - panel.snapshotAccountPool({ + snapshot: async () => + await panel.snapshotAccountPool({ cfg, accountQuota, routingConfig, @@ -565,15 +569,18 @@ const backupService = new BackupService({ projectRoot: cfg.paths.project, configDir: path.dirname(routingConfigPath), }) -backupService.onRestored((db) => { - // A restore swaps the DatabaseSync instance and routing files. Rebuild every +backupService.onRestored(async (db) => { + await loadSettingsState(db) + await loadModelPolicy({ force: true }) + await loadGptModelPolicy({ force: true }) + // A restore swaps the database pool and routing files. Rebuild every // holder before reporting success; otherwise requests keep stale snapshots // or silently write through statements owned by the closed connection. for (const store of [apiKeyStore, apiEndpointStore, panelUsers, accountQuota, stickyRouter, proxyPool, requestLog]) { - store.rebind(db) + await store.rebind(db) } groupsRepo = new GroupsRepo(db) - apiScheduler.reload(apiEndpointStore.listRaw()) + apiScheduler.reload(await apiEndpointStore.listRaw()) routingConfig = loadRoutingConfig() migrateLegacyOpenAIPolicies(routingConfig) @@ -590,7 +597,7 @@ backupService.onRestored((db) => { stickyRouter.reloadConfig(routingConfig) accountQuota.reloadConfig(routingConfig) accountQuota.loadVmQuotaOverrides(listVms(cfg.paths.project)) - requestLog.setConfig({ + await requestLog.setConfig({ mode: process.env.KIN_REQUEST_LOG_MODE || routingConfig.logging?.mode, retainDays: routingConfig.logging?.retain_days, debugRetainDays: routingConfig.logging?.debug_retain_days, @@ -603,11 +610,36 @@ backupService.onRestored((db) => { usageProbeMonitor?.setConfig(routingConfig.usage_probe) notifyMonitor?.setConfig(routingConfig.notify) - initPoolRuntime() + await initPoolRuntime() + await clusterManager.rebind(db) reloadActiveVm(cfg) - vmTestSchedule.reload() + await vmTestSchedule.reload() +}) +backupService.onMaintenance(async (enabled) => { + if (enabled) { + vmTestSchedule.stop() + proxyPool.stopScheduler() + requestLog.stopCleanupScheduler() + for (const monitor of maintenanceMonitors()) await monitor?.stop?.() + while ([...activeRequests].some((request) => !/\/(?:backups)(?:\/|$)/.test(String(request.url || '')))) { + await new Promise((resolve) => setTimeout(resolve, 10)) + } + for (const monitor of maintenanceMonitors()) await monitor?.flush?.() + await vmTestSchedule.flush() + await pausePriceCatalog() + await flushRuntimeWrites() + await pauseVmDbSync() + } else { + if (_shuttingDown) return + resumeVmDbSync() + resumePriceCatalog() + for (const monitor of maintenanceMonitors()) await monitor?.start?.({ immediate: false }) + proxyPool.startScheduler() + requestLog.startCleanupScheduler() + await vmTestSchedule.start() + } }) -backupService.startScheduler() +await backupService.startScheduler() const stats = { requests: 0, @@ -687,10 +719,23 @@ async function refreshWorkerCredentialForVm({ vmId, vmPath, homeDir, vm, force = return published } -const { json, writeSSEHeaders, readBody, readRawBody } = createRespond(cfg, { +const { + json: sendJson, + writeSSEHeaders, + readBody, + readRawBody, +} = createRespond(cfg, { tcpNodelay: () => routingConfig?.inference?.tcp_nodelay !== false, }) +function json(res, ...args) { + if (res._panelResponseState) { + res._panelResponseState.response = [res, ...args] + return + } + return sendJson(res, ...args) +} + function rejectAuth(req, res, error) { req.authError = { code: error?.body?.error?.code || ErrorCode.INVALID_API_KEY, @@ -701,7 +746,7 @@ function rejectAuth(req, res, error) { return false } -function requireAuth(req, res) { +async function requireAuth(req, res) { req.presentedApiKey = extractApiKey(req) const token = extractPanelToken(req) || req.presentedApiKey if (!token) { @@ -717,7 +762,7 @@ function requireAuth(req, res) { ) } // Panel session - const session = verifyPanelSession(token) + const session = await verifyPanelSession(token) if (session) { if (!allowRate('panel:' + session.user)) { return rejectAuth( @@ -732,8 +777,8 @@ function requireAuth(req, res) { ) } req.panelUser = session.user - req.panelRole = session.role || panelUsers.getByUsername(session.user)?.role || 'user' - const rec = panelUsers.getByUsername(session.user) + req.panelRole = session.role || (await panelUsers.getByUsername(session.user))?.role || 'user' + const rec = await panelUsers.getByUsername(session.user) req.panelUserId = rec?.id || null req.panelVmCreateQuota = rec?.vm_create_quota ?? 0 return true @@ -759,7 +804,7 @@ function requireAuth(req, res) { } // Managed multi-keys (sub2api-style) - const managed = apiKeyStore.authenticate(token) + const managed = await apiKeyStore.authenticate(token) if (!managed.ok) { return rejectAuth( req, @@ -772,7 +817,7 @@ function requireAuth(req, res) { }), ) } - const gate = apiKeyStore.canAccept(managed.record) + const gate = await apiKeyStore.canAccept(managed.record) if (!gate.ok) { const type = gate.status === 429 @@ -834,7 +879,7 @@ const importCommit = createImportCommit({ const { commitImportedOauth, requireSlotProxy, officialCcStatsHandler } = importCommit -const { handleProtocol, handleSearch } = createHandleProtocol({ +const protocolHandlers = createHandleProtocol({ json, writeSSEHeaders, readBody, @@ -869,6 +914,23 @@ const { handleProtocol, handleSearch } = createHandleProtocol({ }, }) +async function dispatchProtocol(name, req, res, ...args) { + if (backupService.isMaintaining) { + return json(res, 503, { + error: { type: 'overloaded_error', code: 'maintenance_in_progress', message: 'Database maintenance in progress' }, + }) + } + activeRequests.add(req) + try { + return await protocolHandlers[name](req, res, ...args) + } finally { + activeRequests.delete(req) + } +} + +const handleProtocol = async (...args) => await dispatchProtocol('handleProtocol', ...args) +const handleSearch = async (...args) => await dispatchProtocol('handleSearch', ...args) + const responsesWs = createResponsesWsHandler({ handleProtocol, requireAuth, @@ -911,10 +973,12 @@ const clusterManager = new ClusterManager({ throw new ClusterError(409, 'node_in_use', '节点仍有进行中的请求、虚拟机操作或镜像构建，请完成后重试') } }, - onNodesChanged: () => syncNodeLocalEgress(), + onNodesChanged: () => { + if (!backupService.isMaintaining) syncNodeLocalEgress() + }, // A node's local row follows its link: re-probe as soon as the link settles. onLinkState: (nodeId, state) => { - if (state === 'ready' || state === 'error' || state === 'backoff') { + if (!backupService.isMaintaining && (state === 'ready' || state === 'error' || state === 'backoff')) { proxyPool.probeLocalNode(nodeId).catch(() => {}) } }, @@ -922,7 +986,7 @@ const clusterManager = new ClusterManager({ localEgressCluster = clusterManager bindPlacement({ manager: clusterManager, projectRoot: cfg.paths.project }) await startNodeEgressSocks({ clientFor: (nodeId) => clusterManager.client(nodeId) }) -clusterManager.start() +await clusterManager.start() clusterManager.restoreSlotRelays() syncNodeLocalEgress() if (proxyPool.snapshot().config.ipv6_enabled !== true) { @@ -947,6 +1011,14 @@ cliNodeGuard = createCliNodeGuard({ }) const handlePanel = createPanelHandler({ + onPersistenceFailure: async () => { + backupService.isMaintaining = true + vmTestSchedule.stop() + proxyPool.stopScheduler() + requestLog.stopCleanupScheduler() + stopVmWatch() + for (const monitor of maintenanceMonitors()) await monitor?.stop?.() + }, json, writeSSEHeaders, readBody, @@ -1018,6 +1090,7 @@ const handlePanel = createPanelHandler({ }) const server = http.createServer(async (req, res) => { + activeRequests.add(req) try { res.req = req if (req.method === 'OPTIONS') { @@ -1036,8 +1109,10 @@ const server = http.createServer(async (req, res) => { } if ( - backupService.isRestoring && - (p.startsWith('/v1/') || p === '/messages' || p === '/chat/completions' || p === '/responses') + backupService.isMaintaining && + p !== '/healthz' && + p !== '/health' && + !(req.method === 'POST' && /^\/(?:api\/panel|admin)\/backups\/[^/]+\/restore$/.test(p)) ) { return json( res, @@ -1085,7 +1160,7 @@ const server = http.createServer(async (req, res) => { health_probe: healthMonitor?.getSnapshot?.() || null, api_kernel: { ...apiKernelSnapshot(cfg), - endpoints: apiEndpointStore.listRaw().filter((ep) => !ep.disabled).length, + endpoints: (await apiEndpointStore.listRaw()).filter((ep) => !ep.disabled).length, }, }) } @@ -1111,15 +1186,15 @@ const server = http.createServer(async (req, res) => { } if (req.method === 'GET' && p === '/v1/models') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return if (resolveInferenceBackend(req) === 'api') { return json(res, 200, apiScheduler.catalog()) } - const result = await fetchWorkerModels({ groupType: keyScopeFromRequest(req).group_type }) + const result = await fetchWorkerModels({ groupType: (await keyScopeFromRequest(req)).group_type }) return json(res, 200, result) } if (req.method === 'POST' && p === '/admin/models/refresh') { - if (!requireAuth(req, res)) return + if (!(await requireAuth(req, res))) return const result = await fetchWorkerModels() return json(res, 200, result) } @@ -1181,11 +1256,17 @@ const server = http.createServer(async (req, res) => { res.end() } catch {} } + } finally { + activeRequests.delete(req) } }) -server.on('upgrade', (req, socket, head) => { - if (responsesWs.handleUpgrade(req, socket, head)) return +server.on('upgrade', async (req, socket, head) => { + if (backupService.isMaintaining) { + socket.end('HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n') + return + } + if (await responsesWs.handleUpgrade(req, socket, head)) return if (clusterRoutes.handleUpgrade(req, socket, head)) return if (!slotShell.handleUpgrade(req, socket, head)) socket.destroy() }) @@ -1238,9 +1319,8 @@ fs.writeFileSync(path.join(cfg.paths.root, 'config', 'gateway-v2.public.json'), process.on('uncaughtException', (e) => console.error('[uncaught]', e)) process.on('unhandledRejection', (e) => console.error('[unhandled]', e)) -// Graceful shutdown: stop schedulers/watchers, close the DB (WAL checkpoint). -let _shuttingDown = false -function shutdown(signal) { +// Graceful shutdown: stop schedulers/watchers, drain persistence, close the pool. +async function shutdown(signal) { if (_shuttingDown) return _shuttingDown = true console.log(`[shutdown] ${signal} — closing`) @@ -1285,21 +1365,54 @@ function shutdown(signal) { clusterManager.stop() } catch {} try { - server.close(() => {}) + await new Promise((resolve) => server.close(resolve)) } catch {} try { - closeDatabase() - } catch {} + await backupService.flush() + await pausePriceCatalog() + for (const monitor of maintenanceMonitors()) await monitor?.flush?.() + await vmTestSchedule.flush() + await flushRuntimeWrites() + await closeDatabase() + } catch (error) { + console.error(`[shutdown] persistence failed: ${error.code || error.name}`) + try { + await closeDatabase() + } catch {} + process.exit(1) + } process.exit(0) } -process.on('SIGTERM', () => shutdown('SIGTERM')) -process.on('SIGINT', () => shutdown('SIGINT')) +process.on('SIGTERM', async () => await shutdown('SIGTERM')) +process.on('SIGINT', async () => await shutdown('SIGINT')) + +function maintenanceMonitors() { + return [ + healthMonitor, + credentialRefreshMonitor, + kernelWatchdog, + cliNodeGuard, + usageProbeMonitor, + catalogMonitor, + notifyMonitor, + ] +} + +async function flushRuntimeWrites() { + await requestLog.flush?.() + await flushPendingWrites() + await accountQuota.flush() + await stickyRouter.flush() + await proxyPool.flush() + await runtimeRepo?.flush?.() + await flushVmDbSync() +} await bootApiKernel(cfg) -server.listen(cfg.port, cfg.host, () => { +server.listen(cfg.port, cfg.host, async () => { try { - vmTestSchedule.start() + await vmTestSchedule.start() } catch (e) { console.warn('[vm-test-schedule] start failed', e?.message || e) } diff --git a/test/unit/account-quota.test.mjs b/test/unit/account-quota.test.mjs index df8f20fe3..c6c5b6e98 100644 --- a/test/unit/account-quota.test.mjs +++ b/test/unit/account-quota.test.mjs @@ -11,11 +11,11 @@ function tmpDir() { } test('ensure seeds account and is idempotent', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: {}, accounts: [{ account_id: 'a1', vm_id: 'vm-1', email: 'a@x' }], - }).init()) + }).init() const acc = q.ensure({ account_id: 'a1' }) assert.equal(acc.vm_id, 'vm-1') assert.equal(acc.email, 'a@x') @@ -24,7 +24,7 @@ test('ensure seeds account and is idempotent', async () => { }) test('a model entitlement rejection is persisted for that account and model', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'a-denied' }) q.markModelUnsupported('a-denied', 'claude-fable-5') const until = q.repo.get('a-denied').unified.model_denied_until['claude-fable-5'] @@ -39,7 +39,7 @@ function futureReset(msFromNow) { } test('ingestHeaders updates unified windows + counters + allocations', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'a2', { @@ -62,7 +62,7 @@ test('ingestHeaders updates unified windows + counters + allocations', async () }) test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders('a-health', { 'anthropic-ratelimit-unified-5h-utilization': '0.91', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -87,7 +87,7 @@ test('ingestHeaders countRequest:false does not overwrite Extra with hello 0%', }) test('ingestHeaders accumulates cache tokens and mirrors the session window', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const windows = [] q.attachRuntimeRepo({ updateWindow: (accountId, patch) => windows.push({ accountId, ...patch }) }) q.ingestHeaders( @@ -116,7 +116,7 @@ test('ingestHeaders accumulates cache tokens and mirrors the session window', as }) test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'a-oa', {}, @@ -134,7 +134,7 @@ test('ingestHeaders reads OpenAI prompt_tokens_details cache fields', async () = }) test('test-chat last_probe does not overwrite a successful usage probe', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.recordLastProbe('a-keep', { ok: true, source: 'vm-oauth-usage' }) q.recordLastProbe('a-keep', { ok: false, @@ -147,10 +147,10 @@ test('test-chat last_probe does not overwrite a successful usage probe', async ( }) test('canAccept blocks at safety ratio and records last_blocked', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestHeaders('a3', { 'anthropic-ratelimit-unified-5h-utilization': '0.96' }) const gate = q.canAccept('a3') assert.equal(gate.ok, false) @@ -160,7 +160,7 @@ test('canAccept blocks at safety ratio and records last_blocked', async () => { }) test('cli rate_limit_event blocks via status', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestCliRateLimit('a4', { rateLimitType: 'five_hour', status: 'rejected', resetsAt: 1893456000 }) const gate = q.canAccept('a4') assert.equal(gate.ok, false) @@ -168,10 +168,10 @@ test('cli rate_limit_event blocks via status', async () => { }) test('official 26 percent does not trip the quota gate', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestHeaders('pct-26', { 'anthropic-ratelimit-unified-5h-utilization': '26', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -201,7 +201,7 @@ test('extraHeadersFromLimitError fills Extra 5h + reset from the text when wrap }) test('limit error without a fresh reset does not inherit the elapsed reset', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const elapsed = String(Math.floor((Date.now() - 3_600_000) / 1000)) q.ingestHeaders('elapsed-5h', { 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -219,7 +219,7 @@ test('limit error without a fresh reset does not inherit the elapsed reset', asy }) test('cli rate_limit_event copies unifiedWindows utilization', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestCliRateLimit('a4-util', { rateLimitType: 'five_hour', status: 'allowed', @@ -232,10 +232,10 @@ test('cli rate_limit_event copies unifiedWindows utilization', async () => { }) test('tryAcquire enforces and releases the configured concurrency cap', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), accounts: [{ account_id: 'acc-reserve', max_concurrency: 1, concurrency_override: 1 }], - }).init()) + }).init() assert.equal(q.tryAcquire('acc-reserve').ok, true) assert.equal(q.tryAcquire('acc-reserve').reason, 'concurrency_limit') q.release('acc-reserve') @@ -245,10 +245,10 @@ test('tryAcquire enforces and releases the configured concurrency cap', async () test('state persists across re-open (same dataDir)', async () => { const dir = tmpDir() - const q1 = (await new AccountQuota({ dataDir: dir, config: {} }).init()) + const q1 = await new AccountQuota({ dataDir: dir, config: {} }).init() q1.ingestHeaders('a5', { 'anthropic-ratelimit-unified-7d-utilization': '0.5' }, { input_tokens: 7, output_tokens: 3 }) - const q2 = (await new AccountQuota({ dataDir: dir, config: {} }).init()) + const q2 = await new AccountQuota({ dataDir: dir, config: {} }).init() const acc = q2.snapshot().accounts.find((a) => a.account_id === 'a5') assert.ok(acc) assert.equal(acc.unified['7d'].utilization, 0.5) @@ -258,7 +258,7 @@ test('state persists across re-open (same dataDir)', async () => { }) test('allocations trimmed to 50 per account', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() for (let i = 0; i < 60; i++) { q.ingestHeaders('a6', { 'anthropic-ratelimit-unified-5h-utilization': String(i / 100) }) } @@ -269,7 +269,10 @@ test('allocations trimmed to 50 per account', async () => { }) test('max_concurrency 0 blocks instead of treating as unlimited', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 20 } } }).init()) + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { concurrency: { default_max_per_account: 20 } }, + }).init() q.ensure({ account_id: 'zero', max_concurrency: 0 }) q.setMaxConcurrency('zero', 0, { override: true }) const gate = q.canAccept('zero') @@ -279,7 +282,7 @@ test('max_concurrency 0 blocks instead of treating as unlimited', async () => { }) test('rebindToVm moves the UUID row onto the new slot', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: '1c5a7a73', vm_id: 'vm-02', email: 'old@x' }) q.rebindToVm('1c5a7a73', 'vm-04', { email: 'new@x' }) const acc = q.snapshot().accounts.find((a) => a.account_id === '1c5a7a73') @@ -288,7 +291,10 @@ test('rebindToVm moves the UUID row onto the new slot', async () => { }) test('concurrency inflight gate stays in memory', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 1 } } }).init()) + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { concurrency: { default_max_per_account: 1 } }, + }).init() q.ensure({ account_id: 'a7' }) q.setMaxConcurrency('a7', 1, { override: true }) q.acquire('a7') @@ -300,7 +306,7 @@ test('concurrency inflight gate stays in memory', async () => { }) test('rpm window blocks then recovers', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'rpm-acc', max_concurrency: 8 }) q.setMaxRpm('rpm-acc', 2, { override: true }) assert.equal(q.tryAcquire('rpm-acc').ok, true) @@ -315,7 +321,7 @@ test('rpm window blocks then recovers', async () => { }) test('max_rpm 0 is unlimited', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'rpm-off', max_concurrency: 8, max_rpm: 0 }) for (let i = 0; i < 5; i++) { assert.equal(q.tryAcquire('rpm-off').ok, true) @@ -325,10 +331,10 @@ test('max_rpm 0 is unlimited', async () => { }) test('quota gate still blocks before RPM wait', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ensure({ account_id: 'rpm-quota', max_concurrency: 8, max_rpm: 60 }) q.ingestHeaders('rpm-quota', { 'anthropic-ratelimit-unified-5h-utilization': '1', @@ -341,10 +347,10 @@ test('quota gate still blocks before RPM wait', async () => { }) test('ingestOAuthUsage stores 5h/7d and isolated fable limit', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-fable', { ok: true, five_hour: { utilization: 0.2, resets_at: '2026-08-18T20:00:00Z', status: 'allowed' }, @@ -375,10 +381,10 @@ test('ingestOAuthUsage stores 5h/7d and isolated fable limit', async () => { }) test('official /usage 429 keeps the last successful probe', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-rl', { ok: true, source: 'official-cc-usage', @@ -405,10 +411,10 @@ test('official /usage 429 keeps the last successful probe', async () => { }) test('usage ok plus fable 401 does not poison last_probe with revoke text', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-pro-401', { ok: true, usage_status: 200, @@ -437,7 +443,7 @@ test('usage ok plus fable 401 does not poison last_probe with revoke text', asyn }) test('Fable 403 does not classify Pro when official usage cannot verify the grant', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-uncertain-403', { ok: false, usage_status: 500, @@ -450,7 +456,7 @@ test('Fable 403 does not classify Pro when official usage cannot verify the gran }) test('official usage without Fable evidence does not classify an unprobed account as Pro', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-unprobed', { ok: true, usage_status: 200, @@ -464,7 +470,7 @@ test('official usage without Fable evidence does not classify an unprobed accoun }) test('complete official usage without Fable classifies Pro', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-complete-pro', { ok: true, usage_status: 200, @@ -480,7 +486,7 @@ test('complete official usage without Fable classifies Pro', async () => { }) test('complete official usage overrides a stale profile tier', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'acc-stale-profile' }) q.setAccountTier('acc-stale-profile', 'max', { source: 'profile' }) q.ingestOAuthUsage('acc-stale-profile', { @@ -498,7 +504,7 @@ test('complete official usage overrides a stale profile tier', async () => { }) test('setup-token usage scope failure preserves historical tier and does not mark grant dead', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'acc-setup-scope' }) q.setAccountTier('acc-setup-scope', 'max', { source: 'usage' }) q.ingestOAuthUsage('acc-setup-scope', { @@ -516,7 +522,7 @@ test('setup-token usage scope failure preserves historical tier and does not mar }) test('incomplete official usage without Fable does not downgrade a stored Max', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'acc-incomplete-max' }) q.setAccountTier('acc-incomplete-max', 'max', { source: 'usage' }) q.ingestOAuthUsage('acc-incomplete-max', { @@ -532,10 +538,10 @@ test('incomplete official usage without Fable does not downgrade a stored Max', }) test('clearGrantRevokeLeftover drops stale revoke after refresh', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-stale', { ok: false, usage_status: 401, @@ -553,10 +559,10 @@ test('clearGrantRevokeLeftover drops stale revoke after refresh', async () => { }) test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-max', { ok: true, usage_status: 200, @@ -584,10 +590,10 @@ test('clearGrantRevokeLeftover keeps stored Max off leftover Fable revoke', asyn }) test('fable 429 without 7d_oi window does not invent a full Fable quota', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-pro', { ok: true, five_hour: { utilization: 0.1, status: 'allowed' }, @@ -605,10 +611,10 @@ test('fable 429 without 7d_oi window does not invent a full Fable quota', async }) test('official usage without Fable hop does not turn leftover 429 into Pro', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-pro-skip', { ok: true, five_hour: { utilization: 0.1, status: 'allowed' }, @@ -641,10 +647,10 @@ test('official usage without Fable hop does not turn leftover 429 into Pro', asy }) test('Fable hop alone does not classify Pro; complete usage can flip stored Pro to Max', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-mispro', { ok: true, usage_status: 200, @@ -674,10 +680,10 @@ test('Fable hop alone does not classify Pro; complete usage can flip stored Pro }) test('7d_oi window is Fable-only and does not block the account', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const reset = futureReset(7 * 24 * 3600_000) q.ingestHeaders('acc-oi', { 'anthropic-ratelimit-unified-7d_oi-utilization': '1', @@ -691,10 +697,10 @@ test('7d_oi window is Fable-only and does not block the account', async () => { }) test('7d_oi rejected with elapsed second-level reset is not Fable-limited', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const now = Date.parse('2026-10-19T09:00:00Z') const resetSec = '1792206000' const acc = q.ensure({ account_id: 'acc-oi-elapsed' }) @@ -711,7 +717,7 @@ test('7d_oi rejected with elapsed second-level reset is not Fable-limited', asyn }) test('7d_oi rejected without reset stays Fable-limited', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const acc = q.ensure({ account_id: 'acc-oi-no-reset' }) acc.unified['7d_oi'] = { utilization: 1, status: 'rejected' } q.repo.save(acc) @@ -720,7 +726,7 @@ test('7d_oi rejected without reset stays Fable-limited', async () => { }) test('7d_oi rejected with future second-level reset stays Fable-limited', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const now = Date.parse('2026-10-19T09:00:00Z') const resetSec = '1792810800' const acc = q.ensure({ account_id: 'acc-oi-future' }) @@ -732,7 +738,7 @@ test('7d_oi rejected with future second-level reset stays Fable-limited', async }) test('persistEffectiveWindows clears elapsed unified 7d_oi', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() const now = Date.parse('2026-10-19T09:00:00Z') const acc = q.ensure({ account_id: 'acc-wipe-oi' }) acc.unified['7d_oi'] = { utilization: 1, reset: '1792206000', status: 'rejected' } @@ -746,10 +752,10 @@ test('persistEffectiveWindows clears elapsed unified 7d_oi', async () => { }) test('probe 5h rejected with no last_used stays a hard block until reset', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-sticky', { ok: true, @@ -764,7 +770,7 @@ test('probe 5h rejected with no last_used stays a hard block until reset', async }) test('official probe fills Extra reset when Extra has usage but no reset', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders('acc-no-reset', { 'anthropic-ratelimit-unified-5h-utilization': '0.44', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -784,10 +790,10 @@ test('official probe fills Extra reset when Extra has usage but no reset', async }) test('official leftover 100% does not overwrite live Extra allowed', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestHeaders('acc-live', { 'anthropic-ratelimit-unified-5h-utilization': '0.36', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -808,7 +814,7 @@ test('official leftover 100% does not overwrite live Extra allowed', async () => }) test('successful probe overwrites leftover rejected when official percent is low', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'acc-stale-status', { @@ -832,10 +838,10 @@ test('successful probe overwrites leftover rejected when official percent is low }) test('probe 5h rejected stays blocked when last_used is missing', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-no-used', { ok: true, source: 'official-cc-usage', @@ -853,10 +859,10 @@ test('probe 5h rejected stays blocked when last_used is missing', async () => { }) test('probe 5h rejected stays blocked when the slot was used in this window', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.attachRuntimeRepo({ get: () => ({ last_used_at: Date.now() - 60_000 }), @@ -873,7 +879,7 @@ test('probe 5h rejected stays blocked when the slot was used in this window', as }) test('persistEffectiveWindows writes a cleared 5h after reset elapsed', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestOAuthUsage('acc-persist', { ok: true, source: 'vm-oauth-usage', @@ -893,10 +899,10 @@ test('persistEffectiveWindows writes a cleared 5h after reset elapsed', async () }) test('elapsed 5h reset is not a hard block even after a live header write', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.95, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestHeaders('acc-elapsed', { 'anthropic-ratelimit-unified-5h-utilization': '1', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -906,7 +912,7 @@ test('elapsed 5h reset is not a hard block even after a live header write', asyn }) test('429 unified exhausted marks headers rejected without inventing official 100%', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders( 'acc-429', { @@ -924,7 +930,7 @@ test('429 unified exhausted marks headers rejected without inventing official 10 }) test('local 429 without quota evidence does not reject the 5h window', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ingestHeaders('acc-local-429', { 'anthropic-ratelimit-unified-5h-utilization': '0.23', 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -957,10 +963,10 @@ test('local 429 without quota evidence does not reject the 5h window', async () }) test('stuck 5h header with growing tokens does not lift official utilization', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true }, tiers: { default: { limit_5h: 0.85 } } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-stall', { ok: true, source: 'official-cc-usage', @@ -986,10 +992,10 @@ test('stuck 5h header with growing tokens does not lift official utilization', a }) test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() q.ingestOAuthUsage('acc-40', { ok: true, source: 'official-cc-usage', @@ -1007,10 +1013,10 @@ test('official /usage 40 percent is 0.40 and stays under the 80 percent gate', a }) test('persistEffectiveWindows clears leftover quota cooldown when official is under the gate', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, @@ -1044,10 +1050,10 @@ test('persistEffectiveWindows clears leftover quota cooldown when official is un }) test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, @@ -1086,7 +1092,10 @@ test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', asy }) test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() q.ingestOAuthUsage('acc-scale', { ok: true, source: 'official-cc-usage', @@ -1105,10 +1114,10 @@ test('/usage 40 writes Extra 0.40; 1 percent writes 0.01 not 1.0', async () => { }) test('official 0.90 plus header 0.40 is schedulable', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const acc = q.ensure({ account_id: 'acc-mix-open' }) acc.unified.official = { '5h': { utilization: 0.9, status: 'allowed', reset: futureReset(4 * 3600_000) }, @@ -1126,10 +1135,10 @@ test('official 0.90 plus header 0.40 is schedulable', async () => { }) test('header 0.90 plus official 0.40 hits the 5h gate', async () => { - const q = (await new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, weekly_safety_ratio: 0.8, block_on_5h: true, block_on_7d: true } }, - }).init()) + }).init() const acc = q.ensure({ account_id: 'acc-mix-shut' }) acc.unified.official = { '5h': { utilization: 0.4, status: 'allowed', reset: futureReset(4 * 3600_000) }, @@ -1145,7 +1154,10 @@ test('header 0.90 plus official 0.40 hits the 5h gate', async () => { }) test('elapsed Extra is not restored from leftover official 100%', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() const nextReset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-restore', { ok: true, @@ -1168,7 +1180,10 @@ test('elapsed Extra is not restored from leftover official 100%', async () => { }) test('already-wiped Extra stays empty when official leftover is still 100%', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() const nextReset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestOAuthUsage('acc-stale-wipe', { ok: true, @@ -1193,7 +1208,10 @@ test('already-wiped Extra stays empty when official leftover is still 100%', asy }) test('persistEffectiveWindows heals Extra overwritten by leftover official 100%', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() const reset = futureReset(4 * 3600_000) q.ingestHeaders('acc-heal', { 'anthropic-ratelimit-unified-5h-utilization': '0.36', @@ -1218,7 +1236,10 @@ test('persistEffectiveWindows heals Extra overwritten by leftover official 100%' }) test('elapsed header reset wipes Extra and becomes schedulable', async () => { - const q = (await new AccountQuota({ dataDir: tmpDir(), config: { quota: { safety_ratio: 0.85, block_on_5h: true } } }).init()) + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: { quota: { safety_ratio: 0.85, block_on_5h: true } }, + }).init() q.ingestHeaders('acc-wipe', { 'anthropic-ratelimit-unified-5h-utilization': '0.91', 'anthropic-ratelimit-unified-5h-status': 'rejected', @@ -1231,7 +1252,7 @@ test('elapsed header reset wipes Extra and becomes schedulable', async () => { }) test('complete usage may replace a profile-sourced tier', async () => { - const quota = (await new AccountQuota({ dataDir: tmpDir(), config: {} }).init()) + const quota = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() quota.ensure({ account_id: 'acct-profile', vm_id: 'vm-01' }) quota.setAccountTier('acct-profile', 'pro', { source: 'profile' }) quota.setAccountTier('acct-profile', 'max') @@ -1252,9 +1273,13 @@ test('complete usage may replace a profile-sourced tier', async () => { test('concurrent completed hops persist every quota counter update', async () => { const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'parallel-account' }) - await Promise.all(Array.from({ length: 20 }, () => Promise.resolve().then(() => { - q.ingestHeaders('parallel-account', {}, { input_tokens: 3, output_tokens: 2 }) - }))) + await Promise.all( + Array.from({ length: 20 }, () => + Promise.resolve().then(() => { + q.ingestHeaders('parallel-account', {}, { input_tokens: 3, output_tokens: 2 }) + }), + ), + ) await q.flush() const stored = await q.persistence.get('parallel-account') assert.equal(stored.requests, 20) @@ -1263,7 +1288,11 @@ test('concurrent completed hops persist every quota counter update', async () => }) test('restore reload does not recreate accounts from the original startup seed', async () => { - const q = await new AccountQuota({ dataDir: tmpDir(), config: {}, accounts: [{ account_id: 'old-startup-account' }] }).init() + const q = await new AccountQuota({ + dataDir: tmpDir(), + config: {}, + accounts: [{ account_id: 'old-startup-account' }], + }).init() await q.persistence.remove('old-startup-account') await q.rebind(q.db) assert.equal(q.repo.get('old-startup-account'), null) @@ -1275,12 +1304,22 @@ for (const source of ['headers', 'cli']) { const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'large-counters' }) await q.flush() - await q.db.prepare(`UPDATE accounts SET requests = $1, tokens_in = $1, tokens_out = $1, - cache_read_tokens = $1, cache_creation_tokens = $1 WHERE id = $2`).run('9007199254740993', 'large-counters') + await q.db + .prepare(`UPDATE accounts SET requests = $1, tokens_in = $1, tokens_out = $1, + cache_read_tokens = $1, cache_creation_tokens = $1 WHERE id = $2`) + .run('9007199254740993', 'large-counters') await q.reload() - if (source === 'headers') q.ingestHeaders('large-counters', {}, { - input_tokens: 2, output_tokens: 3, cache_read_input_tokens: 4, cache_creation_input_tokens: 5, - }) + if (source === 'headers') + q.ingestHeaders( + 'large-counters', + {}, + { + input_tokens: 2, + output_tokens: 3, + cache_read_input_tokens: 4, + cache_creation_input_tokens: 5, + }, + ) else q.ingestCliRateLimit('large-counters', null, { input_tokens: 2, output_tokens: 3 }) await q.flush() const account = await q.persistence.get('large-counters') @@ -1289,5 +1328,24 @@ for (const source of ['headers', 'cli']) { assert.equal(account.tokens_out, '9007199254740996') assert.equal(account.cache_read_tokens, source === 'headers' ? '9007199254740997' : '9007199254740993') assert.equal(account.cache_creation_tokens, source === 'headers' ? '9007199254740998' : '9007199254740993') + q.ensure({ account_id: 'large-delta' }) + if (source === 'headers') + q.ingestHeaders( + 'large-delta', + {}, + { + input_tokens: '9007199254740993', + output_tokens: '9007199254740995', + }, + ) + else + q.ingestCliRateLimit('large-delta', null, { + input_tokens: '9007199254740993', + output_tokens: '9007199254740995', + }) + await q.flush() + const delta = await q.persistence.get('large-delta') + assert.equal(delta.tokens_in, '9007199254740993') + assert.equal(delta.tokens_out, '9007199254740995') }) } diff --git a/test/unit/anthropic-console.test.mjs b/test/unit/anthropic-console.test.mjs index 26a04b4e8..17d76649e 100644 --- a/test/unit/anthropic-console.test.mjs +++ b/test/unit/anthropic-console.test.mjs @@ -462,7 +462,7 @@ test('saved Anthropic row reloads its proxy and can test or fetch models without const savedKey = response.body.item.api_key_entries[0] assert.equal(savedKey.proxy_id, 'px-1') - const reloaded = new ApiEndpointStore({ dataDir: dir }).list() + const reloaded = await new ApiEndpointStore({ dataDir: dir }).list() assert.equal(reloaded[0].api_key_entries[0].proxy_id, 'px-1') assert.equal(reloaded[0].api_key_entries[0].api_key.includes('sk-ant-saved'), false) diff --git a/test/unit/api-backend.test.mjs b/test/unit/api-backend.test.mjs index 587be2c9d..bec7682b0 100644 --- a/test/unit/api-backend.test.mjs +++ b/test/unit/api-backend.test.mjs @@ -1,3 +1,4 @@ +import { flushPendingWrites } from '../../src/lib/db/pending-writes.mjs' import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' @@ -295,6 +296,7 @@ test('logs the session id actually sent upstream, not the caller one', async () try { await handler.handleProtocol(req, response, 'anthropic.messages', '/v1/messages') response.finish?.() + await flushPendingWrites() const sent = sessionIdFromOutboundBody(prepared.body) || prepared.meta.sessionId assert.ok(sent) assert.equal(logged.session_id, 'caller-sess') @@ -310,7 +312,10 @@ for (const previousVm of [null, 'vm-01']) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-queued-session-')) const routingFile = path.join(root, 'routing.json') fs.writeFileSync(routingFile, JSON.stringify({ compatibility: {} })) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() const key = sticky.canonicalSessionKey('queued-caller') if (previousVm) { sticky.bind(key, { accountId: 'account-1', vmId: previousVm, sessionId: 'previous-outbound' }) @@ -339,9 +344,9 @@ for (const previousVm of [null, 'vm-01']) { const secondReady = new Promise((resolve) => { enteredSecond = resolve }) - t.after(() => { + t.after(async () => { releaseFirst() - sticky.db.close() + await sticky.db.close() fs.rmSync(root, { recursive: true, force: true }) }) const sent = [] @@ -833,13 +838,16 @@ function sessionBody(userId, extra = {}) { } } -function realStickyRouter() { +async function realStickyRouter() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-identity-sticky-')) - return { dir, router: new StickyRouter({ dataDir: dir, config: { sticky: { enabled: true, ttl_seconds: 600 } } }) } + return { + dir, + router: await new StickyRouter({ dataDir: dir, config: { sticky: { enabled: true, ttl_seconds: 600 } } }).init(), + } } test('inbound session and device keys match across API keys at the protocol entry', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const body = sessionBody({ device_id: 'dev-cross', account_uuid: 'acct-a', session_id: 'sess-cross' }) const a = await captureRunOpts({ body, stickyRouter: router, apiKeyRecord: { id: 'key-a', group_id: 1 } }) @@ -864,7 +872,7 @@ test('inbound session and device keys match across API keys at the protocol entr }) test('the protocol entry hands the device seat key to the pool, independent of the API key', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const turn = { messages: [{ role: 'user', content: 'please refactor the scheduler module and explain every single change' }], @@ -888,7 +896,7 @@ test('the protocol entry hands the device seat key to the pool, independent of t }) test('shared API key keeps different devices apart at the protocol entry', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const apiKeyRecord = { id: 'shared-key', group_id: 1 } const a = await captureRunOpts({ @@ -909,7 +917,7 @@ test('shared API key keeps different devices apart at the protocol entry', async }) test('explicit body device_id is the fallback when metadata has no device', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const opts = await captureRunOpts({ body: sessionBody({ session_id: 'sess-fallback' }, { device_id: 'dev-explicit' }), @@ -925,7 +933,7 @@ test('explicit body device_id is the fallback when metadata has no device', asyn }) test('legacy API-key-scoped session row migrates lazily at the protocol entry', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const apiKeyRecord = { id: 'key-old', group_id: 1 } const legacyKey = 'p:anthropic:kkey-old:sess-legacy' @@ -959,7 +967,7 @@ test('legacy API-key-scoped session row migrates lazily at the protocol entry', }) test('parent and child family key is shared across API keys and inherits a live legacy family', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { router.bind('p:anthropic:kkey-p:family:parent-sess', { accountId: 'acc-fam', vmId: 'vm-fam' }) const parent = await captureRunOpts({ @@ -983,15 +991,15 @@ test('parent and child family key is shared across API keys and inherits a live }) test('Claude Code sub-agent hops run as child sessions of the main session', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const apiKeyRecord = { id: 'key-cc', group_id: 1 } const body = sessionBody({ device_id: 'dev-cc', session_id: 'main-sess' }) const main = await captureRunOpts({ body, stickyRouter: router, apiKeyRecord }) router.bind(main.stickyKey, { accountId: 'acc-cc', vmId: 'vm-cc', slotIndex: 0 }) router.bind(main.familyKey, { accountId: 'acc-cc', vmId: 'vm-cc' }) - const agentHop = (agentId, parentAgentId) => - captureRunOpts({ + const agentHop = async (agentId, parentAgentId) => + await captureRunOpts({ body, stickyRouter: router, apiKeyRecord, @@ -1027,7 +1035,7 @@ test('Claude Code sub-agent hops run as child sessions of the main session', asy }) test('request without trusted session id keeps the legacy scoped sticky key', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const body = { model: 'claude-sonnet-4-5', @@ -1044,7 +1052,7 @@ test('request without trusted session id keeps the legacy scoped sticky key', as }) test('one-shot test call skips the session seat but keeps its sticky key', async () => { - const { dir, router } = realStickyRouter() + const { dir, router } = await realStickyRouter() try { const probe = await captureRunOpts({ body: sessionBody({ device_id: 'dev-probe', session_id: 'sess-probe' }, { max_tokens: 1024 }), diff --git a/test/unit/catalog-sync.test.mjs b/test/unit/catalog-sync.test.mjs index f8c280709..032d0d54d 100644 --- a/test/unit/catalog-sync.test.mjs +++ b/test/unit/catalog-sync.test.mjs @@ -27,17 +27,17 @@ function seedClaudeVm(root, id) { ) } -function withDb(t, catalogMode) { +async function withDb(t, catalogMode) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-catalog-sync-')) - openDatabase({ dbPath: ':memory:' }) - if (catalogMode) new SettingsRepo().set('model_policy', { catalog_mode: catalogMode, models: {} }) - loadModelPolicy({ force: true }) - loadGptModelPolicy({ force: true }) + await openDatabase({ dbPath: ':memory:' }) + if (catalogMode) await new SettingsRepo().set('model_policy', { catalog_mode: catalogMode, models: {} }) + await loadModelPolicy({ force: true }) + await loadGptModelPolicy({ force: true }) clearModelsCache() - t.after(() => { - closeDatabase() - loadModelPolicy({ force: true }) - loadGptModelPolicy({ force: true }) + t.after(async () => { + await closeDatabase() + await loadModelPolicy({ force: true }) + await loadGptModelPolicy({ force: true }) clearModelsCache() fs.rmSync(root, { recursive: true, force: true }) }) @@ -47,7 +47,7 @@ function withDb(t, catalogMode) { const upstream = (ids) => async () => ({ ok: true, status: 200, body: { data: ids.map((id) => ({ id })) } }) test('a Claude id new to the policy is added disabled outside worker_only', async (t) => { - const root = withDb(t, 'policy_only') + const root = await withDb(t, 'policy_only') seedClaudeVm(root, 'vm-1') const before = getModelPolicy().models['claude-opus-5'] const status = await syncClaudeCatalog({ @@ -63,14 +63,14 @@ test('a Claude id new to the policy is added disabled outside worker_only', asyn }) test('worker_only publishes a new Claude id immediately', async (t) => { - const root = withDb(t, 'worker_only') + const root = await withDb(t, 'worker_only') seedClaudeVm(root, 'vm-1') await syncClaudeCatalog({ projectRoot: root, runOauth: upstream(['claude-opus-6-preview']) }) assert.equal(getModelPolicy().models['claude-opus-6-preview'].enabled, true) }) test('Claude sync without a reachable slot records the failure and changes nothing', async (t) => { - const root = withDb(t, 'policy_only') + const root = await withDb(t, 'policy_only') seedClaudeVm(root, 'vm-1') const before = structuredClone(getModelPolicy().models) const status = await syncClaudeCatalog({ @@ -82,9 +82,9 @@ test('Claude sync without a reachable slot records the failure and changes nothi assert.deepEqual(getModelPolicy().models, before) }) -test('a GPT id new to the policy starts disabled; known ids keep their switch', (t) => { - withDb(t) - const result = syncGptIdsIntoPolicy([{ id: 'gpt-5.6' }, { id: 'gpt-7-preview' }]) +test('a GPT id new to the policy starts disabled; known ids keep their switch', async (t) => { + await withDb(t) + const result = await syncGptIdsIntoPolicy([{ id: 'gpt-5.6' }, { id: 'gpt-7-preview' }]) assert.deepEqual(result.added, ['gpt-7-preview']) const models = getGptModelPolicy().models assert.equal(models['gpt-5.6'].enabled, true) diff --git a/test/unit/cluster-direct-link.test.mjs b/test/unit/cluster-direct-link.test.mjs index 3ff849c12..48815ae60 100644 --- a/test/unit/cluster-direct-link.test.mjs +++ b/test/unit/cluster-direct-link.test.mjs @@ -6,7 +6,7 @@ import net from 'node:net' import os from 'node:os' import path from 'node:path' import tls from 'node:tls' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import { ClusterManager, DIRECT_LINK_BACKOFF_MS } from '../../src/lib/cluster/cluster-manager.mjs' import { certSha256, @@ -61,11 +61,11 @@ async function fakeGate(t, { token = TOKEN, slots = new Set(['vm-1']), extra = ' const pair = selfSigned(t) const seen = [] const held = new Set() - const server = tls.createServer({ ...pair, minVersion: 'TLSv1.3', allowHalfOpen: true }, (sock) => { + const server = tls.createServer({ ...pair, minVersion: 'TLSv1.3', allowHalfOpen: true }, async (sock) => { held.add(sock) sock.on('close', () => held.delete(sock)) let buf = Buffer.alloc(0) - const onData = (chunk) => { + const onData = async (chunk) => { buf = Buffer.concat([buf, chunk]) const nl = buf.indexOf(10) if (nl < 0) return @@ -75,7 +75,7 @@ async function fakeGate(t, { token = TOKEN, slots = new Set(['vm-1']), extra = ' seen.push({ vm, kind, early: parts[0].toString() }) if (magic !== 'KSG1' || tok !== token) return sock.end('ERR auth\n') if (kind === 'ping') return sock.end('OK\n') - if (!slots.has(vm)) return sock.end('ERR no_socket\n') + if (!(await slots.has(vm))) return sock.end('ERR no_socket\n') sock.write(`OK\n${extra}`) sock.on('data', (c) => parts.push(c)) sock.on('end', () => sock.end(Buffer.concat(parts).toString().toUpperCase())) @@ -84,9 +84,9 @@ async function fakeGate(t, { token = TOKEN, slots = new Set(['vm-1']), extra = ' sock.on('error', () => {}) }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) - t.after(() => { + t.after(async () => { for (const s of held) s.destroy() - server.close() + await server.close() }) return { address: '127.0.0.1', port: server.address().port, token, certSha256: certSha256(pair.cert), seen } } @@ -144,9 +144,9 @@ test('dialGate reports refusals through ready and silence as unreachable', async const held = [] const silent = net.createServer((sock) => held.push(sock)) await new Promise((resolve) => silent.listen(0, '127.0.0.1', resolve)) - t.after(() => { + t.after(async () => { for (const sock of held) sock.destroy() - silent.close() + await silent.close() }) await assert.rejects(dialGate({ ...gate, port: silent.address().port, kind: 'ping', timeoutMs: 150 }), { code: 'gate_unreachable', @@ -217,15 +217,11 @@ test('SocketRelay: direct dial, SSH fallback on null, local drop when the slot s assert.deepEqual(sshAsked, ['f.sock']) }) -function managerWithRepo(t) { - const db = new DatabaseSync(':memory:') - for (const name of ['026_cluster_nodes.sql', '039_cluster_node_direct_links.sql']) { - db.exec(fs.readFileSync(new URL(`../../src/lib/db/migrations/${name}`, import.meta.url), 'utf8')) - } - db.exec('PRAGMA foreign_keys = ON') - t.after(() => db.close()) +async function managerWithRepo(t) { + const db = getDb() + t.after(async () => await db.close()) const repo = new ClusterNodesRepo(db) - repo.insert({ + await repo.insert({ id: 'node-a', label: 'A', host: '203.0.113.5', @@ -242,26 +238,27 @@ function managerWithRepo(t) { dataDir: os.tmpdir(), logger: { log() {}, warn: (m) => warnings.push(m) }, }) + await manager.reload() return { db, repo, manager, warnings } } test('direct link: token and pin survive edits and never leave; failures fall back to SSH with backoff', async (t) => { - const { db, repo, manager, warnings } = managerWithRepo(t) + const { db, repo, manager, warnings } = await managerWithRepo(t) const gate = await fakeGate(t) await manager.setDirectLink('node-a', { address: '127.0.0.1', port: 1025 }) - const token = repo.getDirectLink('node-a').token + const token = (await repo.getDirectLink('node-a')).token assert.equal(await manager._dialDirect('node-a', 'vm-1', 'kernel'), null, 'no pin yet: SSH') - assert.equal(manager.get('node-a').direct_link.state, 'pending') + assert.equal((await manager.get('node-a')).direct_link.state, 'pending') - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) - repo.setDirectLinkCert('node-a', gate.certSha256) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) + await repo.setDirectLinkCert('node-a', gate.certSha256) await manager.setDirectLink('node-a', { address: gate.address, port: gate.port }) - assert.equal(repo.getDirectLink('node-a').token, TOKEN, 'edit keeps the token') + assert.equal((await repo.getDirectLink('node-a')).token, TOKEN, 'edit keeps the token') assert.notEqual(token, TOKEN) - assert.equal(repo.getDirectLink('node-a').certSha256, gate.certSha256, 'edit keeps the pin') - assert.equal(manager.get('node-a').direct_link.state, 'ok') - const listed = JSON.stringify(manager.list()) + assert.equal((await repo.getDirectLink('node-a')).certSha256, gate.certSha256, 'edit keeps the pin') + assert.equal((await manager.get('node-a')).direct_link.state, 'ok') + const listed = JSON.stringify(await manager.list()) assert.equal(listed.includes(TOKEN), false) assert.equal(listed.includes(gate.certSha256), false) @@ -271,28 +268,28 @@ test('direct link: token and pin survive edits and never leave; failures fall ba conn.sock.destroy() const missing = await manager._dialDirect('node-a', 'vm-2', 'kernel') await assert.rejects(missing.ready, { code: 'gate_no_socket' }) - assert.equal(manager.get('node-a').direct_link.state, 'ok', 'a missing slot socket is not a link fault') + assert.equal((await manager.get('node-a')).direct_link.state, 'ok', 'a missing slot socket is not a link fault') - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: 'c'.repeat(64) }) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: 'c'.repeat(64) }) const refused = await manager._dialDirect('node-a', 'vm-1', 'kernel') await assert.rejects(refused.ready, { code: 'gate_auth' }) assert.match(warnings.at(-1), /gate_auth/) - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) const before = gate.seen.length assert.equal(await manager._dialDirect('node-a', 'vm-1', 'kernel'), null, 'still in backoff') assert.equal(gate.seen.length, before) - assert.equal(manager.get('node-a').direct_link.state, 'down') - assert.ok(manager.directState.get('node-a').downUntil <= Date.now() + DIRECT_LINK_BACKOFF_MS) + assert.equal((await manager.get('node-a')).direct_link.state, 'down') + assert.ok((await manager.directState.get('node-a')).downUntil <= Date.now() + DIRECT_LINK_BACKOFF_MS) await manager._probeDirect('node-a') - assert.equal(manager.get('node-a').direct_link.state, 'ok', 'a good ping lifts the backoff') + assert.equal((await manager.get('node-a')).direct_link.state, 'ok', 'a good ping lifts the backoff') - repo.setDirectLink('node-a', { address: '127.0.0.1', port: 1, token: TOKEN }) + await repo.setDirectLink('node-a', { address: '127.0.0.1', port: 1, token: TOKEN }) assert.equal(await manager._dialDirect('node-a', 'vm-1', 'kernel'), null, 'unreachable: SSH') assert.match(warnings.at(-1), /gate_unreachable/) await manager.clearDirectLink('node-a') - assert.equal(manager.get('node-a').direct_link, null) - repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) - db.prepare('DELETE FROM cluster_nodes WHERE id = ?').run('node-a') - assert.equal(repo.getDirectLink('node-a'), null, 'removing the node drops its link') + assert.equal((await manager.get('node-a')).direct_link, null) + await repo.setDirectLink('node-a', { address: gate.address, port: gate.port, token: TOKEN }) + await db.prepare('DELETE FROM cluster_nodes WHERE id = $1').run('node-a') + assert.equal(await repo.getDirectLink('node-a'), null, 'removing the node drops its link') }) diff --git a/test/unit/cluster-placement.test.mjs b/test/unit/cluster-placement.test.mjs index d7e331317..038790105 100644 --- a/test/unit/cluster-placement.test.mjs +++ b/test/unit/cluster-placement.test.mjs @@ -595,10 +595,11 @@ test('socket relay resolves the remote path lazily and reaches a unix socket ove test('a node with placed VMs cannot be removed', async () => { const manager = new ClusterManager({ - repo: { get: () => ({ id: 'n1' }), dependentsOf: () => [] }, + repo: { list: () => [{ id: 'n1' }], get: () => ({ id: 'n1' }), dependentsOf: () => [] }, dataDir: '/tmp', vmsOnNode: () => [{ id: 'vm-02' }], }) + await manager.reload() await assert.rejects(manager.remove('n1'), { code: 'node_has_vms' }) }) diff --git a/test/unit/distill-detect.test.mjs b/test/unit/distill-detect.test.mjs index 8aaca2e65..ee873a0c7 100644 --- a/test/unit/distill-detect.test.mjs +++ b/test/unit/distill-detect.test.mjs @@ -608,10 +608,10 @@ test('normalize keeps default fingerprints when omitted', () => { test('handleProtocol intercepts before credential hop', () => { const src = fs.readFileSync(path.join(root, 'src/lib/protocol/handle-protocol.mjs'), 'utf8') const early = src.indexOf('await applyProtocolIntercept(') - const codex = src.indexOf('return handleCodexProtocol(') + const codex = src.indexOf('handleCodexProtocol({') assert.ok(early > 0 && early < codex) const before = src.indexOf("applyIntercept(cfg.intercept.rules, 'before_upstream'") - const late = src.indexOf('await applyProtocolIntercept(', early + 1) + const late = src.indexOf('await applyProtocolIntercept(', before) const api = src.indexOf("inferenceBackend === 'api'") assert.ok(before > 0) assert.ok(late > before) diff --git a/test/unit/failover-runner.test.mjs b/test/unit/failover-runner.test.mjs index b5c342e55..e465acc5b 100644 --- a/test/unit/failover-runner.test.mjs +++ b/test/unit/failover-runner.test.mjs @@ -909,9 +909,9 @@ test('aborted signal stops before another account is tried', async () => { for (const cancellation of ['result', 'throw']) { test(`client cancellation via ${cancellation} preserves outbound session beyond fifty turns`, async (t) => { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cancel-session-')) - const sticky = new StickyRouter({ dataDir, config: { sticky: { enabled: true } } }) - t.after(() => { - sticky.db.close() + const sticky = await new StickyRouter({ dataDir, config: { sticky: { enabled: true } } }).init() + t.after(async () => { + await sticky.db.close() fs.rmSync(dataDir, { recursive: true, force: true }) }) const scheduler = new Scheduler([{ ...candidate(1), slotIndex: 3 }]) diff --git a/test/unit/haiku-55-policy.test.mjs b/test/unit/haiku-55-policy.test.mjs index c613ead76..0037f0130 100644 --- a/test/unit/haiku-55-policy.test.mjs +++ b/test/unit/haiku-55-policy.test.mjs @@ -38,11 +38,11 @@ test('haiku 5.5 is in the panel seed and keeps forced tool choice', () => { assert.deepEqual(blocked.thinking, { type: 'adaptive' }) }) -test('loading old Haiku 5.5 seed migrates only the old 1M defaults to 100K', () => { - openDatabase({ dbPath: ':memory:' }) +test('loading old Haiku 5.5 seed migrates only the old 1M defaults to 100K', async () => { + await openDatabase({ dbPath: ':memory:' }) try { const repo = new SettingsRepo() - repo.set('model_policy', { + await repo.set('model_policy', { source: 'panel', models: { 'claude-haiku-5-5': { @@ -56,14 +56,14 @@ test('loading old Haiku 5.5 seed migrates only the old 1M defaults to 100K', () }, }, }) - loadModelPolicy({ force: true }) - const stored = repo.get('model_policy') + await loadModelPolicy({ force: true }) + const stored = await repo.get('model_policy') const row = stored.models['claude-haiku-5-5'] assert.equal(row.capabilities.context_window, 100000) assert.equal(row.params.max_tokens_default, 100000) assert.equal(row.params.max_tokens_cap, 100000) assert.equal(row.params.default_effort, 'medium') } finally { - closeDatabase() + await closeDatabase() } }) diff --git a/test/unit/handle-codex-search.test.mjs b/test/unit/handle-codex-search.test.mjs index 5b5cecac4..61d867529 100644 --- a/test/unit/handle-codex-search.test.mjs +++ b/test/unit/handle-codex-search.test.mjs @@ -1,3 +1,4 @@ +import { flushPendingWrites } from '../../src/lib/db/pending-writes.mjs' import { test } from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' @@ -234,6 +235,7 @@ test('the alpha/search route authenticates, logs and reaches the Codex search ho } await handleSearch(req, res, '/v1/alpha/search') res.finish() + await flushPendingWrites() assert.equal(res.statusCode, 502) assert.equal(res.body.error.code, 'proxy_required') assert.equal(rows.length, 1) diff --git a/test/unit/handle-codex.test.mjs b/test/unit/handle-codex.test.mjs index dc6fe9040..42f0d7ceb 100644 --- a/test/unit/handle-codex.test.mjs +++ b/test/unit/handle-codex.test.mjs @@ -1077,7 +1077,10 @@ async function hopCodex({ root, stickyRouter, routing = {}, headers = {}, body, test('codex rebuild outbound session is not the inbound session', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-sess-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const inbound = '11111111-1111-4111-8111-111111111111' const { envelopes } = await hopCodex({ @@ -1101,13 +1104,16 @@ test('codex rebuild outbound session is not the inbound session', async () => { assert.equal(envelopes[0].body.prompt_cache_key, session.session_id) assert.equal(envelopes[0].body.conversation_id, undefined) assert.equal(envelopes[0].body.session_id, undefined) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex kernel envelope preserves official routing headers without client auth', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-routing-headers-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const { envelopes } = await hopCodex({ root, @@ -1143,13 +1149,16 @@ test('codex kernel envelope preserves official routing headers without client au assert.equal(envelopes[0].headers['session-id'], session.session_id) assert.equal(envelopes[0].headers.authorization, undefined) assert.equal(envelopes[0].body.service_tier, 'priority') - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex rebuild outbound session is stable on the same slot', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-stable-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const inbound = 'sess-stable' const first = await hopCodex({ @@ -1165,7 +1174,7 @@ test('codex rebuild outbound session is stable on the same slot', async () => { body: { model: 'gpt-5.4', input: 'hi again', stream: false }, }) assert.equal(first.envelopes[0].session.session_id, second.envelopes[0].session.session_id) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) @@ -1173,7 +1182,10 @@ test('codex rebuild remints session after failover to another slot', async () => const rt = await import('../../src/lib/pool/openai-account-runtime.mjs') rt.resetOpenAIAccountRuntime() const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-fail-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') writeGptVm(root, 'vm-gpt-b') const inbound = 'sess-fail' @@ -1227,13 +1239,16 @@ test('codex rebuild remints session after failover to another slot', async () => assert.notEqual(envelopes[1].session.session_id, inbound) assert.notEqual(envelopes[0].session.session_id, envelopes[1].session.session_id) rt.resetOpenAIAccountRuntime() - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex passthrough keeps the inbound session and cache key', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-pass-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const inbound = 'sess-pass' const { envelopes } = await hopCodex({ @@ -1252,13 +1267,16 @@ test('codex passthrough keeps the inbound session and cache key', async () => { assert.equal(envelopes[0].session.session_id, inbound) assert.equal(envelopes[0].body.prompt_cache_key, 'cache-pass') assert.equal(envelopes[0].body.conversation_id, inbound) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) test('codex passthrough without an inbound session derives a stable fallback session', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-pass-fallback-')) - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() writeGptVm(root, 'vm-gpt-a') const opener = (question) => ({ model: 'gpt-5.4', @@ -1282,7 +1300,7 @@ test('codex passthrough without an inbound session derives a stable fallback ses assert.match(sessionId, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) assert.equal(again.envelopes[0].session.session_id, sessionId) assert.notEqual(other.envelopes[0].session.session_id, sessionId) - sticky.db?.close?.() + await sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) diff --git a/test/unit/jev-account-scope.test.mjs b/test/unit/jev-account-scope.test.mjs index c1a7fce38..15972088f 100644 --- a/test/unit/jev-account-scope.test.mjs +++ b/test/unit/jev-account-scope.test.mjs @@ -186,13 +186,13 @@ test('without a scope the GPT switch still decides and Claude still scans', asyn assert.equal(claude.action, 'block') }) -test('the panel PUT round-trips platform-plan pairs and rejects junk', () => { +test('the panel PUT round-trips platform-plan pairs and rejects junk', async () => { const stored = new Map() const settings = { get: (key, fallback) => (stored.has(key) ? stored.get(key) : fallback), set: (key, value) => stored.set(key, value), } - const saved = applyJevPatch(settings, { + const saved = await applyJevPatch(settings, { account_platform_plans: ['Claude:Max', 'gpt:plus'], account_ids: ['vm-02', 'ops@example.com'], account_types: [], @@ -202,7 +202,7 @@ test('the panel PUT round-trips platform-plan pairs and rejects junk', () => { assert.deepEqual(saved.config.account_platform_plans, ['claude:max', 'gpt:plus']) assert.equal(jevScopeConfigured(stored.get('jev_intercept')), true) - const cleared = applyJevPatch(settings, { account_platform_plans: [], account_ids: [] }) + const cleared = await applyJevPatch(settings, { account_platform_plans: [], account_ids: [] }) assert.equal(cleared.ok, true) assert.equal(jevScopeConfigured(stored.get('jev_intercept')), false) assert.deepEqual(publicJevConfig(null).account_platform_plans, []) diff --git a/test/unit/jev-intercept.test.mjs b/test/unit/jev-intercept.test.mjs index ddc522319..683fb9ded 100644 --- a/test/unit/jev-intercept.test.mjs +++ b/test/unit/jev-intercept.test.mjs @@ -128,9 +128,9 @@ test('hard regex blocks jailbreak and crack, not ordinary debugging', () => { assert.equal(matchHardPolicy('explain the difference between phishing and spam for a security class'), null) }) -test('saved rules replace the builtins and a bad regex is rejected', () => { +test('saved rules replace the builtins and a bad regex is rejected', async () => { let saved = null - const patched = applyJevPatch( + const patched = await applyJevPatch( { get: () => null, set: (_key, value) => { @@ -143,11 +143,11 @@ test('saved rules replace the builtins and a bad regex is rejected', () => { assert.equal(patched.config.rules_customized, true) assert.equal(matchHardPolicy('feat vmpanel-keygen', [], saved.rules).keyword, 'vmpanel') assert.equal(matchHardPolicy('write a keygen', [], saved.rules), null) - const bad = applyJevPatch({ get: () => null, set() {} }, { rules: [{ category: 'custom', source: '(' }] }) + const bad = await applyJevPatch({ get: () => null, set() {} }, { rules: [{ category: 'custom', source: '(' }] }) assert.equal(bad.ok, false) }) -test('an empty rule list falls back to the builtins instead of clearing the layer', () => { +test('an empty rule list falls back to the builtins instead of clearing the layer', async () => { let saved = null const settings = { get: () => null, @@ -156,7 +156,7 @@ test('an empty rule list falls back to the builtins instead of clearing the laye }, } assert.equal(normalizeJevConfig({ rules: [] }).rules, null) - const patched = applyJevPatch(settings, { rules: [] }) + const patched = await applyJevPatch(settings, { rules: [] }) assert.equal(patched.ok, true) assert.equal(patched.config.rules_customized, false) assert.equal(patched.config.rules.length, builtinHardRules().length) @@ -165,7 +165,7 @@ test('an empty rule list falls back to the builtins instead of clearing the laye 'jailbreak', ) // The panel can still turn the layer off with its own switch. - const off = applyJevPatch(settings, { hard_regex_enabled: false }) + const off = await applyJevPatch(settings, { hard_regex_enabled: false }) assert.equal(off.config.hard_regex_enabled, false) assert.equal(saved.rules, null) }) @@ -339,23 +339,30 @@ test('content-policy error codes are refusals; generic api errors are not', () = assert.equal(isUpstreamRefusal({ body: { error: { code: 'distill_blocked', message: '不允许蒸馏' } } }), false) }) -test('panel patch keeps a stored key unless cleared', () => { +test('panel patch keeps a stored key unless cleared', async () => { const saved = new Map() const settings = { get: (key, fallback) => (saved.has(key) ? saved.get(key) : fallback), set: (key, value) => saved.set(key, value), } assert.equal( - applyJevPatch(settings, { enabled: true, base_url: 'http://127.0.0.1:9/v1', model: 'jev', api_key: 'secret' }).ok, + ( + await applyJevPatch(settings, { + enabled: true, + base_url: 'http://127.0.0.1:9/v1', + model: 'jev', + api_key: 'secret', + }) + ).ok, true, ) - const kept = applyJevPatch(settings, { timeout_ms: 1500 }) + const kept = await applyJevPatch(settings, { timeout_ms: 1500 }) assert.equal(kept.ok, true) assert.equal(kept.config.api_key_set, true) const stored = saved.get('jev_intercept') assert.equal(publicJevConfig(stored).api_key_set, true) assert.equal(stored.api_key, 'secret') - assert.equal(applyJevPatch(settings, { api_key: '' }).config.api_key_set, false) + assert.equal((await applyJevPatch(settings, { api_key: '' })).config.api_key_set, false) assert.equal(validateJevPatch({ timeout_ms: 10 }).length, 1) }) @@ -763,7 +770,10 @@ test('401 retires that key until the key list is saved again', async () => { settings.value = value }, } - assert.equal(applyJevPatch(settings, { api_keys: ['k-auth-retire'], base_url: cfg.base_url, enabled: true }).ok, true) + assert.equal( + (await applyJevPatch(settings, { api_keys: ['k-auth-retire'], base_url: cfg.base_url, enabled: true })).ok, + true, + ) let retried = 0 await classifyJev('retire-auth-3', { ...cfg, api_keys: ['k-auth-retire'] }, async () => { retried += 1 diff --git a/test/unit/key-scope.test.mjs b/test/unit/key-scope.test.mjs index 45c6be6af..2949aa612 100644 --- a/test/unit/key-scope.test.mjs +++ b/test/unit/key-scope.test.mjs @@ -13,12 +13,12 @@ import { const claude = { id: 'vm-01', platform: 'anthropic' } const codex = { id: 'vm-02', platform: 'openai', family: 'codex' } -test('all is global and ignores a stored VM list', () => { +test('all is global and ignores a stored VM list', async () => { const scope = keyScopeFromRecord({ group_type: 'all', allowed_vms: '["vm-01"]' }) assert.deepEqual(scope, { group_type: 'all', allowed_vms: [], vm_pool_id: null }) assert.equal(keyAllowsVm(scope, claude), true) assert.equal(keyAllowsVm(scope, codex), true) - assert.deepEqual(keyScopeFromRequest({ apiKeyKind: 'master', apiKeyRecord: { group_type: 'openai' } }), { + assert.deepEqual(await keyScopeFromRequest({ apiKeyKind: 'master', apiKeyRecord: { group_type: 'openai' } }), { group_type: 'all', allowed_vms: [], vm_pool_id: null, @@ -52,17 +52,17 @@ test('a status-only patch does not widen or clear an existing scope', () => { }) }) -test('a named pool is the only schedule set and a dead pool does not widen', () => { +test('a named pool is the only schedule set and a dead pool does not widen', async () => { const bound = normalizeKeyScope({ vm_pool_id: 'pool_abcd1234', group_type: 'all' }) assert.deepEqual(bound, { group_type: 'all', allowed_vms: [], vm_pool_id: 'pool_abcd1234' }) - const live = applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-02'] })) + const live = await applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-02'] })) assert.equal(keyAllowsVm(live, codex), true) assert.equal(keyAllowsVm(live, claude), false) - const moved = applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-01'] })) + const moved = await applyVmPool(bound, () => ({ id: 'pool_abcd1234', enabled: true, vm_ids: ['vm-01'] })) assert.equal(keyAllowsVm(moved, codex), false) assert.equal(keyAllowsVm(moved, claude), true) for (const vm_pool_error of ['missing', 'disabled', 'empty']) { - const dead = applyVmPool(bound, () => + const dead = await applyVmPool(bound, () => vm_pool_error === 'missing' ? null : { enabled: vm_pool_error !== 'disabled', vm_ids: [] }, ) assert.equal(dead.vm_pool_error, vm_pool_error) diff --git a/test/unit/local-egress-per-node.test.mjs b/test/unit/local-egress-per-node.test.mjs index 5d3d8b9fe..c7a3e44ed 100644 --- a/test/unit/local-egress-per-node.test.mjs +++ b/test/unit/local-egress-per-node.test.mjs @@ -17,9 +17,9 @@ import { const NODES = { 'vm-ctrl': null, 'vm-a1': 'node-a', 'vm-a2': 'node-a', 'vm-b1': 'node-b' } -function makePool(t, opts = {}) { +async function makePool(t, opts = {}) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-local-node-')) - const pool = new ProxyPool({ dataDir: dir, vmNodeOf: (vmId) => NODES[vmId] ?? null, ...opts }) + const pool = await new ProxyPool({ dataDir: dir, vmNodeOf: (vmId) => NODES[vmId] ?? null, ...opts }).init() pool.stopScheduler() t.after(() => { pool.stopScheduler() @@ -43,8 +43,8 @@ test('local egress ids name their VPS and share one bridge per host', () => { assert.deepEqual(portsForProxy('px-local-node-a'), portsForProxy('px-local')) }) -test('a generic px-local bind lands on the slot’s own VPS row, with its own capacity', (t) => { - const pool = makePool(t) +test('a generic px-local bind lands on the slot’s own VPS row, with its own capacity', async (t) => { + const pool = await makePool(t) pool.ensureLocal() pool.updateConfig({ bind_limit: 1 }) assert.equal(pool.bind('px-local', 'vm-ctrl').proxy.id, 'px-local') @@ -57,8 +57,8 @@ test('a generic px-local bind lands on the slot’s own VPS row, with its own ca assert.equal(pool.getProxyForVm('vm-a1').id, 'px-local-node-a') }) -test('another machine’s local row is refused', (t) => { - const pool = makePool(t) +test('another machine’s local row is refused', async (t) => { + const pool = await makePool(t) pool.ensureLocal() pool.ensureLocalForNode('node-a') const wrong = pool.bind('px-local-node-a', 'vm-b1') @@ -68,16 +68,16 @@ test('another machine’s local row is refused', (t) => { assert.equal(pool.bind('px-local-node-a', 'vm-ctrl').error, 'local_proxy_node_mismatch') }) -test('auto-allocation only offers the slot’s own local row', (t) => { - const pool = makePool(t) +test('auto-allocation only offers the slot’s own local row', async (t) => { + const pool = await makePool(t) pool.ensureLocal() const picked = pool.allocateForVm('vm-b1') assert.equal(picked.id, 'px-local-node-b') assert.equal(pool.allocateForVm('vm-ctrl').id, 'px-local') }) -test('node rows follow the cluster and keep rows that still hold slots', (t) => { - const pool = makePool(t) +test('node rows follow the cluster and keep rows that still hold slots', async (t) => { + const pool = await makePool(t) // No local egress in use: nothing is created for nodes. assert.deepEqual(pool.syncLocalNodes(['node-a']), { created: [], removed: [] }) pool.ensureLocal() @@ -90,8 +90,8 @@ test('node rows follow the cluster and keep rows that still hold slots', (t) => assert.ok(pool.state.proxies.slice(0, 2).every((p) => isLocalEgressProxy(p))) }) -test('slots left on the shared row move to their node row, past a full limit', (t) => { - const pool = makePool(t) +test('slots left on the shared row move to their node row, past a full limit', async (t) => { + const pool = await makePool(t) pool.ensureLocal() pool.updateConfig({ bind_limit: 1 }) // Pre-split data: every local slot sits on px-local. @@ -108,9 +108,10 @@ test('slots left on the shared row move to their node row, past a full limit', ( assert.deepEqual(pool.adoptNodeLocalBindings(), []) }) -test('migration leaves ids of deleted slots alone', (t) => { - const pool = makePool(t, { vmNodeOf: (vmId) => (vmId === 'vm-gone' ? undefined : (NODES[vmId] ?? null)) }) +test('migration leaves ids of deleted slots alone', async (t) => { + const pool = await makePool(t, { vmNodeOf: (vmId) => (vmId === 'vm-gone' ? undefined : (NODES[vmId] ?? null)) }) pool.ensureLocal() + await pool.flush() pool.state.proxies.find((p) => p.id === 'px-local').bound_vm_ids = ['vm-gone', 'vm-a1'] assert.deepEqual( pool.adoptNodeLocalBindings().map((m) => m.vm_id), @@ -121,7 +122,7 @@ test('migration leaves ids of deleted slots alone', (t) => { test('a node row probes its node link; a down link marks it failed without parking slots', async (t) => { let healthy = true const disabled = [] - const pool = makePool(t, { + const pool = await makePool(t, { localExitHealth: (nodeId) => healthy ? { ok: true, latency_ms: 42 } : { ok: false, error: `node_link_down:${nodeId}` }, onDisableVm: (vmId) => disabled.push(vmId), @@ -144,7 +145,7 @@ test('a node row probes its node link; a down link marks it failed without parki test('a node link still dialling leaves the row alone until it settles', async (t) => { let health = { pending: true, error: 'node_link_connecting' } - const pool = makePool(t, { localExitHealth: () => health }) + const pool = await makePool(t, { localExitHealth: () => health }) pool.ensureLocal() pool.bind('px-local', 'vm-a1') const row = pool.state.proxies.find((p) => p.id === 'px-local-node-a') @@ -164,7 +165,7 @@ test('a node link still dialling leaves the row alone until it settles', async ( test('a node row looks up its geo through that node’s route', async (t) => { const seen = [] const geo = { ok: true, geo: { ip: '203.0.113.7', country: 'Japan', timezone: 'Asia/Tokyo' } } - const pool = makePool(t, { + const pool = await makePool(t, { localExitUrl: (nodeId) => `socks5h://${nodeId}@127.0.0.1:1080`, geoLookup: async (url) => { seen.push(url) @@ -199,7 +200,7 @@ test('tearing down a node row never touches the control plane bridge', () => { test('an operator-disabled node row stays disabled through link changes', async (t) => { let health = { ok: false, error: 'node_link_backoff' } - const pool = makePool(t, { localExitHealth: () => health }) + const pool = await makePool(t, { localExitHealth: () => health }) pool.ensureLocal() pool.bind('px-local', 'vm-a1') const row = pool.state.proxies.find((p) => p.id === 'px-local-node-a') @@ -213,9 +214,10 @@ test('an operator-disabled node row stays disabled through link changes', async assert.equal(row.status, 'dead') }) -test('migration persists once for the whole batch', (t) => { - const pool = makePool(t) +test('migration persists once for the whole batch', async (t) => { + const pool = await makePool(t) pool.ensureLocal() + await pool.flush() pool.state.proxies.find((p) => p.id === 'px-local').bound_vm_ids = ['vm-a1', 'vm-a2', 'vm-b1'] let writes = 0 const replaceAll = pool.repo.replaceAll.bind(pool.repo) @@ -224,11 +226,12 @@ test('migration persists once for the whole batch', (t) => { return replaceAll(rows) } assert.equal(pool.adoptNodeLocalBindings().length, 3) + await pool.flush() assert.equal(writes, 1) }) -test('resolveLocalTarget lands a generic pick on the target VPS', (t) => { - const pool = makePool(t) +test('resolveLocalTarget lands a generic pick on the target VPS', async (t) => { + const pool = await makePool(t) pool.ensureLocal() assert.deepEqual(pool.resolveLocalTarget('px-local', 'node-a'), { ok: true, id: 'px-local-node-a' }) assert.ok(pool.state.proxies.some((p) => p.id === 'px-local-node-a')) diff --git a/test/unit/model-policy-fable51.test.mjs b/test/unit/model-policy-fable51.test.mjs index af6a7c342..e7b06f9c0 100644 --- a/test/unit/model-policy-fable51.test.mjs +++ b/test/unit/model-policy-fable51.test.mjs @@ -82,15 +82,15 @@ test('explicit canonical overrides win over legacy fields in either insertion or } }) -test('loading a persisted disabled legacy model keeps both request spellings disabled', () => { - openDatabase({ dbPath: ':memory:' }) +test('loading a persisted disabled legacy model keeps both request spellings disabled', async () => { + await openDatabase({ dbPath: ':memory:' }) try { - new SettingsRepo().set('model_policy', { + await new SettingsRepo().set('model_policy', { catalog_mode: 'worker_only', models: { [legacy]: { enabled: false } }, aliases: { private: legacy }, }) - loadModelPolicy() + await loadModelPolicy() clearModelsCache() const ids = getPolicyCatalogIds() assert.ok(ids.includes(canonical)) @@ -101,11 +101,11 @@ test('loading a persisted disabled legacy model keeps both request spellings dis assert.equal(result.error.code, 'model_disabled', model) } assert.deepEqual(filterPublicModelIds([legacy, canonical]), [canonical]) - syncWorkerModelsIntoPolicy([legacy]) + await syncWorkerModelsIntoPolicy([legacy]) assert.ok(!getPolicyCatalogIds().includes(legacy)) } finally { - closeDatabase() - loadModelPolicy() + await closeDatabase() + await loadModelPolicy() clearModelsCache() } }) diff --git a/test/unit/onboarding.test.mjs b/test/unit/onboarding.test.mjs index 00a90edd5..f492edaf8 100644 --- a/test/unit/onboarding.test.mjs +++ b/test/unit/onboarding.test.mjs @@ -102,14 +102,14 @@ test('guide pops until skip or finish, even when setup steps are already done', assert.equal(legacy.guide_used, false) }) -test('PUT complete / dismiss / reset round-trip through settings; invalid action rejected', () => { +test('PUT complete / dismiss / reset round-trip through settings; invalid action rejected', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-onboarding-')) - const db = createDatabase({ dataDir: dir }) + const db = await createDatabase({ dataDir: dir }) const settings = new SettingsRepo(db) - assert.equal(recordOf(loadOnboardingState(settings), 'admin').status, 'pending') + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').status, 'pending') - saveOnboardingState(settings, applyOnboardingAction(loadOnboardingState(settings), 'admin', 'complete')) - const completed = recordOf(loadOnboardingState(settings), 'admin') + await saveOnboardingState(settings, applyOnboardingAction(await loadOnboardingState(settings), 'admin', 'complete')) + const completed = recordOf(await loadOnboardingState(settings), 'admin') assert.equal(completed.status, 'completed') assert.equal(completed.guide_used, true) assert.ok(completed.updated_at) @@ -121,17 +121,17 @@ test('PUT complete / dismiss / reset round-trip through settings; invalid action }) assert.equal(afterComplete.needs_welcome, false) - saveOnboardingState(settings, applyOnboardingAction(loadOnboardingState(settings), 'admin', 'dismiss')) - assert.equal(recordOf(loadOnboardingState(settings), 'admin').status, 'dismissed') - assert.equal(recordOf(loadOnboardingState(settings), 'admin').guide_used, true) + await saveOnboardingState(settings, applyOnboardingAction(await loadOnboardingState(settings), 'admin', 'dismiss')) + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').status, 'dismissed') + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').guide_used, true) const legacy = recordOf({ admin: { status: 'completed', updated_at: '2026-01-01T00:00:00.000Z' } }, 'admin') assert.equal(legacy.guide_used, false) assert.equal(buildOnboardingPayload({ user: 'admin', role: 'admin', ...legacy }).needs_welcome, true) - assert.equal(recordOf(loadOnboardingState(settings), 'bob').status, 'pending') + assert.equal(recordOf(await loadOnboardingState(settings), 'bob').status, 'pending') - saveOnboardingState(settings, applyOnboardingAction(loadOnboardingState(settings), 'admin', 'reset')) - assert.equal(recordOf(loadOnboardingState(settings), 'admin').status, 'pending') - assert.equal(loadOnboardingState(settings).admin, undefined) + await saveOnboardingState(settings, applyOnboardingAction(await loadOnboardingState(settings), 'admin', 'reset')) + assert.equal(recordOf(await loadOnboardingState(settings), 'admin').status, 'pending') + assert.equal((await loadOnboardingState(settings)).admin, undefined) assert.throws( () => applyOnboardingAction({}, 'admin', 'skip'), @@ -159,48 +159,48 @@ test('only VMs visible to the user drive slot and schedulable steps', () => { assert.equal(steps('u2').find((s) => s.id === 'schedulable').done, true) }) -test('first_request EXISTS uses the same ownerPred as request-log stats', () => { +test('first_request EXISTS uses the same ownerPred as request-log stats', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-onboarding-logs-')) - const db = createDatabase({ dbPath: path.join(dir, 'logs.db') }) - db.exec(` + const db = await createDatabase({ dbPath: path.join(dir, 'logs.db') }) + await db.exec(` INSERT INTO users (id, email, username, password_hash) VALUES ('u1', 'a@x', 'alice', 'x'), ('u2', 'b@x', 'bob', 'x'); INSERT INTO api_keys (id, name, key, user_id) VALUES ('k1', 'alice-key', 'sk-1', 'u1'), ('k2', 'bob-key', 'sk-2', 'u2'); `) const repo = new UsageLogsRepo(db) - repo.insertSummary({ + await repo.insertSummary({ id: 'log_fail', request_id: 'req-fail', created_at: '2026-01-01T00:00:00.000Z', status: 401, user_id: 'u1', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) - repo.insertSummary({ + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) + await repo.insertSummary({ id: 'log_other', request_id: 'req-other', created_at: '2026-01-01T00:00:01.000Z', status: 200, user_id: 'u2', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u2' }), true) - repo.insertSummary({ + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), false) + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u2' }), true) + await repo.insertSummary({ id: 'log_ok', request_id: 'req-ok', created_at: '2026-01-01T00:00:02.000Z', status: 200, user_id: 'u1', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), true) - assert.equal(repo.hasSuccessfulRequest({}), true) - repo.insertSummary({ + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u1' }), true) + assert.equal(await repo.hasSuccessfulRequest({}), true) + await repo.insertSummary({ id: 'log_redirect', request_id: 'req-3xx', created_at: '2026-01-01T00:00:03.000Z', status: 301, user_id: 'u3', }) - assert.equal(repo.hasSuccessfulRequest({ owner_user_id: 'u3' }), false) + assert.equal(await repo.hasSuccessfulRequest({ owner_user_id: 'u3' }), false) }) test('master API key identity is __master__ / admin; base_url strips trailing slash', () => { diff --git a/test/unit/opus-55-policy.test.mjs b/test/unit/opus-55-policy.test.mjs index edcda7196..86dcbecab 100644 --- a/test/unit/opus-55-policy.test.mjs +++ b/test/unit/opus-55-policy.test.mjs @@ -48,17 +48,17 @@ test('a saved dotted Opus 5.5 row keeps administrator overrides on the hyphen id assert.equal(migrated.aliases[legacy], canonical) }) -test('loading settings that predate Opus 5.5 appends the model without resetting Opus 5', () => { - openDatabase({ dbPath: ':memory:' }) +test('loading settings that predate Opus 5.5 appends the model without resetting Opus 5', async () => { + await openDatabase({ dbPath: ':memory:' }) try { const repo = new SettingsRepo() - repo.set('model_policy', { + await repo.set('model_policy', { source: 'panel', models: { 'claude-opus-5': { enabled: false, display_name: 'kept' } }, aliases: { opus: 'claude-opus-5' }, }) - loadModelPolicy({ force: true }) - const stored = repo.get('model_policy') + await loadModelPolicy({ force: true }) + const stored = await repo.get('model_policy') assert.equal(stored.models['claude-opus-5'].enabled, false) assert.equal(stored.models['claude-opus-5'].display_name, 'kept') assert.equal(stored.models[canonical].display_name, 'Opus 5.5') @@ -72,7 +72,7 @@ test('loading settings that predate Opus 5.5 appends the model without resetting assert.deepEqual(thinking.thinking, { type: 'adaptive' }) assert.equal(thinking.thinking.budget_tokens, undefined) } finally { - closeDatabase() + await closeDatabase() clearModelsCache() } }) diff --git a/test/unit/panel-api-proxy.test.mjs b/test/unit/panel-api-proxy.test.mjs index f37780ae7..594c92187 100644 --- a/test/unit/panel-api-proxy.test.mjs +++ b/test/unit/panel-api-proxy.test.mjs @@ -29,7 +29,7 @@ function writeVm(project, rec) { fs.writeFileSync(path.join(dir, `${rec.id}.json`), JSON.stringify(rec, null, 2)) } -test('summarizeProxyPool includes disconnect_on_error', () => { +test('summarizeProxyPool includes disconnect_on_error', async () => { assert.deepEqual(summarizeProxyPool(null), { total: 0, free: 0, @@ -40,7 +40,7 @@ test('summarizeProxyPool includes disconnect_on_error', () => { disconnect_on_error: false, }) const dir = tmpDir() - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.importLines('10.0.0.9:1080') pool.updateConfig({ disconnect_on_error: true }) pool.stopScheduler() @@ -63,7 +63,7 @@ test('dashboard and vm detail merge pool health onto bound VM', async () => { claude: {}, policy: { maxConcurrency: 4, weight: 1, priority: 8 }, }) - const pool = new ProxyPool({ dataDir }) + const pool = await new ProxyPool({ dataDir }).init() pool.importLines('10.8.8.8:1080') const px = pool.snapshot().proxies[0] pool.bind(px.id, 'vm-02') @@ -157,7 +157,7 @@ test('dead bound proxy blocks credential import', async () => { claude: {}, policy: {}, }) - const pool = new ProxyPool({ dataDir }) + const pool = await new ProxyPool({ dataDir }).init() pool.importLines('10.7.7.7:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-03') @@ -358,10 +358,10 @@ test('local proxy node labels are present for admin and absent for tenant', asyn { id: 'vm-tokyo', node_id: 'node-a' }, { id: 'vm-tenant', node_id: 'node-a', owner_user_id: 'user-1' }, ] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir, vmNodeOf: (vmId) => vms.find((vm) => vm.id === vmId)?.node_id || null, - }) + }).init() t.after(() => pool.stopScheduler()) pool.ensureLocal() // A generic px-local bind lands each slot on its own VPS's local row. diff --git a/test/unit/panel-persistence.test.mjs b/test/unit/panel-persistence.test.mjs index b21a8a95f..64221fcee 100644 --- a/test/unit/panel-persistence.test.mjs +++ b/test/unit/panel-persistence.test.mjs @@ -12,18 +12,27 @@ function setup({ failBefore = false } = {}) { json: (_res, status, body) => sent.push({ status, body }), readBody: async () => ({ enabled: false }), requireAuth: () => true, - clusterRoutes: { handle: (_req, res, _url, respond) => { - changed += 1 - return respond(res, 201, { ok: true }) - } }, - onPersistenceFailure: () => { maintenance += 1 }, + clusterRoutes: { + handle: (_req, res, _url, respond) => { + changed += 1 + return respond(res, 201, { ok: true }) + }, + }, + onPersistenceFailure: () => { + maintenance += 1 + }, proxyPool: { - writes: { flush: async () => { - writes += 1 - if (failBefore || writes > 1) throw new Error('synthetic persistence failure') - } }, + writes: { + flush: async () => { + writes += 1 + if (failBefore || writes > 1) throw new Error('synthetic persistence failure') + }, + }, snapshot: () => ({ config: {}, proxies: [] }), - updateConfig: () => { changed += 1; return { ok: true, config: { enabled: false } } }, + updateConfig: () => { + changed += 1 + return { ok: true, config: { enabled: false } } + }, }, }) return { handler, sent, changed: () => changed, maintenance: () => maintenance } @@ -61,10 +70,16 @@ test('a staged panel response remains handled after persistence succeeds', async const handler = createPanelHandler({ cfg: {}, requireAuth: () => true, - json: (_res, status, body) => { sent.push({ status, body }) }, + json: (_res, status, body) => { + sent.push({ status, body }) + }, }) - const handled = await handler({ method: 'POST', apiKeyKind: 'managed' }, {}, new URL('http://test/admin/models/refresh')) + const handled = await handler( + { method: 'POST', apiKeyKind: 'managed' }, + {}, + new URL('http://test/admin/models/refresh'), + ) assert.equal(handled, true) assert.equal(sent.length, 1) assert.equal(sent[0].status, 403) diff --git a/test/unit/panel-tenant-guards.test.mjs b/test/unit/panel-tenant-guards.test.mjs index 3e418fe45..f684d2992 100644 --- a/test/unit/panel-tenant-guards.test.mjs +++ b/test/unit/panel-tenant-guards.test.mjs @@ -140,17 +140,17 @@ test('tenant proxy import cannot bind a slot it does not own', async (t) => { ) }) -function keyFixture(t) { +async function keyFixture(t) { const root = makeProject(t) claudeVm(root, 'vm-mine', 'u-1') claudeVm(root, 'vm-other', 'u-2') const store = new ApiKeyStore({ dataDir: fs.mkdtempSync(path.join(root, 'keys-')) }) - const key = store.create({ name: 'mine', user_id: 'u-1', group_id: 1 }) + const key = await store.create({ name: 'mine', user_id: 'u-1', group_id: 1 }) return { root, store, key } } test('tenant key PATCH drops user_id and group_id', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const out = await makeHandler( root, TENANT, @@ -158,7 +158,7 @@ test('tenant key PATCH drops user_id and group_id', async (t) => { { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(out.status, 200, JSON.stringify(out.body)) - const saved = store.getById(key.id) + const saved = await store.getById(key.id) assert.equal(saved.name, 'renamed') assert.equal(saved.user_id, 'u-1') assert.equal(saved.group_id, 1) @@ -168,11 +168,11 @@ test('tenant key PATCH drops user_id and group_id', async (t) => { `/api/panel/api-keys/${key.id}`, ) assert.equal(cleared.status, 200) - assert.equal(store.getById(key.id).user_id, 'u-1') + assert.equal((await store.getById(key.id)).user_id, 'u-1') }) test('tenant key scope only accepts its own slots, on PATCH and create', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const foreign = { group_type: 'anthropic', allowed_vms: ['vm-other'] } const missing = { group_type: 'anthropic', allowed_vms: ['vm-nope'] } @@ -187,7 +187,7 @@ test('tenant key scope only accepts its own slots, on PATCH and create', async ( `/api/panel/api-keys/${key.id}`, ) assert.deepEqual(patchForeign.body, patchMissing.body) - assert.equal(store.getById(key.id).group_type, 'all') + assert.equal((await store.getById(key.id)).group_type, 'all') const patchOwn = await makeHandler( root, @@ -196,7 +196,7 @@ test('tenant key scope only accepts its own slots, on PATCH and create', async ( { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(patchOwn.status, 200, JSON.stringify(patchOwn.body)) - assert.deepEqual(parseAllowedVms(store.getById(key.id).allowed_vms), ['vm-mine']) + assert.deepEqual(parseAllowedVms((await store.getById(key.id)).allowed_vms), ['vm-mine']) const before = store.list().length const createForeign = await makeHandler(root, TENANT, { name: 'x', ...foreign }, { apiKeyStore: store }).call( @@ -214,13 +214,13 @@ test('tenant key scope only accepts its own slots, on PATCH and create', async ( { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(created.status, 201, JSON.stringify(created.body)) - const rec = store.getById(created.body.item.id) + const rec = await store.getById(created.body.item.id) assert.equal(rec.user_id, 'u-1') assert.equal(rec.group_id, 1) }) test('admin key edits keep user_id, group_id and any slot', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const out = await makeHandler( root, ADMIN, @@ -228,7 +228,7 @@ test('admin key edits keep user_id, group_id and any slot', async (t) => { { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(out.status, 200, JSON.stringify(out.body)) - const saved = store.getById(key.id) + const saved = await store.getById(key.id) assert.equal(saved.user_id, 'u-2') assert.equal(saved.group_id, 7) assert.deepEqual(parseAllowedVms(saved.allowed_vms), ['vm-other']) @@ -240,15 +240,15 @@ test('admin key edits keep user_id, group_id and any slot', async (t) => { { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(created.status, 201, JSON.stringify(created.body)) - const rec = store.getById(created.body.item.id) + const rec = await store.getById(created.body.item.id) assert.equal(rec.user_id, 'u-2') assert.equal(rec.group_id, 7) }) test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) => { - const { root, store, key } = keyFixture(t) + const { root, store, key } = await keyFixture(t) const poolRepo = new VmPoolsRepo(store.db) - const pool = createVmPool(poolRepo, { name: 'Shared Pool', vm_ids: [] }) + const pool = await createVmPool(poolRepo, { name: 'Shared Pool', vm_ids: [] }) // 1. Tenant POST /api/panel/api-keys with non-empty vm_pool_id -> 403 key_pool_forbidden const countBefore = store.list().length @@ -273,7 +273,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(tenantCreateEmpty.status, 201, JSON.stringify(tenantCreateEmpty.body)) - const emptyKey = store.getById(tenantCreateEmpty.body.item.id) + const emptyKey = await store.getById(tenantCreateEmpty.body.item.id) assert.equal(emptyKey.vm_pool_id, null) const tenantCreateNull = await makeHandler( @@ -283,7 +283,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(tenantCreateNull.status, 201, JSON.stringify(tenantCreateNull.body)) - const nullKey = store.getById(tenantCreateNull.body.item.id) + const nullKey = await store.getById(tenantCreateNull.body.item.id) assert.equal(nullKey.vm_pool_id, null) // 3. Admin POST with vm_pool_id -> 201 success @@ -294,7 +294,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('POST', '/api/panel/api-keys') assert.equal(adminCreate.status, 201, JSON.stringify(adminCreate.body)) - const adminKey = store.getById(adminCreate.body.item.id) + const adminKey = await store.getById(adminCreate.body.item.id) assert.equal(adminKey.vm_pool_id, pool.id) // 4. Admin assigns pool to tenant key @@ -303,7 +303,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = `/api/panel/api-keys/${key.id}`, ) assert.equal(adminPatchPool.status, 200, JSON.stringify(adminPatchPool.body)) - assert.equal(store.getById(key.id).vm_pool_id, pool.id) + assert.equal((await store.getById(key.id)).vm_pool_id, pool.id) // 5. Tenant PATCH with non-empty vm_pool_id -> 403 key_pool_forbidden const tenantPatchWithPool = await makeHandler( @@ -317,7 +317,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = ok: false, error: { code: 'key_pool_forbidden', message: '只有管理员可以设置账号池' }, }) - assert.equal(store.getById(key.id).vm_pool_id, pool.id, 'retained existing pool') + assert.equal((await store.getById(key.id)).vm_pool_id, pool.id, 'retained existing pool') // 6. Tenant PATCH with null or empty string vm_pool_id -> retains existing pool without error const tenantPatchNull = await makeHandler( @@ -327,7 +327,7 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = { apiKeyStore: store }, ).call('PATCH', `/api/panel/api-keys/${key.id}`) assert.equal(tenantPatchNull.status, 200, JSON.stringify(tenantPatchNull.body)) - const keyAfterTenantNull = store.getById(key.id) + const keyAfterTenantNull = await store.getById(key.id) assert.equal(keyAfterTenantNull.name, 'renamed-by-tenant') assert.equal(keyAfterTenantNull.vm_pool_id, pool.id, 'pool was not wiped by tenant') @@ -337,46 +337,46 @@ test('tenant key create/patch vm_pool_id guard; admin can set pool', async (t) = `/api/panel/api-keys/${key.id}`, ) assert.equal(tenantPatchOmit.status, 200, JSON.stringify(tenantPatchOmit.body)) - assert.equal(store.getById(key.id).vm_pool_id, pool.id) + assert.equal((await store.getById(key.id)).vm_pool_id, pool.id) }) -function usersFixture(t) { +async function usersFixture(t) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-tenant-guards-users-')) - const db = createDatabase({ dataDir: dir }) - t.after(() => { - db.close() + const db = await createDatabase({ dataDir: dir }) + t.after(async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) }) const panelUsers = new PanelUserStore({ db }) - panelUsers.create({ username: 'boss', password: 'boss-pass-1', role: 'admin' }) + await panelUsers.create({ username: 'boss', password: 'boss-pass-1', role: 'admin' }) return panelUsers } test('changing a user role or disabling it ends that user panel sessions', async (t) => { - const panelUsers = usersFixture(t) + const panelUsers = await usersFixture(t) const root = makeProject(t) for (const [i, patch] of [{ role: 'super' }, { enabled: false }].entries()) { - const user = panelUsers.create({ username: `tenant${i}`, password: 'tenant-pass', role: 'user' }) - const token = createPanelSession(user.username, { role: 'user' }) - const other = createPanelSession('boss', { role: 'admin' }) + const user = await panelUsers.create({ username: `tenant${i}`, password: 'tenant-pass', role: 'user' }) + const token = await createPanelSession(user.username, { role: 'user' }) + const other = await createPanelSession('boss', { role: 'admin' }) const out = await makeHandler(root, ADMIN, patch, { panelUsers }).call('PATCH', `/api/panel/users/${user.id}`) assert.equal(out.status, 200, JSON.stringify(out.body)) - assert.equal(verifyPanelSession(token), null, JSON.stringify(patch)) - assert.equal(verifyPanelSession(other)?.user, 'boss') + assert.equal(await verifyPanelSession(token), null, JSON.stringify(patch)) + assert.equal((await verifyPanelSession(other))?.user, 'boss') } }) test('unrelated user edits keep sessions alive', async (t) => { - const panelUsers = usersFixture(t) + const panelUsers = await usersFixture(t) const root = makeProject(t) - const user = panelUsers.create({ username: 'quota', password: 'tenant-pass', role: 'user' }) - const token = createPanelSession(user.username, { role: 'user' }) + const user = await panelUsers.create({ username: 'quota', password: 'tenant-pass', role: 'user' }) + const token = await createPanelSession(user.username, { role: 'user' }) const out = await makeHandler(root, ADMIN, { vm_create_quota: 3, role: 'user', enabled: true }, { panelUsers }).call( 'PATCH', `/api/panel/users/${user.id}`, ) assert.equal(out.status, 200, JSON.stringify(out.body)) - assert.equal(verifyPanelSession(token)?.user, 'quota') + assert.equal((await verifyPanelSession(token))?.user, 'quota') }) test('test-models is readable by user and super; other routes stay closed', () => { diff --git a/test/unit/panel-vm-create.test.mjs b/test/unit/panel-vm-create.test.mjs index 62e53a58a..0f2865893 100644 --- a/test/unit/panel-vm-create.test.mjs +++ b/test/unit/panel-vm-create.test.mjs @@ -248,7 +248,7 @@ test('auto-numbering skips ids a deleted VM left behind in account history', asy test('create binds an explicitly chosen exit instead of auto-allocating', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-create-exit-')) - const pool = new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }) + const pool = await new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }).init() pool.stopScheduler() try { const socks = pool.importLines('1.2.3.4:1080').items[0].id @@ -295,7 +295,7 @@ test('create stamps docker machine from routing defaults', async () => { test('create refuses an unknown or full exit before writing the slot', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-create-exit-bad-')) - const pool = new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }) + const pool = await new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }).init() pool.stopScheduler() try { const missing = makeCreateHandler(root, { id: 'vm-x', start: false, proxy_id: 'px-nope' }, pool) diff --git a/test/unit/panel-vm-delete.test.mjs b/test/unit/panel-vm-delete.test.mjs index 474129f86..48d604f90 100644 --- a/test/unit/panel-vm-delete.test.mjs +++ b/test/unit/panel-vm-delete.test.mjs @@ -95,13 +95,13 @@ test('startup reconcile does not resurrect a deleted sole active VM', async () = const { setVmWriteHook } = await import('../../src/lib/vm/vm-file.mjs') const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-del-db-')) try { - openDatabase({ dataDir: path.join(root, 'data') }) + await openDatabase({ dataDir: path.join(root, 'data') }) seed(root, 'vm-01', { active: true }) - initVmDbSync(root, { watch: false }) + await initVmDbSync(root, { watch: false }) const { handlePanel, response } = handler(root) await handlePanel({ method: 'DELETE' }, {}, new URL('http://localhost/api/panel/vms/vm-01')) assert.equal(response.status, 200, JSON.stringify(response.body)) - const again = reconcileVms(root) + const again = await reconcileVms(root) assert.equal(again.rebuilt, 0) assert.equal(again.active, null) assert.equal(fs.existsSync(path.join(root, 'vms', 'vm-01.json')), false) @@ -109,7 +109,7 @@ test('startup reconcile does not resurrect a deleted sole active VM', async () = } finally { stopVmWatch() setVmWriteHook(null) - closeDatabase() + await closeDatabase() fs.rmSync(root, { recursive: true, force: true }) } }) diff --git a/test/unit/panel-vm-exit-probe.test.mjs b/test/unit/panel-vm-exit-probe.test.mjs index 113969465..c6f66e643 100644 --- a/test/unit/panel-vm-exit-probe.test.mjs +++ b/test/unit/panel-vm-exit-probe.test.mjs @@ -12,10 +12,10 @@ function seed(root, vm) { fs.writeFileSync(path.join(dir, `${vm.id}.json`), JSON.stringify({ status: 'stopped', policy: {}, ...vm })) } -function setup(t) { +async function setup(t) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-exit-probe-')) const lookups = [] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: path.join(root, 'data'), vmNodeOf: (vmId) => JSON.parse(fs.readFileSync(path.join(root, 'vms', `${vmId}.json`), 'utf8')).node_id || null, localExitUrl: (nodeId) => `socks5h://${nodeId}@127.0.0.1:1080`, @@ -24,7 +24,7 @@ function setup(t) { return { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', city: 'Tokyo', timezone: 'Asia/Tokyo' } } }, geoV6Lookup: async () => ({ ok: false, error: 'no_v6' }), - }) + }).init() pool.stopScheduler() t.after(() => { pool.stopScheduler() @@ -52,7 +52,7 @@ function setup(t) { } test('an unbound slot has no exit to probe', async (t) => { - const { root, probe, lookups } = setup(t) + const { root, probe, lookups } = await setup(t) seed(root, { id: 'vm-free', proxy: null }) const res = await probe('vm-free') assert.equal(res.status, 502) @@ -61,7 +61,7 @@ test('an unbound slot has no exit to probe', async (t) => { }) test('a node slot on local egress probes its own node row and refreshes it', async (t) => { - const { root, pool, probe, lookups } = setup(t) + const { root, pool, probe, lookups } = await setup(t) seed(root, { id: 'vm-tokyo', node_id: 'node-a', proxy: { id: 'px-local', host: 'local', scheme: 'local' } }) pool.ensureLocal() assert.equal(pool.bind('px-local', 'vm-tokyo').proxy.id, 'px-local-node-a') @@ -76,7 +76,7 @@ test('a node slot on local egress probes its own node row and refreshes it', asy }) test('a control-plane Codex slot is probed through the deployment proxy', async (t) => { - const { root, pool, probe, lookups } = setup(t) + const { root, pool, probe, lookups } = await setup(t) const saved = process.env.HTTPS_PROXY process.env.HTTPS_PROXY = 'socks5://deploy.example:1080' t.after(() => { diff --git a/test/unit/pool-scheduler.test.mjs b/test/unit/pool-scheduler.test.mjs index 2cfc63ab8..1f9fad486 100644 --- a/test/unit/pool-scheduler.test.mjs +++ b/test/unit/pool-scheduler.test.mjs @@ -675,11 +675,11 @@ test('unpinned slot follows live tier concurrency after reloadConfig', async (t) vm.policy.maxConcurrency = 2 vm.policy.concurrencyOverride = false fs.writeFileSync(file, JSON.stringify(vm)) - const accountQuota = new AccountQuota({ + const accountQuota = await new AccountQuota({ dataDir: path.join(root, 'data'), config: { tiers: { max: { max_concurrency: 2, max_rpm: 0 } } }, accounts: [{ account_id: 'account-1', vm_id: 'vm-01', max_concurrency: 2 }], - }) + }).init() accountQuota.setAccountTier('account-1', 'max') const pool = scheduler(root, { accountQuota }) accountQuota.reloadConfig({ @@ -1333,7 +1333,7 @@ test('syncQuotaSchedule keeps Extra 5h exhausted schedulable and skips the slot' const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) q.ensure({ account_id: 'account-2', vm_id: 'vm-02' }) const reset = new Date(Date.now() + 4 * 3600_000).toISOString() @@ -1369,7 +1369,7 @@ test('syncQuotaSchedule auto-picks Extra 5h again after the window opens', async const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) const reset = new Date(Date.now() + 4 * 3600_000).toISOString() q.ingestHeaders('account-1', { @@ -1411,7 +1411,7 @@ test('syncQuotaSchedule does not turn a manual-off slot back on', async (t) => { const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) q.ensure({ account_id: 'account-2', vm_id: 'vm-02' }) q.ingestHeaders('account-1', { @@ -1443,11 +1443,11 @@ test('syncQuotaSchedule does not turn a manual-off slot back on', async (t) => { picked.release() }) -test('leftover quota 调度关 is restored to restricted without flipping the operator off', (t) => { +test('leftover quota 调度关 is restored to restricted without flipping the operator off', async (t) => { const root = project() let pool t.after(() => cleanupProject(root, pool)) - const q = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const q = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() q.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) q.ingestHeaders('account-1', { 'anthropic-ratelimit-unified-5h-utilization': '1', @@ -1539,14 +1539,14 @@ test('platform scope skips tenant-owned VMs', async (t) => { test('deleting a tenant makes its assigned spare selectable while the primary quota is exhausted', async (t) => { const root = project() - const quota = new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }) + const quota = await new AccountQuota({ dataDir: path.join(root, 'data'), config: {} }).init() let pool - t.after(() => { - quota.db.close() + t.after(async () => { + await quota.db.close() cleanupProject(root, pool) }) const users = new PanelUserStore({ db: quota.db, projectRoot: root }) - const user = users.create({ username: 'tenant', password: 'tenant-pass' }) + const user = await users.create({ username: 'tenant', password: 'tenant-pass' }) persistVmOwner(root, 'vm-02', { ownerUserId: user.id, origin: 'admin_assigned' }) quota.ensure({ account_id: 'account-1', vm_id: 'vm-01' }) quota.ensure({ account_id: 'account-2', vm_id: 'vm-02' }) @@ -1559,7 +1559,7 @@ test('deleting a tenant makes its assigned spare selectable while the primary qu const exhausted = await pool.selectAndReserve({ model: 'claude-test', allowWait: false }) assert.equal(exhausted.ok, false) - assert.equal(users.remove(user.id), true) + assert.equal(await users.remove(user.id), true) const recovered = await pool.selectAndReserve({ model: 'claude-test', allowWait: false }) assert.equal(recovered.ok, true) assert.equal(recovered.vmId, 'vm-02') @@ -2067,10 +2067,13 @@ test('#A08: a refused reservation hands back exactly what it took; release is id test('#A15: a hard account exit moves every pinned conversation but not live leases', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2206,10 +2209,13 @@ function verifiedHop(text) { test('family without pin schedules parent and child on one VM', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2305,10 +2311,13 @@ test('family without pin schedules parent and child on one VM', async (t) => { test('family whose VM is gated moves as a whole and releases its sticky pins', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2363,10 +2372,13 @@ test('family whose VM is gated moves as a whole and releases its sticky pins', a test('family VM left only for kernel slot_busy keeps the family pin', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2438,10 +2450,13 @@ function applyRebuildAttempt(callerSession, extra = {}) { test('rebuild outbound session stays on one VM and changes after failover', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2488,10 +2503,13 @@ test('rebuild outbound session stays on one VM and changes after failover', asyn test('consecutive turns without an inbound session keep one outbound session', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2521,10 +2539,13 @@ test('consecutive turns without an inbound session keep one outbound session', a test('family migration remints parent and child outbound sessions', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2604,10 +2625,13 @@ test('family migration remints parent and child outbound sessions', async (t) => test('intra-request failover remints and persists the second outbound session', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) @@ -2652,10 +2676,13 @@ test('intra-request failover remints and persists the second outbound session', test('returning to a previously used VM mints a new outbound session', async (t) => { const root = project() - const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) - t.after(() => { + const sticky = await new StickyRouter({ + dataDir: path.join(root, 'data'), + config: { sticky: { enabled: true } }, + }).init() + t.after(async () => { try { - sticky.db?.close?.() + await sticky.db?.close?.() } catch {} fs.rmSync(root, { recursive: true, force: true }) }) diff --git a/test/unit/price-catalog.test.mjs b/test/unit/price-catalog.test.mjs index 8848c8d34..e9740dbc4 100644 --- a/test/unit/price-catalog.test.mjs +++ b/test/unit/price-catalog.test.mjs @@ -94,20 +94,20 @@ test('layer precedence: override provider/model > provider/* > */model > models. assert.equal(fromDev.source, 'models.dev') assert.equal(fromDev.price.cache_read, 9) - assert.equal(setPriceOverride(`*/${id}`, { ...FULL, cache_read: 3 }).ok, true) + assert.equal((await setPriceOverride(`*/${id}`, { ...FULL, cache_read: 3 })).ok, true) assert.equal(resolvePrice(id).source, 'override') assert.equal(resolvePrice(id).price.cache_read, 3) - assert.equal(setPriceOverride('anthropic/*', { ...FULL, cache_read: 2 }).ok, true) + assert.equal((await setPriceOverride('anthropic/*', { ...FULL, cache_read: 2 })).ok, true) assert.equal(resolvePrice(id).price.cache_read, 2) - assert.equal(setPriceOverride(`anthropic/${id}`, { ...FULL, cache_read: 1 }).ok, true) + assert.equal((await setPriceOverride(`anthropic/${id}`, { ...FULL, cache_read: 1 })).ok, true) assert.equal(resolvePrice(id).price.cache_read, 1) assert.equal(resolvePrice(id).source, 'override') }) -test('a provider/* override on a dated id keeps the canonical model builtin fast mode', () => { - assert.equal(setPriceOverride('anthropic/*', FULL).ok, true) +test('a provider/* override on a dated id keeps the canonical model builtin fast mode', async () => { + assert.equal((await setPriceOverride('anthropic/*', FULL)).ok, true) const hit = resolvePrice('claude-opus-5-5-20261001') assert.equal(hit.id, 'claude-opus-5-5') assert.equal(hit.source, 'override') @@ -123,19 +123,19 @@ test('an override on the canonical id also prices its dated snapshot', async () jsonResponse({ anthropic: { models: { [dated]: { cost: { input: 1, output: 5, cache_read: 0.1 } } } } }), }) assert.equal(resolvePrice(dated).source, 'models.dev') - assert.equal(setPriceOverride('anthropic/claude-haiku-4-5', FULL).ok, true) + assert.equal((await setPriceOverride('anthropic/claude-haiku-4-5', FULL)).ok, true) const hit = resolvePrice(dated) assert.equal(hit.source, 'override') assert.equal(hit.price.input, FULL.input) }) -test('partial override is rejected', () => { - const missingRead = setPriceOverride('anthropic/claude-sonnet-5-5', { input: 1, output: 2 }) +test('partial override is rejected', async () => { + const missingRead = await setPriceOverride('anthropic/claude-sonnet-5-5', { input: 1, output: 2 }) assert.equal(missingRead.ok, false) assert.match(missingRead.error, /cache_read/) assert.equal(resolvePrice('claude-sonnet-5-5').source, 'builtin') - const missingInput = setPriceOverride('openai/gpt-5.5', { output: 1, cache_read: 0 }) + const missingInput = await setPriceOverride('openai/gpt-5.5', { output: 1, cache_read: 0 }) assert.equal(missingInput.ok, false) assert.match(missingInput.error, /input/) }) @@ -259,10 +259,10 @@ test('validatePrice fills optional cache writes', () => { assert.equal(o.price.cache_1h, 0) }) -test('saving an override invalidates the memoized catalog', () => { +test('saving an override invalidates the memoized catalog', async () => { const before = listPriceCatalog().find((e) => e.id === 'claude-sonnet-5-5') assert.equal(before.override, null) - assert.equal(setPriceOverride('anthropic/claude-sonnet-5-5', FULL).ok, true) + assert.equal((await setPriceOverride('anthropic/claude-sonnet-5-5', FULL)).ok, true) const after = listPriceCatalog().find((e) => e.id === 'claude-sonnet-5-5') assert.equal(after.source, 'override') assert.equal(after.override.input, 9) diff --git a/test/unit/probe-status.test.mjs b/test/unit/probe-status.test.mjs index 0d1228473..24cc7a0a8 100644 --- a/test/unit/probe-status.test.mjs +++ b/test/unit/probe-status.test.mjs @@ -16,7 +16,7 @@ function usageCache(probe) { } } -function fixture(t, { mode = 'setup-token', headers = true, failed = false } = {}) { +async function fixture(t, { mode = 'setup-token', headers = true, failed = false } = {}) { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'probe-card-')) fs.mkdirSync(path.join(project, 'vms')) fs.writeFileSync( @@ -29,8 +29,8 @@ function fixture(t, { mode = 'setup-token', headers = true, failed = false } = { }), ) const dataDir = path.join(project, 'data') - let db = createDatabase({ dataDir }) - let quota = new AccountQuota({ db }) + let db = await createDatabase({ dataDir }) + let quota = await new AccountQuota({ db }).init() const acc = quota.ensure({ account_id: 'account-test', vm_id: 'vm-02', max_concurrency: 20 }) acc.unified.headers = headers ? { '5h': { utilization: 0.15, status: 'allowed' }, sampled_at: '2026-09-20T00:00:00Z' } @@ -45,18 +45,18 @@ function fixture(t, { mode = 'setup-token', headers = true, failed = false } = { acc.unified.usage_rate_limited_until = '2099-01-01T00:00:00Z' } quota.repo.save(acc) - t.after(() => { - db.close() + t.after(async () => { + await db.close() fs.rmSync(project, { recursive: true, force: true }) }) const args = { cfg: { paths: { project }, rewrite: {} }, id: 'vm-02', force: true, hop: true } return { quota, args, - reopen() { - db.close() - db = createDatabase({ dataDir }) - quota = new AccountQuota({ db }) + async reopen() { + await db.close() + db = await createDatabase({ dataDir }) + quota = await new AccountQuota({ db }).init() return quota }, async detail(q = quota) { @@ -66,7 +66,7 @@ function fixture(t, { mode = 'setup-token', headers = true, failed = false } = { } test('Setup Token probe persists the card check time/source across DB reopen and detail reload', async (t) => { - const f = fixture(t) + const f = await fixture(t) const probe = { ok: true, source: 'official-cc-usage', @@ -77,7 +77,7 @@ test('Setup Token probe persists the card check time/source across DB reopen and const result = (await buildProbeOne({ ...f.args, accountQuota: f.quota, usageCache: usageCache(probe) })).data assert.equal(result.ok, true) assert.equal(result.source, 'official-cc-usage') - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.account.last_probe_check?.at, result.probed_at) assert.equal(detail.account.last_probe_check.source, result.source) assert.equal(detail.account.last_probe_check.via, 'worker') @@ -86,7 +86,7 @@ test('Setup Token probe persists the card check time/source across DB reopen and }) test('cached-header check preserves active-probe failures, backoff, quota and counters', async (t) => { - const f = fixture(t, { failed: true }) + const f = await fixture(t, { failed: true }) const before = f.quota.repo.get('account-test') await buildProbeOne({ ...f.args, @@ -109,7 +109,7 @@ test('cached-header check preserves active-probe failures, backoff, quota and co }) test('no header samples reports unavailable and persists the attempt without fake success', async (t) => { - const f = fixture(t, { headers: false }) + const f = await fixture(t, { headers: false }) const probe = { ok: false, source: 'official-cc-usage', @@ -125,13 +125,13 @@ test('no header samples reports unavailable and persists the attempt without fak assert.equal(check.ok, false) assert.equal(check.data_at, result.probed_at) assert.equal(f.quota.repo.get('account-test').unified.last_probe?.ok, false) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.account_tier, 'pro') assert.equal(detail.vm.usage_has_fable, null) }) test('Setup Token Fable hop does not classify the plan without complete official usage', async (t) => { - const f = fixture(t) + const f = await fixture(t) f.quota.setAccountTier('account-test', 'pro') const result = ( await buildProbeOne({ @@ -148,12 +148,12 @@ test('Setup Token Fable hop does not classify the plan without complete official assert.equal(f.quota.repo.get('account-test').unified.usage_has_fable, undefined) assert.equal(f.quota.repo.get('account-test').unified.fable.ok, false) assert.equal(f.quota.repo.get('account-test').unified.fable.plan_denied, false) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.account_tier, 'pro') }) test('Setup Token usage scope failure reports auth scope and preserves history', async (t) => { - const f = fixture(t) + const f = await fixture(t) f.quota.setAccountTier('account-test', 'max', { source: 'usage' }) let fableCalled = false const result = ( @@ -189,7 +189,7 @@ test('Setup Token usage scope failure reports auth scope and preserves history', }) test('official OAuth probe still ingests real usage and exposes its actual result to the card', async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) const probe = { ok: false, source: 'official-cc-usage', @@ -210,7 +210,7 @@ test('official OAuth probe still ingests real usage and exposes its actual resul }) ).data assert.equal(result.ok, false) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.account.last_probe_check?.at, probe.probed_at) assert.equal(detail.account.last_probe_check.ok, false) assert.equal(detail.account.last_probe_check.error, 'probe_failed') @@ -227,7 +227,7 @@ test('probe errors remain text across the response, persisted checks and detail ] for (const { expected, ...errors } of cases) { await t.test(JSON.stringify(errors), async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) const result = ( await buildProbeOne({ ...f.args, @@ -244,7 +244,7 @@ test('probe errors remain text across the response, persisted checks and detail ).data assert.equal(result.ok, false) assert.equal(result.error, expected) - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) for (const record of [detail.vm, detail.account]) { assert.equal(record.last_probe.ok, false) assert.equal(record.last_probe.error, expected) @@ -256,7 +256,7 @@ test('probe errors remain text across the response, persisted checks and detail }) test('confirmed usage revoke parks the VM and reports revoked status', async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) const result = ( await buildProbeOne({ ...f.args, @@ -275,13 +275,13 @@ test('confirmed usage revoke parks the VM and reports revoked status', async (t) const vm = JSON.parse(fs.readFileSync(path.join(f.args.cfg.paths.project, 'vms/vm-02.json'), 'utf8')) assert.equal(vm.schedulable, false) assert.equal(vm.schedule_disabled_reason, 'oauth_revoked') - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.cred_status.key, 'bad') assert.equal(detail.vm.cred_status.text, 'revoke') }) test('complete Pro usage replaces stale Max windows in persisted panel data', async (t) => { - const f = fixture(t, { mode: 'oauth' }) + const f = await fixture(t, { mode: 'oauth' }) f.quota.ingestOAuthUsage('account-test', { ok: true, usage_status: 200, @@ -307,7 +307,7 @@ test('complete Pro usage replaces stale Max windows in persisted panel data', as ).data assert.equal(result.account_tier, 'pro') assert.equal(result.quota.account_tier, 'pro') - const detail = await f.detail(f.reopen()) + const detail = await f.detail(await f.reopen()) assert.equal(detail.vm.account_tier, 'pro') assert.equal(detail.vm.usage_has_fable, false) assert.equal(detail.vm.utilization_7d_oi, null) diff --git a/test/unit/proxy-cli-enabled.test.mjs b/test/unit/proxy-cli-enabled.test.mjs index 164f675b8..42906a772 100644 --- a/test/unit/proxy-cli-enabled.test.mjs +++ b/test/unit/proxy-cli-enabled.test.mjs @@ -166,11 +166,11 @@ test('OAuth commit after bind does not reset proxy_cli_enabled', async () => { test('package import with a proxy keeps proxy_cli_enabled even if the spec says false', async (t) => { const root = tmpRoot() t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const db = createDatabase({ dbPath: ':memory:' }) - const pool = new ProxyPool({ db }) - t.after(() => { + const db = await createDatabase({ dbPath: ':memory:' }) + const pool = await new ProxyPool({ db }).init() + t.after(async () => { pool.stopScheduler() - db.close() + await db.close() }) const doc = buildVmPackage({ vm: { diff --git a/test/unit/proxy-geo.test.mjs b/test/unit/proxy-geo.test.mjs index 32ac8c0e3..e6fe1353f 100644 --- a/test/unit/proxy-geo.test.mjs +++ b/test/unit/proxy-geo.test.mjs @@ -18,8 +18,8 @@ function tmpDir(prefix = 'kin-geo-') { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) } -function makePool(geoLookup) { - const pool = new ProxyPool({ dataDir: tmpDir(), geoLookup }) +async function makePool(geoLookup) { + const pool = await new ProxyPool({ dataDir: tmpDir(), geoLookup }).init() pool.stopScheduler() return pool } @@ -138,7 +138,7 @@ test('geo lookup surfaces transport and HTTP failures instead of throwing', asyn test('detectGeo caches per proxy and publishes location on the snapshot', async () => { let calls = 0 - const pool = makePool(async () => { + const pool = await makePool(async () => { calls += 1 return { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', country_code: 'JP', timezone: 'Asia/Tokyo' } } }) @@ -170,7 +170,7 @@ test('detectGeo caches per proxy and publishes location on the snapshot', async }) test('detectGeo keeps IPv4 and IPv6 results separate and records IPv6 unavailability', async () => { - const pool = makePool(async () => ({ + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) @@ -184,7 +184,7 @@ test('detectGeo keeps IPv4 and IPv6 results separate and records IPv6 unavailabi }) test('detectGeo rejects IPv4 masquerading as IPv6 on the v6 path', async () => { - const pool = makePool(async () => ({ + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) @@ -199,7 +199,7 @@ test('detectGeo rejects IPv4 masquerading as IPv6 on the v6 path', async () => { test('a failed lookup keeps the last known location and records the error', async () => { let ok = true - const pool = makePool(async () => + const pool = await makePool(async () => ok ? { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' } } : { ok: false, error: 'geo_http_500' }, @@ -216,7 +216,7 @@ test('a failed lookup keeps the last known location and records the error', asyn }) test('binding a proxy moves the slot onto the exit node timezone', async () => { - const pool = makePool(async () => ({ + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) @@ -238,7 +238,7 @@ test('binding a proxy moves the slot onto the exit node timezone', async () => { }) test('a hand-pinned slot timezone survives a proxy bind unless forced', async () => { - const pool = makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) + const pool = await makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) const id = pool.importLines('1.2.3.4:1080').items[0].id pool.bind(id, 'vm-02') const root = seedProject({ @@ -259,7 +259,7 @@ test('a hand-pinned slot timezone survives a proxy bind unless forced', async () }) test('follow_proxy_timezone=false stops the bind-time default', async () => { - const pool = makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) + const pool = await makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) pool.updateConfig({ follow_proxy_timezone: false }) // updateConfig restarts the probe loop; left running it keeps the test // process alive forever. @@ -275,7 +275,7 @@ test('follow_proxy_timezone=false stops the bind-time default', async () => { }) test('an unresolvable exit zone leaves the slot timezone alone', async () => { - const pool = makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: null } })) + const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: null } })) const id = pool.importLines('1.2.3.4:1080').items[0].id pool.bind(id, 'vm-04') const root = seedProject({ id: 'vm-04', timezone: 'America/Denver', proxy: { id } }) @@ -287,7 +287,7 @@ test('an unresolvable exit zone leaves the slot timezone alone', async () => { test('a node slot on local egress follows its node exit, not the control plane row', async (t) => { const lookups = [] - const pool = makePool(async (url) => { + const pool = await makePool(async (url) => { lookups.push(url) // '' = the control plane's own route (px-local row); a SOCKS URL = the node forwarder. return { ok: true, geo: { timezone: url ? 'Europe/Berlin' : 'Asia/Tokyo' } } diff --git a/test/unit/proxy-ipv6.test.mjs b/test/unit/proxy-ipv6.test.mjs index fb5565c99..cabeb9cf8 100644 --- a/test/unit/proxy-ipv6.test.mjs +++ b/test/unit/proxy-ipv6.test.mjs @@ -10,18 +10,18 @@ import { resolveImportProxy } from '../../src/lib/vm/proxy-resolve.mjs' import { evaluateSlotGate } from '../../src/lib/pool/schedule-eligibility.mjs' import { isCodexSlotReady, pickCodexSlots } from '../../src/lib/pool/codex-slot-pool.mjs' -function makePool(t) { - const pool = new ProxyPool({ db: openDatabase({ dbPath: ':memory:' }) }) +async function makePool(t) { + const pool = await new ProxyPool({ db: await openDatabase({ dbPath: ':memory:' }) }).init() pool.updateConfig({ enabled: false }) - t.after(() => { + t.after(async () => { pool.stopScheduler() - closeDatabase() + await closeDatabase() }) return pool } -test('IPv6 aliases normalize and deduplicate after loading bracketed historical hosts', (t) => { - const pool = makePool(t) +test('IPv6 aliases normalize and deduplicate after loading bracketed historical hosts', async (t) => { + const pool = await makePool(t) const first = pool.importLines('socks5h://a%3Ab:p%40%3A%2F%25@[2001:0db8:0:0:0:0:0:1]:1080') const id = first.items[0].id const full = pool.getProxyByIdWithAuth(id) @@ -30,8 +30,9 @@ test('IPv6 aliases normalize and deduplicate after loading bracketed historical assert.equal(full.password, 'p@:/%') assert.equal(full.url, 'socks5://a%3Ab:p%40%3A%2F%25@[2001:db8::1]:1080') assert.equal(parseSocks5Line(full.url).password, full.password) - pool.db.prepare('UPDATE proxies SET host = ? WHERE id = ?').run('[2001:0db8::1]', id) - pool.reload() + await pool.flush() + await pool.db.prepare('UPDATE proxies SET host = $1 WHERE id = $2').run('[2001:0db8::1]', id) + await pool.reload() assert.equal(pool.state.proxies[0].host, '2001:db8::1') const result = pool.importLines('', { fields: [{ host: '2001:db8::1', port: 1080, username: 'a:b', password: 'p@:/%' }], @@ -59,7 +60,7 @@ test('bracketed IPv6 input works without changing hostname vendor credentials', }) test('malformed historical IPv6 hosts cannot silently probe localhost', async (t) => { - const pool = makePool(t) + const pool = await makePool(t) let connections = 0 const sockets = new Set() const server = net.createServer((socket) => { @@ -77,15 +78,16 @@ test('malformed historical IPv6 hosts cannot silently probe localhost', async (t const { items: [proxy], } = pool.importLines(`127.0.0.1:${server.address().port}`) - pool.db.prepare('UPDATE proxies SET host = ? WHERE id = ?').run('[::1', proxy.id) - pool.reload() + await pool.flush() + await pool.db.prepare('UPDATE proxies SET host = $1 WHERE id = $2').run('[::1', proxy.id) + await pool.reload() const result = await pool.probeById(proxy.id) assert.equal(result.probe.error, 'invalid_proxy_host') assert.equal(connections, 0) }) test('IPv6 policy survives reload and preserves binding and health instead of reallocating', async (t) => { - const pool = makePool(t) + const pool = await makePool(t) const { items: [ipv6], } = pool.importLines('[2001:db8::1]:1080\n127.0.0.1:1080') @@ -99,7 +101,7 @@ test('IPv6 policy survives reload and preserves binding and health instead of re Object.assign(proxy, { status: 'ok', latency_ms: 20, last_probe_at: '2026-10-01T00:00:00Z', consecutive_failures: 2 }) const history = { status: proxy.status, last_probe_at: proxy.last_probe_at, failures: proxy.consecutive_failures } pool.updateConfig({ ipv6_enabled: false, disconnect_on_error: true }) - pool.reload() + await pool.reload() assert.equal(pool.snapshot().config.ipv6_enabled, false) assert.equal(pool.getProxyForVm('vm-01'), null) assert.equal(pool.ensureBoundToVm('vm-01', ipv6.id), null) @@ -128,13 +130,13 @@ test('IPv6 policy survives reload and preserves binding and health instead of re assert.equal(isCodexSlotReady(codex), false) assert.equal(pickCodexSlots([codex], { pin: codex.id }).error, 'ipv6_disabled') pool.updateConfig({ ipv6_enabled: true }) - pool.reload() + await pool.reload() assert.equal(pool.getProxyForVm('vm-01').id, ipv6.id) assert.equal(boundProxyUrl(vm.proxy), 'socks5h://[2001:db8::1]:1080') }) test('URL-imported IPv6 probes connect only when enabled and closing cancels in-flight sockets', async (t) => { - const pool = makePool(t) + const pool = await makePool(t) const sockets = new Set() let connections = 0 const server = net.createServer((socket) => { diff --git a/test/unit/proxy-pool-bind.test.mjs b/test/unit/proxy-pool-bind.test.mjs index 3afb27ba5..3cdc6db97 100644 --- a/test/unit/proxy-pool-bind.test.mjs +++ b/test/unit/proxy-pool-bind.test.mjs @@ -10,16 +10,16 @@ function tmpDir(prefix = 'kin-px-') { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) } -function makePool() { - const pool = new ProxyPool({ dataDir: tmpDir() }) +async function makePool() { + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() return pool } -test('proxy pool persists custom dns_primary and rejects invalid URLs without changing it', (t) => { +test('proxy pool persists custom dns_primary and rejects invalid URLs without changing it', async (t) => { const dir = tmpDir() t.after(() => fs.rmSync(dir, { recursive: true, force: true })) - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.stopScheduler() assert.equal(pool.updateConfig({ dns_primary: '8.8.8.8:53' }).ok, true) const primary = 'https://cloudflare-dns.com:8443/dns-query?key=a%2Cb' @@ -37,15 +37,15 @@ test('proxy pool persists custom dns_primary and rejects invalid URLs without ch assert.equal(result.error, 'invalid_dns_primary', invalid) assert.equal(pool.snapshot().config.dns_primary, primary) } - const reopened = new ProxyPool({ dataDir: dir }) + const reopened = await new ProxyPool({ dataDir: dir }).init() reopened.stopScheduler() assert.equal(reopened.snapshot().config.dns_primary, primary) }) -test('DNS type suppression persists both toggle states and rejects non-booleans', (t) => { +test('DNS type suppression persists both toggle states and rejects non-booleans', async (t) => { const dir = tmpDir() t.after(() => fs.rmSync(dir, { recursive: true, force: true })) - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.stopScheduler() assert.equal(pool.snapshot().config.dns_disable_svcb_https, false) assert.equal(pool.updateConfig({ dns_disable_svcb_https: true }).ok, true) @@ -55,11 +55,11 @@ test('DNS type suppression persists both toggle states and rejects non-booleans' assert.equal(pool.snapshot().config.dns_disable_svcb_https, true) assert.equal(pool.snapshot().config.dns_primary, 'auto') } - const reopened = new ProxyPool({ dataDir: dir }) + const reopened = await new ProxyPool({ dataDir: dir }).init() reopened.stopScheduler() assert.equal(reopened.snapshot().config.dns_disable_svcb_https, true) assert.equal(reopened.updateConfig({ dns_disable_svcb_https: false }).ok, true) - const disabled = new ProxyPool({ dataDir: dir }) + const disabled = await new ProxyPool({ dataDir: dir }).init() disabled.stopScheduler() assert.equal(disabled.snapshot().config.dns_disable_svcb_https, false) }) @@ -74,8 +74,8 @@ test('clampBindLimit defaults to 5 and stays in 1..32', () => { assert.equal(clampBindLimit('x'), MAX_VMS_PER_PROXY) }) -test('bind_limit defaults to 5 and is configurable', () => { - const pool = makePool() +test('bind_limit defaults to 5 and is configurable', async () => { + const pool = await makePool() pool.importLines('10.0.0.1:1080') const id = pool.snapshot().proxies[0].id assert.equal(pool.bindLimit(), 5) @@ -106,8 +106,8 @@ test('bind_limit defaults to 5 and is configurable', () => { assert.equal(pool.bind(id, 'vm-03').max, 2) }) -test('remove deletes a proxy and updateConfig rejects bad bind_limit', () => { - const pool = makePool() +test('remove deletes a proxy and updateConfig rejects bad bind_limit', async () => { + const pool = await makePool() pool.importLines('10.0.0.2:1080') const id = pool.snapshot().proxies[0].id const removed = pool.remove(id) @@ -131,10 +131,10 @@ async function listenSocks() { test('probeOne marks egress_down after SOCKS greeting if helper is dead', async () => { const socks = await listenSocks() - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: false, reason: 'not_running' }), - }) + }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -150,13 +150,13 @@ test('probeOne marks egress_down after SOCKS greeting if helper is dead', async test('probeOne repairs egress then passes', async () => { const socks = await listenSocks() let healthy = false - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: healthy }), repairEgress: () => { healthy = true }, - }) + }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -173,14 +173,14 @@ test('startScheduler immediately repairs enabled egress', async () => { const socks = await listenSocks() let healthy = false let repairs = 0 - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: healthy, reason: healthy ? null : 'not_running' }), repairEgress: () => { repairs += 1 healthy = true }, - }) + }).init() try { pool.importLines(`${socks.host}:${socks.port}`) pool.startScheduler() @@ -196,10 +196,10 @@ test('startScheduler immediately repairs enabled egress', async () => { test('egress_down does not mark a live SOCKS proxy dead', async () => { const socks = await listenSocks() - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: false, reason: 'not_running' }), - }) + }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -220,7 +220,7 @@ test('TCP connect then close is not a SOCKS death', async () => { const server = net.createServer((socket) => socket.destroy()) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const addr = server.address() - const pool = new ProxyPool({ dataDir: tmpDir() }) + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() try { pool.importLines(`${addr.address}:${addr.port}`) @@ -234,7 +234,7 @@ test('TCP connect then close is not a SOCKS death', async () => { test('probeAll resurrects a probe-killed proxy', async () => { const socks = await listenSocks() - const pool = new ProxyPool({ dataDir: tmpDir() }) + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() try { pool.importLines(`${socks.host}:${socks.port}`) @@ -253,8 +253,8 @@ test('probeAll resurrects a probe-killed proxy', async () => { } }) -test('ensureLocal adds a single local egress row', () => { - const pool = makePool() +test('ensureLocal adds a single local egress row', async () => { + const pool = await makePool() const first = pool.ensureLocal() const second = pool.ensureLocal() assert.equal(first.ok, true) @@ -272,11 +272,11 @@ test('ensureLocal adds a single local egress row', () => { test('local egress probe stays direct when kin-egress is down', async () => { const disabled = [] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: tmpDir(), egressCheck: () => ({ ok: false, reason: 'local_network_missing' }), onDisableVm: (vmId) => disabled.push(vmId), - }) + }).init() pool.stopScheduler() pool.ensureLocal() pool.bind('px-local', 'vm-01') diff --git a/test/unit/proxy-pool-import.test.mjs b/test/unit/proxy-pool-import.test.mjs index 57750094b..fb6d6c0f3 100644 --- a/test/unit/proxy-pool-import.test.mjs +++ b/test/unit/proxy-pool-import.test.mjs @@ -3,25 +3,25 @@ import assert from 'node:assert/strict' import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' import { createDatabase } from '../../src/lib/db/database.mjs' -function makePool(t) { - const db = createDatabase({ dbPath: ':memory:' }) - const pool = new ProxyPool({ db }) - t.after(() => { +async function makePool(t) { + const db = await createDatabase({ dbPath: ':memory:' }) + const pool = await new ProxyPool({ db }).init() + t.after(async () => { pool.stopScheduler() - db.close() + await db.close() }) return pool } -test('import: different passwords at the same endpoint are distinct within a batch and after reload', (t) => { - const pool = makePool(t) +test('import: different passwords at the same endpoint are distinct within a batch and after reload', async (t) => { + const pool = await makePool(t) const first = pool.importLines('1.2.3.4:1080:alice:first\n1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:first') assert.equal(first.added, 2) assert.equal(first.skipped, 1) assert.equal(first.skip_details[0].reason, 'duplicate') assert.notEqual(first.items[0].id, first.items[1].id) - pool.reload() + await pool.reload() assert.deepEqual(pool.state.proxies.map((p) => p.password).sort(), ['first', 'second']) const next = pool.importLines('1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:third') assert.equal(next.added, 1) @@ -29,8 +29,8 @@ test('import: different passwords at the same endpoint are distinct within a bat assert.equal(pool.state.proxies.length, 3) }) -test('import: structured credentials containing colons do not collide, equivalent URLs still deduplicate', (t) => { - const pool = makePool(t) +test('import: structured credentials containing colons do not collide, equivalent URLs still deduplicate', async (t) => { + const pool = await makePool(t) const result = pool.importLines('', { fields: [ { host: '1.2.3.4', port: 1080, username: 'alice:region', password: 'secret' }, diff --git a/test/unit/proxy-pool-update.test.mjs b/test/unit/proxy-pool-update.test.mjs index 2e762ec64..a0cc5579c 100644 --- a/test/unit/proxy-pool-update.test.mjs +++ b/test/unit/proxy-pool-update.test.mjs @@ -5,8 +5,8 @@ import os from 'node:os' import path from 'node:path' import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' -function makePool() { - const pool = new ProxyPool({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-px-upd-')) }) +async function makePool() { + const pool = await new ProxyPool({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-px-upd-')) }).init() pool.stopScheduler() return pool } @@ -15,8 +15,8 @@ function seed(pool, line = 'socks5://alice:s3cret@1.2.3.4:1080') { return pool.state.proxies[0] } -test('update: 只改 port，账密保留；raw 明文密码被清除', () => { - const pool = makePool() +test('update: 只改 port，账密保留；raw 明文密码被清除', async () => { + const pool = await makePool() const p = seed(pool) assert.match(p.raw, /s3cret/, '前提：导入后 raw 含明文密码') const r = pool.update(p.id, { port: 1081 }) @@ -28,16 +28,16 @@ test('update: 只改 port，账密保留；raw 明文密码被清除', () => { assert.doesNotMatch(p.raw, /s3cret/, 'raw 不应再含明文密码') }) -test('update: username 传空串清除账密（password 一并清）', () => { - const pool = makePool() +test('update: username 传空串清除账密（password 一并清）', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.update(p.id, { username: '' }).ok, true) assert.equal(p.username, null) assert.equal(p.password, null) }) -test('update: 不传的字段保持不动', () => { - const pool = makePool() +test('update: 不传的字段保持不动', async () => { + const pool = await makePool() const p = seed(pool) pool.update(p.id, { password: 'newpw' }) assert.equal(p.host, '1.2.3.4') @@ -46,8 +46,8 @@ test('update: 不传的字段保持不动', () => { assert.equal(p.password, 'newpw') }) -test('update: 只给密码不给用户名被拒，不静默存成「仍无账密」', () => { - const pool = makePool() +test('update: 只给密码不给用户名被拒，不静默存成「仍无账密」', async () => { + const pool = await makePool() pool.importLines('9.9.9.9:1080') const p = pool.state.proxies[0] assert.equal(p.username, null, '前提：这条本来没有账密') @@ -57,16 +57,16 @@ test('update: 只给密码不给用户名被拒，不静默存成「仍无账密 assert.equal(pool.update(p.id, { username: '', password: 'x' }).error, 'password_without_username') }) -test('update: 只换用户名时沿用原密码（不传的键就是不动）', () => { - const pool = makePool() +test('update: 只换用户名时沿用原密码（不传的键就是不动）', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.update(p.id, { username: 'bob' }).ok, true) assert.equal(p.username, 'bob') assert.equal(p.password, 's3cret', 'password 没传，按语义保持原值') }) -test('update: 非法值与空 patch 被拒，且不改动原记录', () => { - const pool = makePool() +test('update: 非法值与空 patch 被拒，且不改动原记录', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.update(p.id, { port: 99999 }).error, 'invalid_proxy') assert.equal(pool.update(p.id, { host: '' }).error, 'invalid_proxy') @@ -75,8 +75,8 @@ test('update: 非法值与空 patch 被拒，且不改动原记录', () => { assert.equal(p.port, 1080, '被拒后原值不变') }) -test('update: 响应里的 publicProxy 不含账密', () => { - const pool = makePool() +test('update: 响应里的 publicProxy 不含账密', async () => { + const pool = await makePool() const p = seed(pool) const r = pool.update(p.id, { port: 1081 }) const s = JSON.stringify(r.proxy) @@ -84,8 +84,8 @@ test('update: 响应里的 publicProxy 不含账密', () => { assert.equal(r.proxy.has_auth, true, '但要能看出有账密') }) -test('getProxyByIdWithAuth: 拼出完整 URI，特殊字符转义', () => { - const pool = makePool() +test('getProxyByIdWithAuth: 拼出完整 URI，特殊字符转义', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://alice:s3cret@1.2.3.4:1080') pool.update(p.id, { username: 'u@x', password: 'p:w/d' }) @@ -94,8 +94,8 @@ test('getProxyByIdWithAuth: 拼出完整 URI，特殊字符转义', () => { assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://1.2.3.4:1080', '无账密时不带 @') }) -test('getProxyByIdWithAuth: 禁用/失效的代理仍可读回（getProxyForVm 做不到）', () => { - const pool = makePool() +test('getProxyByIdWithAuth: 禁用/失效的代理仍可读回（getProxyForVm 做不到）', async () => { + const pool = await makePool() const p = seed(pool) pool.bind(p.id, 'vm-01') assert.ok(pool.getProxyForVm('vm-01'), '前提：启用时 getProxyForVm 能取到') @@ -104,28 +104,28 @@ test('getProxyByIdWithAuth: 禁用/失效的代理仍可读回（getProxyForVm assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://alice:s3cret@1.2.3.4:1080', '新方法不受影响') }) -test('snapshot 回归：列表接口永不返回账密', () => { - const pool = makePool() +test('snapshot 回归：列表接口永不返回账密', async () => { + const pool = await makePool() seed(pool) const s = JSON.stringify(pool.snapshot()) assert.doesNotMatch(s, /alice|s3cret|socks5:\/\//, 'GET /proxies 的载荷不得含账密') }) -test('update: 只改代理名称不动连接字段，也不要求重载槽位；重开后名称仍在', () => { - const pool = makePool() +test('update: 只改代理名称不动连接字段，也不要求重载槽位；重开后名称仍在', async () => { + const pool = await makePool() const p = seed(pool) const r = pool.update(p.id, { label: ' 东京-2 号机 ' }) assert.equal(r.ok, true) assert.equal(r.connection_changed, false) assert.equal(r.proxy.label, '东京-2 号机') assert.equal(p.raw, 'socks5://alice:s3cret@1.2.3.4:1080', '只改名称不应改写 raw') - pool.load() + await pool.load() assert.equal(pool.state.proxies[0].label, '东京-2 号机') assert.equal(pool.snapshot().proxies[0].label, '东京-2 号机') }) -test('update: 代理名称传空串清除；超长或非字符串被拒', () => { - const pool = makePool() +test('update: 代理名称传空串清除；超长或非字符串被拒', async () => { + const pool = await makePool() const p = seed(pool) pool.update(p.id, { label: 'a' }) assert.equal(pool.update(p.id, { label: '' }).proxy.label, null) @@ -133,8 +133,8 @@ test('update: 代理名称传空串清除；超长或非字符串被拒', () => assert.equal(pool.update(p.id, { label: 123 }).error, 'invalid_label') }) -test('update: 连接字段与名称同改时报告 connection_changed', () => { - const pool = makePool() +test('update: 连接字段与名称同改时报告 connection_changed', async () => { + const pool = await makePool() const p = seed(pool) const r = pool.update(p.id, { port: 1082, label: 'box' }) assert.equal(r.connection_changed, true) @@ -142,8 +142,8 @@ test('update: 连接字段与名称同改时报告 connection_changed', () => { assert.equal(p.label, 'box') }) -test('labelOf 只给名称，不带账密；未知代理返回 null', () => { - const pool = makePool() +test('labelOf 只给名称，不带账密；未知代理返回 null', async () => { + const pool = await makePool() const p = seed(pool) assert.equal(pool.labelOf(p.id), null) pool.update(p.id, { label: 'tw-box' }) diff --git a/test/unit/proxy-resolve.test.mjs b/test/unit/proxy-resolve.test.mjs index a70b39682..a7888755f 100644 --- a/test/unit/proxy-resolve.test.mjs +++ b/test/unit/proxy-resolve.test.mjs @@ -12,8 +12,8 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-resolve-')) } -test('prefers pool credentials over host-only vm.proxy', () => { - const pool = new ProxyPool({ dataDir: tmpDir() }) +test('prefers pool credentials over host-only vm.proxy', async () => { + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.importLines('socks5://user:s3cret@10.0.0.5:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-02') @@ -26,8 +26,8 @@ test('prefers pool credentials over host-only vm.proxy', () => { assert.equal(resolved.proxyUrl, 'socks5h://user:s3cret@10.0.0.5:1080') }) -test('blocks import when bound proxy is fail or dead', () => { - const pool = new ProxyPool({ dataDir: tmpDir() }) +test('blocks import when bound proxy is fail or dead', async () => { + const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.importLines('10.0.0.6:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-03') diff --git a/test/unit/quota-critical-band.test.mjs b/test/unit/quota-critical-band.test.mjs index ae2a89988..3568d43ad 100644 --- a/test/unit/quota-critical-band.test.mjs +++ b/test/unit/quota-critical-band.test.mjs @@ -28,8 +28,8 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-band-')) } -function quota(extra = {}) { - return new AccountQuota({ +async function quota(extra = {}) { + return await new AccountQuota({ dataDir: tmpDir(), config: { quota: { @@ -48,7 +48,7 @@ function quota(extra = {}) { max: { limit_5h: 0.8, limit_7d: 0.8, max_concurrency: 2 }, }, }, - }) + }).init() } const smallRisk = { @@ -77,8 +77,8 @@ function setUtil(q, id, { u5 = null, u7 = null } = {}) { q.ingestHeaders(id, headers) } -test('79% admits a small request and rejects a single large cold with inflight=0', () => { - const q = quota() +test('79% admits a small request and rejects a single large cold with inflight=0', async () => { + const q = await quota() setUtil(q, 'a', { u5: 0.79 }) assert.equal(q.canAccept('a', { requestRisk: smallRisk }).ok, true) const large = q.canAccept('a', { requestRisk: largeColdRisk }) @@ -88,8 +88,8 @@ test('79% admits a small request and rejects a single large cold with inflight=0 assert.equal(q.inflight.get('a') || 0, 0) }) -test('5h and 7d bands are independent', () => { - const q = quota() +test('5h and 7d bands are independent', async () => { + const q = await quota() setUtil(q, 'five', { u5: 0.79, u7: 0.2 }) assert.equal(q.canAccept('five', { requestRisk: largeColdRisk }).reason, 'quota_guard_request_too_large') assert.equal(q.canAccept('five', { requestRisk: smallRisk }).ok, true) @@ -102,8 +102,8 @@ test('5h and 7d bands are independent', () => { assert.equal(q.canAccept('open', { requestRisk: largeColdRisk }).ok, true) }) -test('past the line plus overshoot still blocks ordinary requests', () => { - const q = quota() +test('past the line plus overshoot still blocks ordinary requests', async () => { + const q = await quota() setUtil(q, 'line', { u5: 0.8 }) assert.equal(q.canAccept('line', { requestRisk: smallRisk }).ok, true, 'at the line, inside the 5% overshoot') setUtil(q, 'full', { u5: 0.85 }) @@ -112,8 +112,8 @@ test('past the line plus overshoot still blocks ordinary requests', () => { assert.equal(q.canAccept('over').reason, 'quota_5h_safety') }) -test('failed or cancelled release does not leave a reservation or stale band debt', () => { - const q = quota() +test('failed or cancelled release does not leave a reservation or stale band debt', async () => { + const q = await quota() setUtil(q, 'rel', { u5: 0.79 }) assert.equal(q.tryAcquire('rel', { requestRisk: smallRisk }).ok, true) assert.equal(q.inflight.get('rel'), 1) @@ -123,8 +123,8 @@ test('failed or cancelled release does not leave a reservation or stale band deb assert.equal(q.canAccept('rel', { requestRisk: smallRisk }).ok, true) }) -test('stale reading after an in-band settle blocks the next large request', () => { - const q = quota() +test('stale reading after an in-band settle blocks the next large request', async () => { + const q = await quota() setUtil(q, 'stale', { u5: 0.79 }) assert.equal(q.tryAcquire('stale', { requestRisk: smallRisk }).ok, true) q.noteBandSettled('stale') @@ -135,8 +135,8 @@ test('stale reading after an in-band settle blocks the next large request', () = assert.equal(q.canAccept('stale', { requestRisk: smallRisk }).reason, 'quota_5h_safety') }) -test('a same-percent header after settle keeps the stale large-cold block', () => { - const q = quota() +test('a same-percent header after settle keeps the stale large-cold block', async () => { + const q = await quota() setUtil(q, 'lag', { u5: 0.78 }) assert.equal(q.tryAcquire('lag', { requestRisk: smallRisk }).ok, true) q.noteBandSettled('lag') @@ -145,8 +145,8 @@ test('a same-percent header after settle keeps the stale large-cold block', () = assert.equal(q.canAccept('lag', { requestRisk: largeColdRisk }).reason, 'quota_guard_request_too_large') }) -test('official usage refresh clears stale band debt', () => { - const q = quota() +test('official usage refresh clears stale band debt', async () => { + const q = await quota() setUtil(q, 'off', { u5: 0.79 }) assert.equal(q.tryAcquire('off', { requestRisk: smallRisk }).ok, true) q.noteBandSettled('off') @@ -207,7 +207,7 @@ class RuntimeRepo { test('scheduler switches using requestRisk on the quotaRequest body estimate', async (t) => { const root = schedulerProject() t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const accountQuota = quota() + const accountQuota = await quota() setUtil(accountQuota, 'account-1', { u5: 0.79 }) setUtil(accountQuota, 'account-2', { u5: 0.2 }) const orig = accountQuota.canAccept.bind(accountQuota) @@ -234,7 +234,7 @@ test('scheduler switches using requestRisk on the quotaRequest body estimate', a test('all in-band accounts yield quota_guard_request_too_large', async (t) => { const root = schedulerProject() t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const accountQuota = quota() + const accountQuota = await quota() setUtil(accountQuota, 'account-1', { u5: 0.79 }) setUtil(accountQuota, 'account-2', { u5: 0.77 }) const orig = accountQuota.canAccept.bind(accountQuota) @@ -314,7 +314,7 @@ test('token estimate counts a long prompt as large cold on first turn', () => { assert.equal(risk.largeCold, true) }) -test('output ceiling does not make a short prompt large in the critical band', () => { +test('output ceiling does not make a short prompt large in the critical band', async () => { clearCachePrefixSessions() const account = 'band' const session = 'sess-demo' @@ -358,7 +358,7 @@ test('output ceiling does not make a short prompt large in the critical band', ( assert.equal(omitted.largeCold, false) assert.ok(omitted.uncachedTokens < DEFAULT_LARGE_COLD_UNCACHED_TOKENS) - const q = quota() + const q = await quota() setUtil(q, account, { u5: 0.79 }) const warmReq = { body: turn(2), sessionId: session } assert.equal(q.canAccept(account, { quotaRequest: warmReq }).ok, true) @@ -381,7 +381,7 @@ test('output ceiling does not make a short prompt large in the critical band', ( ) }) -test('stale band still blocks a warm turn whose new user text is itself large', () => { +test('stale band still blocks a warm turn whose new user text is itself large', async () => { clearCachePrefixSessions() const base = { model: 'claude-opus-5-5', @@ -402,7 +402,7 @@ test('stale band still blocks a warm turn whose new user text is itself large', assert.equal(risk.cold, false) assert.equal(risk.largeCold, false) assert.ok(risk.uncachedTokens >= DEFAULT_LARGE_COLD_UNCACHED_TOKENS) - const q = quota() + const q = await quota() setUtil(q, 'band', { u5: 0.79 }) const quotaRequest = { body: extended, sessionId: 'sess' } assert.equal(q.canAccept('band', { quotaRequest }).ok, true) @@ -423,9 +423,9 @@ test('quota_guard_request_too_large stays a 429 and is not rewritten to pool_una assert.equal(rewritten.body.error.code, 'quota_guard_request_too_large') }) -test('prompt estimate is skipped below the band and walked once for in-band accounts', () => { +test('prompt estimate is skipped below the band and walked once for in-band accounts', async () => { resetPromptEstimateStats() - const q = quota() + const q = await quota() setUtil(q, 'low', { u5: 0.5, u7: 0.2 }) setUtil(q, 'band-a', { u5: 0.79 }) setUtil(q, 'band-b', { u5: 0.76 }) @@ -445,15 +445,15 @@ test('prompt estimate is skipped below the band and walked once for in-band acco q.release('band-a') }) -test('admission and the panel agree near the gate (vm-10: pro, 7d 90% of a 95% line)', () => { +test('admission and the panel agree near the gate (vm-10: pro, 7d 90% of a 95% line)', async () => { const tier = { limit_5h: 0.95, limit_7d: 0.95, max_concurrency: 2 } - const q = new AccountQuota({ + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true, critical_band: 0.05, gate_overshoot: 0.05 }, tiers: { default: tier, pro: tier, max: tier }, }, - }) + }).init() const panel = (u7) => { q.ingestHeaders('vm10', { 'anthropic-ratelimit-unified-5h-utilization': '0.12', diff --git a/test/unit/quota-tiers.test.mjs b/test/unit/quota-tiers.test.mjs index bf4a33147..84167be72 100644 --- a/test/unit/quota-tiers.test.mjs +++ b/test/unit/quota-tiers.test.mjs @@ -109,8 +109,8 @@ test('buildRouting exposes effective Codex routing to the panel', () => { assert.equal(data.codex.protocols['openai.responses'].enabled, true) }) -test('canAccept uses per-tier 5h and weekly lines', () => { - const q = new AccountQuota({ +test('canAccept uses per-tier 5h and weekly lines', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true, gate_overshoot: 0 }, @@ -120,7 +120,7 @@ test('canAccept uses per-tier 5h and weekly lines', () => { max: { max_concurrency: 4, safety_ratio: 0.95, weekly_safety_ratio: 0.95, warn_ratio: 0.85 }, }, }, - }) + }).init() q.ensure({ account_id: 'pro-1' }) q.setAccountTier('pro-1', 'pro') q.ingestHeaders('pro-1', { 'anthropic-ratelimit-unified-5h-utilization': '0.86' }) @@ -138,8 +138,8 @@ test('canAccept uses per-tier 5h and weekly lines', () => { assert.equal(q.canAccept('max-1').reason, 'quota_7d_safety') }) -test('reloadConfig picks up saved weekly line without process restart shape', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('reloadConfig picks up saved weekly line without process restart shape', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'p' }) q.setAccountTier('p', 'pro') // Pro default 7d line 0.80 + default 0.05 overshoot. @@ -152,15 +152,15 @@ test('reloadConfig picks up saved weekly line without process restart shape', () assert.equal(q.canAccept('p').ok, true) }) -test('applyTierConcurrency writes only the matching tier', () => { - const q = new AccountQuota({ +test('applyTierConcurrency writes only the matching tier', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { concurrency: { default_max_per_account: 20 } }, accounts: [ { account_id: 'a-pro', vm_id: 'vm-1', max_concurrency: 20 }, { account_id: 'a-max', vm_id: 'vm-2', max_concurrency: 20 }, ], - }) + }).init() q.setAccountTier('a-pro', 'pro') q.setAccountTier('a-max', 'max') q.applyTierConcurrency({ @@ -172,8 +172,8 @@ test('applyTierConcurrency writes only the matching tier', () => { assert.equal(q.repo.get('a-max').max_concurrency, 4) }) -test('applyTierRpm writes only the matching tier and skips override', () => { - const q = new AccountQuota({ +test('applyTierRpm writes only the matching tier and skips override', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {}, accounts: [ @@ -181,7 +181,7 @@ test('applyTierRpm writes only the matching tier and skips override', () => { { account_id: 'r-max', vm_id: 'vm-2', max_rpm: 0 }, { account_id: 'r-pin', vm_id: 'vm-3', max_rpm: 99 }, ], - }) + }).init() q.setAccountTier('r-pro', 'pro') q.setAccountTier('r-max', 'max') q.setMaxRpm('r-pin', 99, { override: true }) @@ -207,8 +207,8 @@ test('resolveTierPolicy reads live config', () => { assert.equal(p.weekly_safety_ratio, 0.7) }) -test('saving Pro 5h=80% lands on policyFor limit_5h', () => { - const q = new AccountQuota({ +test('saving Pro 5h=80% lands on policyFor limit_5h', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true, gate_overshoot: 0 }, @@ -216,7 +216,7 @@ test('saving Pro 5h=80% lands on policyFor limit_5h', () => { pro: { limit_5h: 0.8, limit_7d: 0.8, max_concurrency: 2 }, }), }, - }) + }).init() q.ensure({ account_id: 'pro-80' }) q.setAccountTier('pro-80', 'pro') assert.equal(q.policyFor(q.repo.get('pro-80')).limit_5h, 0.8) @@ -224,8 +224,8 @@ test('saving Pro 5h=80% lands on policyFor limit_5h', () => { assert.equal(q.canAccept('pro-80').reason, 'quota_5h_safety') }) -test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', () => { - const q = new AccountQuota({ +test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true }, @@ -233,7 +233,7 @@ test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', pro: { limit_5h: 0.95, limit_7d: 0.95, max_concurrency: 4 }, }), }, - }) + }).init() q.ensure({ account_id: 'pro-95' }) q.setAccountTier('pro-95', 'pro') q.ingestHeaders('pro-95', { @@ -256,14 +256,14 @@ test('Pro 95% runs one call at a time from 90% and blocks at line + overshoot', assert.equal(q.canAccept('pro-95').reason, 'quota_5h_safety') }) -test('reloadConfig applies conc and rpm without rewriting account rows', () => { - const q = new AccountQuota({ +test('reloadConfig applies conc and rpm without rewriting account rows', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: { quota: { block_on_5h: true, block_on_7d: true }, tiers: { default: { max_concurrency: 2, max_rpm: 0 } }, }, - }) + }).init() q.ensure({ account_id: 'live' }) const acc = q.repo.get('live') assert.equal(q.limitFor(acc), 2) @@ -276,8 +276,8 @@ test('reloadConfig applies conc and rpm without rewriting account rows', () => { assert.equal(q.rpmLimitFor(q.repo.get('live')), 12) }) -test('manual conc and rpm pins ignore live tier', () => { - const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) +test('manual conc and rpm pins ignore live tier', async () => { + const q = await new AccountQuota({ dataDir: tmpDir(), config: {} }).init() q.ensure({ account_id: 'pin' }) q.setMaxConcurrency('pin', 1, { override: true }) q.setMaxRpm('pin', 2, { override: true }) diff --git a/test/unit/rate-limit-service.test.mjs b/test/unit/rate-limit-service.test.mjs index 9b5ae05ee..f9a2b9992 100755 --- a/test/unit/rate-limit-service.test.mjs +++ b/test/unit/rate-limit-service.test.mjs @@ -4,15 +4,15 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { createDatabase } from '../../src/lib/db/database.mjs' -import { AccountRuntimeRepo } from '../../src/lib/db/repos/account-runtime-repo.mjs' +import { SchedulingRuntime as AccountRuntimeRepo } from '../../src/lib/pool/scheduling-runtime.mjs' import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' import { RateLimitService, anthropic429Reset, hardBlockOf } from '../../src/lib/pool/rate-limit-service.mjs' -function setup(config = {}) { +async function setup(config = {}) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-rate-limit-')) - const db = createDatabase({ dataDir: dir }) - const runtimeRepo = new AccountRuntimeRepo(db) - const accountQuota = new AccountQuota({ db, config: {} }) + const db = await createDatabase({ dataDir: dir }) + const runtimeRepo = await new AccountRuntimeRepo(db).init() + const accountQuota = await new AccountQuota({ db, config: {} }).init() accountQuota.attachRuntimeRepo(runtimeRepo) runtimeRepo.upsert({ account_id: 'acc-1', vm_id: 'vm-01', status: 'ready' }) const probes = [] @@ -22,8 +22,8 @@ function setup(config = {}) { config, onUsageProbe: (item) => probes.push(item), }) - const close = () => { - db.close() + const close = async () => { + await db.close() fs.rmSync(dir, { recursive: true, force: true }) } return { service, runtimeRepo, accountQuota, probes, close } @@ -41,8 +41,8 @@ function limitResult(message) { } } -test('plan limit writes rate_limit_reset_at from the text and survives a passive Extra re-read', () => { - const { service, runtimeRepo, accountQuota, probes, close } = setup() +test('plan limit writes rate_limit_reset_at from the text and survives a passive Extra re-read', async () => { + const { service, runtimeRepo, accountQuota, probes, close } = await setup() try { const now = Date.now() const block = service.handleUpstreamError({ @@ -64,12 +64,12 @@ test('plan limit writes rate_limit_reset_at from the text and survives a passive accountQuota.clearQuotaExhaustedCooldown('acc-1') assert.equal(runtimeRepo.get('acc-1').rate_limit_reset_at, state.rate_limit_reset_at) } finally { - close() + await close() } }) -test('plan limit without any reset falls back to the configured cooldown and asks for one /usage hop', () => { - const { service, runtimeRepo, probes, close } = setup({ fallback_cooldown_min: 30 }) +test('plan limit without any reset falls back to the configured cooldown and asks for one /usage hop', async () => { + const { service, runtimeRepo, probes, close } = await setup({ fallback_cooldown_min: 30 }) try { const now = Date.now() const block = service.handleUpstreamError({ @@ -84,12 +84,12 @@ test('plan limit without any reset falls back to the configured cooldown and ask assert.equal(Math.abs(runtimeRepo.get('acc-1').rate_limit_reset_at - block.until) < 1000, true) assert.deepEqual(probes, [{ accountId: 'acc-1', vmId: 'vm-01' }]) } finally { - close() + await close() } }) -test('model-scoped 429 does not write an account hard block', () => { - const { service, runtimeRepo, close } = setup() +test('model-scoped 429 does not write an account hard block', async () => { + const { service, runtimeRepo, close } = await setup() try { const block = service.handleUpstreamError({ accountId: 'acc-1', @@ -100,12 +100,12 @@ test('model-scoped 429 does not write an account hard block', () => { assert.equal(block, null) assert.equal(runtimeRepo.get('acc-1').rate_limit_reset_at, null) } finally { - close() + await close() } }) -test('#163: an unknown bare 429 asks for one usage probe and fabricates no 5h block', () => { - const { service, runtimeRepo, probes, close } = setup() +test('#163: an unknown bare 429 asks for one usage probe and fabricates no 5h block', async () => { + const { service, runtimeRepo, probes, close } = await setup() try { const block = service.handleUpstreamError({ accountId: 'acc-1', @@ -117,12 +117,12 @@ test('#163: an unknown bare 429 asks for one usage probe and fabricates no 5h bl assert.equal(runtimeRepo.get('acc-1').rate_limit_reset_at, null) assert.deepEqual(probes, [{ accountId: 'acc-1', vmId: 'vm-01' }]) } finally { - close() + await close() } }) -test('529 writes overload_until for the configured minutes', () => { - const { service, runtimeRepo, close } = setup({ overload_cooldown_min: 10 }) +test('529 writes overload_until for the configured minutes', async () => { + const { service, runtimeRepo, close } = await setup({ overload_cooldown_min: 10 }) try { const now = Date.now() const block = service.handleUpstreamError({ @@ -135,12 +135,12 @@ test('529 writes overload_until for the configured minutes', () => { assert.equal(block.until, now + 10 * 60_000) assert.equal(hardBlockOf(runtimeRepo.get('acc-1'), now).reason, 'overloaded') } finally { - close() + await close() } }) -test('only a live allowed header lifts a rate limit early', () => { - const { service, runtimeRepo, close } = setup() +test('only a live allowed header lifts a rate limit early', async () => { + const { service, runtimeRepo, close } = await setup() try { const now = Date.now() service.handleUpstreamError({ @@ -170,12 +170,12 @@ test('only a live allowed header lifts a rate limit early', () => { assert.equal(hardBlockOf(state, now), null) assert.equal(state.session_window_end, Number(reset) * 1000) } finally { - close() + await close() } }) -test('an out-of-range 5h reset header is ignored for the session window', () => { - const { service, runtimeRepo, close } = setup() +test('an out-of-range 5h reset header is ignored for the session window', async () => { + const { service, runtimeRepo, close } = await setup() try { const now = Date.now() service.updateSessionWindow({ @@ -188,12 +188,12 @@ test('an out-of-range 5h reset header is ignored for the session window', () => }) assert.equal(runtimeRepo.get('acc-1').session_window_end, null) } finally { - close() + await close() } }) -test('clearExpired drops elapsed rate limit and overload columns', () => { - const { runtimeRepo, close } = setup() +test('clearExpired drops elapsed rate limit and overload columns', async () => { + const { runtimeRepo, close } = await setup() try { const now = Date.now() runtimeRepo.updateWindow('acc-1', { rateLimitResetAt: now - 1000, overloadUntil: now - 1000 }) @@ -202,12 +202,12 @@ test('clearExpired drops elapsed rate limit and overload columns', () => { assert.equal(state.rate_limit_reset_at, null) assert.equal(state.overload_until, null) } finally { - close() + await close() } }) -test('an incomplete hop does not write an empty_response cooldown', () => { - const { service, runtimeRepo, close } = setup({ empty_response_cooldown_sec: 60 }) +test('an incomplete hop does not write an empty_response cooldown', async () => { + const { service, runtimeRepo, close } = await setup({ empty_response_cooldown_sec: 60 }) try { const block = service.handleUpstreamError({ accountId: 'acc-1', @@ -220,7 +220,7 @@ test('an incomplete hop does not write an empty_response cooldown', () => { assert.equal(runtimeRepo.get('acc-1').cooldown_until, null) assert.equal(typeof service.noteDistinctEmptyHop, 'undefined') } finally { - close() + await close() } }) diff --git a/test/unit/resource-owner.test.mjs b/test/unit/resource-owner.test.mjs index cf61089c2..68c0860ca 100644 --- a/test/unit/resource-owner.test.mjs +++ b/test/unit/resource-owner.test.mjs @@ -10,6 +10,7 @@ import { countUserCreatedVms, filterVmsForPanel, ownerScopeFromApiKey, + ownerScopeFromRequest, vmMatchesOwnerScope, } from '../../src/lib/admin/resource-owner.mjs' @@ -63,3 +64,12 @@ test('unassign origin and proxy bind rules', () => { assert.equal(canBindProxyToVm({ owner_user_id: 'u1' }, {}, { role: 'admin' }), false) assert.equal(canBindProxyToVm({}, { owner_user_id: 'u1' }, { role: 'admin' }), true) }) + +test('request ownership waits for the asynchronous owner role lookup', async () => { + const request = { apiKeyKind: 'managed', apiKeyRecord: { user_id: 'owner' } } + assert.deepEqual(await ownerScopeFromRequest(request, { getById: async () => ({ role: 'admin' }) }), PLATFORM_SCOPE) + assert.deepEqual(await ownerScopeFromRequest(request, { getById: async () => ({ role: 'user' }) }), { + type: 'user', + userId: 'owner', + }) +}) diff --git a/test/unit/routing-concurrency-overrides.test.mjs b/test/unit/routing-concurrency-overrides.test.mjs index 4671d701f..3032763af 100644 --- a/test/unit/routing-concurrency-overrides.test.mjs +++ b/test/unit/routing-concurrency-overrides.test.mjs @@ -6,10 +6,10 @@ import path from 'node:path' import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' import { createRoutingRuntime } from '../../src/lib/admin/routing-runtime.mjs' -function fixture(t, cap, { override = true, tier = 'default' } = {}) { +async function fixture(t, cap, { override = true, tier = 'default' } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-concurrency-')) const accountId = 'test-account' - const quota = new AccountQuota({ + const quota = await new AccountQuota({ dataDir: path.join(root, 'data'), accounts: [ { @@ -19,7 +19,7 @@ function fixture(t, cap, { override = true, tier = 'default' } = {}) { concurrency_override: override, }, ], - }) + }).init() quota.setAccountTier(accountId, tier) const vmPath = path.join(root, 'vms/vm-02.json') fs.mkdirSync(path.dirname(vmPath), { recursive: true }) @@ -36,8 +36,8 @@ function fixture(t, cap, { override = true, tier = 'default' } = {}) { accountQuota: quota, routingConfig: { tiers: {}, quota: {}, concurrency: {} }, }) - t.after(() => { - quota.db.close() + t.after(async () => { + await quota.db.close() fs.rmSync(root, { recursive: true, force: true }) }) return { @@ -49,8 +49,8 @@ function fixture(t, cap, { override = true, tier = 'default' } = {}) { } for (const cap of [0, 2, 16, 20]) { - test(`tier synchronization preserves a manual concurrency value of ${cap}`, (t) => { - const f = fixture(t, cap) + test(`tier synchronization preserves a manual concurrency value of ${cap}`, async (t) => { + const f = await fixture(t, cap) const applied = f.runtime.applyRoutingTierConcurrency({ default: { max_concurrency: 4 } }) assert.equal(applied.skipped, 1) assert.equal(f.policy().maxConcurrency, cap) @@ -59,8 +59,8 @@ for (const cap of [0, 2, 16, 20]) { assert.equal(Boolean(f.account().concurrency_override), true) }) - test(`global default updates preserve a manual concurrency value of ${cap}`, (t) => { - const f = fixture(t, cap) + test(`global default updates preserve a manual concurrency value of ${cap}`, async (t) => { + const f = await fixture(t, cap) f.runtime.applyRoutingConcurrency(4) assert.equal(f.policy().maxConcurrency, cap) assert.equal(f.policy().concurrencyOverride, true) @@ -70,8 +70,8 @@ for (const cap of [0, 2, 16, 20]) { } for (const tier of ['default', 'pro', 'max']) { - test(`inherited ${tier} concurrency receives the tier default`, (t) => { - const f = fixture(t, 2, { override: false, tier }) + test(`inherited ${tier} concurrency receives the tier default`, async (t) => { + const f = await fixture(t, 2, { override: false, tier }) const applied = f.runtime.applyRoutingTierConcurrency({ [tier]: { max_concurrency: 6 } }) assert.equal(applied[tier], 1) assert.equal(f.policy().maxConcurrency, 6) @@ -80,29 +80,29 @@ for (const tier of ['default', 'pro', 'max']) { }) } -test('global default updates still reach inherited concurrency', (t) => { - const f = fixture(t, 2, { override: false }) +test('global default updates still reach inherited concurrency', async (t) => { + const f = await fixture(t, 2, { override: false }) f.runtime.applyRoutingConcurrency(4) assert.equal(f.policy().maxConcurrency, 4) assert.equal(f.policy().concurrencyOverride, false) assert.equal(f.account().max_concurrency, 4) }) -test('tier synchronization does not rewrite an unchanged inherited VM record', (t) => { - const f = fixture(t, 4, { override: false }) +test('tier synchronization does not rewrite an unchanged inherited VM record', async (t) => { + const f = await fixture(t, 4, { override: false }) const before = fs.readFileSync(f.vmPath, 'utf8') const applied = f.runtime.applyRoutingTierConcurrency({ default: { max_concurrency: 4 } }) assert.equal(applied.default, 0) assert.equal(fs.readFileSync(f.vmPath, 'utf8'), before) }) -function rpmFixture(t, rpm, { override = true } = {}) { +async function rpmFixture(t, rpm, { override = true } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-rpm-')) const accountId = 'test-account' - const quota = new AccountQuota({ + const quota = await new AccountQuota({ dataDir: path.join(root, 'data'), accounts: [{ account_id: accountId, vm_id: 'vm-02', max_rpm: rpm }], - }) + }).init() const vmPath = path.join(root, 'vms/vm-02.json') fs.mkdirSync(path.dirname(vmPath), { recursive: true }) fs.writeFileSync( @@ -118,8 +118,8 @@ function rpmFixture(t, rpm, { override = true } = {}) { accountQuota: quota, routingConfig: { tiers: {}, quota: {}, concurrency: {} }, }) - t.after(() => { - quota.db.close() + t.after(async () => { + await quota.db.close() fs.rmSync(root, { recursive: true, force: true }) }) return { @@ -130,8 +130,8 @@ function rpmFixture(t, rpm, { override = true } = {}) { } } -test('tier RPM synchronization preserves a manual RPM value', (t) => { - const f = rpmFixture(t, 30) +test('tier RPM synchronization preserves a manual RPM value', async (t) => { + const f = await rpmFixture(t, 30) const applied = f.runtime.applyRoutingTierRpm({ default: { max_rpm: 10 } }) assert.equal(applied.skipped, 1) assert.equal(f.policy().maxRpm, 30) @@ -139,8 +139,8 @@ test('tier RPM synchronization preserves a manual RPM value', (t) => { assert.equal(f.account().max_rpm, 30) }) -test('tier RPM synchronization updates inherited RPM and skips unchanged records', (t) => { - const f = rpmFixture(t, 10, { override: false }) +test('tier RPM synchronization updates inherited RPM and skips unchanged records', async (t) => { + const f = await rpmFixture(t, 10, { override: false }) const before = fs.readFileSync(f.vmPath, 'utf8') assert.equal(f.runtime.applyRoutingTierRpm({ default: { max_rpm: 10 } }).default, 0) assert.equal(fs.readFileSync(f.vmPath, 'utf8'), before) diff --git a/test/unit/routing-runtime-load.test.mjs b/test/unit/routing-runtime-load.test.mjs index 82e08040f..414c5f9c1 100644 --- a/test/unit/routing-runtime-load.test.mjs +++ b/test/unit/routing-runtime-load.test.mjs @@ -50,7 +50,7 @@ test('applyVmSessionSlots updates only native admission policy', () => { } }) -test('persistRoutingPatch reconciles inherited Claude session slots and preserves overrides', () => { +test('persistRoutingPatch reconciles inherited Claude session slots and preserves overrides', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-session-default-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -104,7 +104,7 @@ test('persistRoutingPatch reconciles inherited Claude session slots and preserve requestLog: { setConfig() {} }, }) - const applied = runtime.persistRoutingPatch({ inference: { session_slots: 4 } }) + const applied = await runtime.persistRoutingPatch({ inference: { session_slots: 4 } }) const inherited = JSON.parse(fs.readFileSync(path.join(vms, 'vm-inherited.json'), 'utf8')) const overridden = JSON.parse(fs.readFileSync(path.join(vms, 'vm-override.json'), 'utf8')) @@ -120,7 +120,7 @@ test('persistRoutingPatch reconciles inherited Claude session slots and preserve } }) -test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel configs', () => { +test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel configs', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-cache-ttl-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -147,7 +147,7 @@ test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel conf requestLog: { setConfig() {} }, }) - runtime.persistRoutingPatch({ compatibility: { cache_ttl: '5m' } }) + await runtime.persistRoutingPatch({ compatibility: { cache_ttl: '5m' } }) const kernel = JSON.parse(fs.readFileSync(path.join(vms, 'vm-claude', 'run', 'kernel.json'), 'utf8')) assert.equal(kernel.default_cache_ttl, '5m') @@ -157,7 +157,7 @@ test('persistRoutingPatch writes compatibility cache_ttl into Claude kernel conf } }) -test('persistRoutingPatch writes a changed stream idle timeout into Claude kernel configs', () => { +test('persistRoutingPatch writes a changed stream idle timeout into Claude kernel configs', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-idle-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -182,11 +182,11 @@ test('persistRoutingPatch writes a changed stream idle timeout into Claude kerne requestLog: { setConfig() {} }, }) - const same = runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 600000 } }) + const same = await runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 600000 } }) assert.equal(same.kernel_persona, null) assert.equal(fs.existsSync(kernelFile), false) - const changed = runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 900000 } }) + const changed = await runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 900000 } }) assert.equal(changed.kernel_reload, 0) assert.equal(JSON.parse(fs.readFileSync(kernelFile, 'utf8')).idle_timeout_seconds, 900) } finally { @@ -194,7 +194,7 @@ test('persistRoutingPatch writes a changed stream idle timeout into Claude kerne } }) -test('persistRoutingPatch writes a changed request body limit into Claude kernel configs', () => { +test('persistRoutingPatch writes a changed request body limit into Claude kernel configs', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-body-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -217,10 +217,10 @@ test('persistRoutingPatch writes a changed request body limit into Claude kernel }, requestLog: { setConfig() {} }, }) - const same = runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 180000 } }) + const same = await runtime.persistRoutingPatch({ failover: { stream_idle_timeout_ms: 180000 } }) assert.equal(same.kernel_persona, null) assert.equal(fs.existsSync(kernelFile), false) - const changed = runtime.persistRoutingPatch({ failover: { max_request_mib: 64 } }) + const changed = await runtime.persistRoutingPatch({ failover: { max_request_mib: 64 } }) assert.equal(changed.kernel_reload, 0) assert.equal(JSON.parse(fs.readFileSync(kernelFile, 'utf8')).max_request_bytes, 64 * 1024 * 1024) } finally { @@ -228,7 +228,7 @@ test('persistRoutingPatch writes a changed request body limit into Claude kernel } }) -test('persistRoutingPatch writes persona_preset into Claude kernel system_layout', () => { +test('persistRoutingPatch writes persona_preset into Claude kernel system_layout', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-persona-kernel-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -259,7 +259,7 @@ test('persistRoutingPatch writes persona_preset into Claude kernel system_layout requestLog: { setConfig() {} }, }) - const applied = runtime.persistRoutingPatch({ compatibility: { persona_preset: 'zero' } }) + const applied = await runtime.persistRoutingPatch({ compatibility: { persona_preset: 'zero' } }) assert.equal(applied.kernel_persona.updated, 1) const kernel = JSON.parse(fs.readFileSync(path.join(vms, 'vm-claude', 'run', 'kernel.json'), 'utf8')) @@ -427,7 +427,7 @@ test('loadRoutingConfig normalizes a missing or invalid vm section', () => { } }) -test('persistRoutingPatch merges a valid vm patch', () => { +test('persistRoutingPatch merges a valid vm patch', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-vm-persist-')) const routingFile = path.join(root, 'routing.json') const routingConfig = { concurrency: {}, tiers: {}, vm: { memory: '1g', vcpus: 2 } } @@ -435,7 +435,7 @@ test('persistRoutingPatch merges a valid vm patch', () => { fs.writeFileSync(routingFile, JSON.stringify(routingConfig)) try { const runtime = persistHarness(root, routingFile, routingConfig) - runtime.persistRoutingPatch({ vm: { memory: '4g', vcpus: 4, smbios: { product: 'Precision 5860' } } }) + await runtime.persistRoutingPatch({ vm: { memory: '4g', vcpus: 4, smbios: { product: 'Precision 5860' } } }) const saved = JSON.parse(fs.readFileSync(routingFile, 'utf8')) assert.equal(saved.vm.memory, '4g') assert.equal(saved.vm.vcpus, 4) @@ -471,7 +471,7 @@ test('persistRoutingPatch rewrites Claude worker.json when the upload interval c t.mock.method(console, 'warn', (line) => warned.push(String(line))) try { const runtime = persistHarness(root, routingFile, routingConfig) - const changed = runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }) + const changed = await runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }) assert.deepEqual(changed.telemetry_sync, { updated: 1, queued: 1, skipped: 0, failed: [] }) assert.equal(JSON.parse(fs.readFileSync(routingFile, 'utf8')).telemetry.upload_interval_seconds, 3600) assert.equal(worker('vm-claude').upload_interval_seconds, 3600) @@ -482,8 +482,11 @@ test('persistRoutingPatch rewrites Claude worker.json when the upload interval c const push = JSON.parse(warned.find((line) => line.includes('remote_slot_push_failed'))) assert.equal(push.vm_id, 'vm-node') assert.deepEqual(push.sets, ['run']) - assert.equal(runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }).telemetry_sync, null) - assert.equal(runtime.persistRoutingPatch({ sticky: {} }).telemetry_sync, null) + assert.equal( + (await runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } })).telemetry_sync, + null, + ) + assert.equal((await runtime.persistRoutingPatch({ sticky: {} })).telemetry_sync, null) assert.equal(JSON.parse(fs.readFileSync(routingFile, 'utf8')).telemetry.upload_interval_seconds, 3600) } finally { fs.rmSync(root, { recursive: true, force: true }) @@ -517,7 +520,7 @@ test('PUT /api/panel/routing rejects an invalid vm patch before persist', async assert.match(response.body?.error?.message, /memory/) }) -test('persistRoutingPatch round-trips websocket gates and analytics', () => { +test('persistRoutingPatch round-trips websocket gates and analytics', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-ws-persist-')) const routingFile = path.join(root, 'routing.json') const routingConfig = { concurrency: {}, tiers: {}, codex: { enabled: true } } @@ -525,7 +528,7 @@ test('persistRoutingPatch round-trips websocket gates and analytics', () => { fs.writeFileSync(routingFile, JSON.stringify(routingConfig)) try { const runtime = persistHarness(root, routingFile, routingConfig) - runtime.persistRoutingPatch({ + await runtime.persistRoutingPatch({ websocket: { enabled: false }, codex: { websocket: { enabled: false }, diff --git a/test/unit/seat-planner.test.mjs b/test/unit/seat-planner.test.mjs index 819e12bc1..66d94a6b1 100644 --- a/test/unit/seat-planner.test.mjs +++ b/test/unit/seat-planner.test.mjs @@ -473,21 +473,21 @@ test('seat stream frames carry per-VM seats, global queue and queue_max', async req.emit('close') }) -function gatedQuota(root) { +async function gatedQuota(root) { const tier = { limit_5h: 0.8, limit_7d: 0.8, max_concurrency: 4 } - return new AccountQuota({ + return await new AccountQuota({ dataDir: fs.mkdtempSync(path.join(root, 'quota-')), config: { quota: { block_on_5h: true, block_on_7d: true, critical_band: 0.05, gate_overshoot: 0.05 }, tiers: { default: tier, pro: tier, max: tier }, }, - }) + }).init() } -function gatedSetup(t, specs) { +async function gatedSetup(t, specs) { const root = project(specs) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const quota = gatedQuota(root) + const quota = await gatedQuota(root) const pool = new PoolScheduler({ projectRoot: root, runtimeRepo: new RuntimeRepo(), @@ -508,7 +508,7 @@ function setUtil5h(quota, accountId, value) { } test('near the gate a seated device waits on its home, one call at a time, and never moves', async (t) => { - const { pool, quota } = gatedSetup(t, [ + const { pool, quota } = await gatedSetup(t, [ { id: 'vm-01', policy: { sessionSlots: 4 } }, { id: 'vm-02', policy: { sessionSlots: 4 } }, ]) @@ -534,7 +534,7 @@ test('near the gate a seated device waits on its home, one call at a time, and n }) test('at line plus overshoot a seated device leaves and its old seat frees at once', async (t) => { - const { pool, quota } = gatedSetup(t, [ + const { pool, quota } = await gatedSetup(t, [ { id: 'vm-01', policy: { sessionSlots: 4 } }, { id: 'vm-02', policy: { sessionSlots: 4 } }, ]) @@ -550,7 +550,7 @@ test('at line plus overshoot a seated device leaves and its old seat frees at on }) test('a new device takes an open VM before a draining one, and a draining one when nothing else is open', async (t) => { - const { pool, quota } = gatedSetup(t, [ + const { pool, quota } = await gatedSetup(t, [ { id: 'vm-01', policy: { sessionSlots: 1 } }, { id: 'vm-02', policy: { sessionSlots: 1 } }, ]) diff --git a/test/unit/security.test.mjs b/test/unit/security.test.mjs index 791912823..0e5eed3a0 100644 --- a/test/unit/security.test.mjs +++ b/test/unit/security.test.mjs @@ -38,9 +38,9 @@ test('panel session persistence stores only token hashes', async () => { process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?session=${Date.now()}`) - const token = security.createPanelSession('admin', 60_000) + const token = await security.createPanelSession('admin', 60_000) assert.match(token, /^kin-panel-/) - assert.equal(security.verifyPanelSession(token).user, 'admin') + assert.equal((await security.verifyPanelSession(token)).user, 'admin') const file = path.join(dir, 'panel-sessions.json') const stored = fs.readFileSync(file, 'utf8') assert.doesNotMatch(stored, /kin-panel-/) @@ -51,7 +51,7 @@ test('panel session persistence stores only token hashes', async () => { [], ) security.revokePanelSession(token) - assert.equal(security.verifyPanelSession(token), null) + assert.equal(await security.verifyPanelSession(token), null) } finally { if (previous == null) delete process.env.KIN_DATA_DIR else process.env.KIN_DATA_DIR = previous @@ -65,14 +65,14 @@ test('verifyPanelSession sliding-renews exp when remaining TTL is under half', a process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?slide=${Date.now()}`) - const token = security.createPanelSession('admin', 60_000) + const token = await security.createPanelSession('admin', 60_000) const file = path.join(dir, 'panel-sessions.json') const before = JSON.parse(fs.readFileSync(file, 'utf8')) const storedKey = Object.keys(before)[0] const expBeforeVerify = before[storedKey].exp const security2 = await import(`../../src/lib/core/security.mjs?slide2=${Date.now()}`) - assert.equal(security2.verifyPanelSession(token)?.user, 'admin') + assert.equal((await security2.verifyPanelSession(token))?.user, 'admin') const after = JSON.parse(fs.readFileSync(file, 'utf8')) const expAfterVerify = after[storedKey].exp assert.ok(expBeforeVerify <= Date.now() + 61_000) @@ -91,7 +91,7 @@ test('verifyPanelSession does not renew when more than half of TTL remains', asy process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?norel=${Date.now()}`) - const token = security.createPanelSession('admin') + const token = await security.createPanelSession('admin') const file = path.join(dir, 'panel-sessions.json') const raw = JSON.parse(fs.readFileSync(file, 'utf8')) const storedKey = Object.keys(raw)[0] @@ -100,7 +100,7 @@ test('verifyPanelSession does not renew when more than half of TTL remains', asy fs.writeFileSync(file, JSON.stringify(raw)) const security2 = await import(`../../src/lib/core/security.mjs?norel2=${Date.now()}`) - assert.equal(security2.verifyPanelSession(token)?.user, 'admin') + assert.equal((await security2.verifyPanelSession(token))?.user, 'admin') const after = JSON.parse(fs.readFileSync(file, 'utf8')) assert.equal(after[storedKey].exp, raw[storedKey].exp) } finally { @@ -116,17 +116,17 @@ test('verifyPanelSession skips a second disk write when TTL was just renewed', a process.env.KIN_DATA_DIR = dir try { const security = await import(`../../src/lib/core/security.mjs?twice=${Date.now()}`) - const token = security.createPanelSession('admin', 60_000) + const token = await security.createPanelSession('admin', 60_000) const file = path.join(dir, 'panel-sessions.json') const security2 = await import(`../../src/lib/core/security.mjs?twice2=${Date.now()}`) - assert.ok(security2.verifyPanelSession(token)) + assert.ok(await security2.verifyPanelSession(token)) const afterFirst = JSON.parse(fs.readFileSync(file, 'utf8')) const storedKey = Object.keys(afterFirst)[0] const expAfterFirst = afterFirst[storedKey].exp const mtimeAfterFirst = fs.statSync(file).mtimeMs - assert.ok(security2.verifyPanelSession(token)) + assert.ok(await security2.verifyPanelSession(token)) const afterSecond = JSON.parse(fs.readFileSync(file, 'utf8')) assert.equal(afterSecond[storedKey].exp, expAfterFirst) assert.equal(fs.statSync(file).mtimeMs, mtimeAfterFirst) @@ -150,7 +150,7 @@ test('existing loose panel session file is tightened on load', async () => { try { const security = await import(`../../src/lib/core/security.mjs?tighten=${Date.now()}`) assert.equal(fs.statSync(file).mode & 0o777, 0o600) - assert.equal(security.verifyPanelSession('kin-panel-load').user, 'admin') + assert.equal((await security.verifyPanelSession('kin-panel-load')).user, 'admin') } finally { if (previous == null) delete process.env.KIN_DATA_DIR else process.env.KIN_DATA_DIR = previous diff --git a/test/unit/settings-state.test.mjs b/test/unit/settings-state.test.mjs new file mode 100644 index 000000000..f845911ca --- /dev/null +++ b/test/unit/settings-state.test.mjs @@ -0,0 +1,30 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { RuntimeSettings } from '../../src/lib/db/settings-state.mjs' + +test('settings cache changes only when its transaction commits', async () => { + const db = getDb() + const settings = new RuntimeSettings(db) + await settings.set('runtime_cache_rollback', { version: 1 }) + await assert.rejects( + withTransaction(db, async () => { + await settings.set('runtime_cache_rollback', { version: 2 }) + throw new Error('synthetic rollback') + }), + /synthetic rollback/, + ) + assert.deepEqual(settings.get('runtime_cache_rollback'), { version: 1 }) + await assert.rejects( + withTransaction(db, async () => { + await settings.remove('runtime_cache_rollback') + throw new Error('synthetic rollback') + }), + /synthetic rollback/, + ) + assert.deepEqual(settings.get('runtime_cache_rollback'), { version: 1 }) + await withTransaction(db, async () => { + await settings.set('runtime_cache_rollback', { version: 3 }) + }) + assert.deepEqual(settings.get('runtime_cache_rollback'), { version: 3 }) +}) diff --git a/test/unit/sticky-router.test.mjs b/test/unit/sticky-router.test.mjs index 75903d2c8..0da51bd73 100644 --- a/test/unit/sticky-router.test.mjs +++ b/test/unit/sticky-router.test.mjs @@ -16,8 +16,8 @@ function tmpDir(prefix = 'kin-sticky-') { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)) } -test('string and header parent ids stay on one family and do not cross API keys', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('string and header parent ids stay on one family and do not cross API keys', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key-a' }, headers: {} } const other = { apiKeyRecord: { id: 'key-b' }, headers: {} } const child = { @@ -56,8 +56,8 @@ test('a declared child without parent or root is rejected as a relation', () => assert.equal(childDeclaredWithoutParent({ metadata: { user_id: { device_id: 'dev' } } }, {}), false) }) -test('bind + resolve + hits increment', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('bind + resolve + hits increment', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-1', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'sess-9' }) const hit = r.resolve('conv-1') assert.equal(hit.accountId, 'acc-1') @@ -69,8 +69,8 @@ test('bind + resolve + hits increment', () => { assert.equal(r.stats().sessions['conv-1'].session_id, 'sess-9') }) -test('bind keeps outbound session on the same VM and overwrites after a VM change', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('bind keeps outbound session on the same VM and overwrites after a VM change', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-s', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'S1' }) r.bind('conv-s', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'S2' }) assert.equal(r.resolve('conv-s').sessionId, 'S1') @@ -86,8 +86,8 @@ test('bind keeps outbound session on the same VM and overwrites after a VM chang assert.equal(r.resolve('conv-s').sessionId, 'S4') }) -test('bind stays locked when both VM and account differ', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('bind stays locked when both VM and account differ', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-lock', { accountId: 'acc-1', vmId: 'vm-1', sessionId: 'S1' }) r.bind('conv-lock', { accountId: 'acc-2', vmId: 'vm-2', sessionId: 'S2' }) assert.equal(r.resolve('conv-lock').accountId, 'acc-1') @@ -95,51 +95,54 @@ test('bind stays locked when both VM and account differ', () => { assert.equal(r.resolve('conv-lock').sessionId, 'S1') }) -test('mergeStickyConfig normalizes outbound_session to rebuild or passthrough', () => { +test('mergeStickyConfig normalizes outbound_session to rebuild or passthrough', async () => { assert.equal(mergeStickyConfig({}).outbound_session, 'rebuild') assert.equal(mergeStickyConfig({ sticky: {} }).outbound_session, 'rebuild') assert.equal(mergeStickyConfig({ sticky: { outbound_session: 'rebuild' } }).outbound_session, 'rebuild') assert.equal(mergeStickyConfig({ sticky: { outbound_session: 'passthrough' } }).outbound_session, 'passthrough') assert.equal(mergeStickyConfig({ sticky: { outbound_session: 'other' } }).outbound_session, 'rebuild') - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() assert.equal(r.config.outbound_session, 'rebuild') r.reloadConfig({ sticky: { enabled: true, outbound_session: 'passthrough' } }) assert.equal(r.config.outbound_session, 'passthrough') }) -test('expired sessions purge on resolve/stats', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: -1 } } }) +test('expired sessions purge on resolve/stats', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: -1 } } }).init() r.bind('conv-2', { accountId: 'a', vmId: 'v' }) assert.equal(r.resolve('conv-2'), null) assert.equal(r.stats().active_sessions, 0) }) -test('disabled sticky returns null and binds nothing', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: false } } }) +test('disabled sticky returns null and binds nothing', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: false } } }).init() r.bind('conv-3', { accountId: 'a', vmId: 'v' }) assert.equal(r.resolve('conv-3'), null) }) -test('sticky sessions persist across re-open', () => { +test('sticky sessions persist across re-open', async () => { const dir = tmpDir() const cfg = { sticky: { enabled: true, ttl_seconds: 3600 } } - const r1 = new StickyRouter({ dataDir: dir, config: cfg }) + const r1 = await new StickyRouter({ dataDir: dir, config: cfg }).init() r1.bind('conv-4', { accountId: 'acc-4', vmId: 'vm-4' }) - const r2 = new StickyRouter({ dataDir: dir, config: cfg }) + const r2 = await new StickyRouter({ dataDir: dir, config: cfg }).init() const hit = r2.resolve('conv-4') assert.ok(hit) assert.equal(hit.accountId, 'acc-4') }) -test('extractKey uses default header_keys when config omits them', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('extractKey uses default header_keys when config omits them', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const key = r.extractKey({ headers: { 'x-session-id': 'sess-default' } }, {}) assert.equal(key, 'sess-default') }) -test('extractKey prefers metadata.user_id session over headers', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractKey prefers metadata.user_id session over headers', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const key = r.extractKey( { headers: { 'x-session-id': 'header-sess' } }, { metadata: { user_id: { session_id: 'meta-sess' } } }, @@ -147,8 +150,11 @@ test('extractKey prefers metadata.user_id session over headers', () => { assert.equal(key, 'meta-sess') }) -test('caller session wins over a persistable envelope', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('caller session wins over a persistable envelope', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const a = r.extractPoolKey( { apiKeyRecord: { id: 'key_f041' }, headers: { 'thread-id': '01a0b947-aaaa' } }, { @@ -180,8 +186,11 @@ test('caller session wins over a persistable envelope', () => { assert.notEqual(a, b) }) -test('envelope without a session id stays one key per API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('envelope without a session id stays one key per API key', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const body = { messages: [{ role: 'user', content: 'Persistable response items (JSON):\n[{"text":"任务"}]' }], } @@ -198,8 +207,11 @@ test('envelope without a session id stays one key per API key', () => { ) }) -test('persistable envelope keys stay isolated per API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'conversation' } } }) +test('persistable envelope keys stay isolated per API key', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, mode: 'conversation' } }, + }).init() const body = { messages: [{ role: 'user', content: 'Persistable response items (JSON):\n[{"text":"任务"}]' }], } @@ -207,8 +219,8 @@ test('persistable envelope keys stay isolated per API key', () => { assert.equal(r.extractKey({ apiKeyRecord: { id: 'key_b' } }, body), 'kkey_b:envelope') }) -test('ip mode does not switch envelope traffic onto the API-key envelope key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }) +test('ip mode does not switch envelope traffic onto the API-key envelope key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }).init() const key = r.extractKey( { apiKeyRecord: { id: 'key_f041' }, headers: { 'x-forwarded-for': '203.0.113.9' } }, { messages: [{ role: 'user', content: 'Persistable response items (JSON):\n[{"text":"任务"}]' }] }, @@ -216,8 +228,8 @@ test('ip mode does not switch envelope traffic onto the API-key envelope key', ( assert.equal(key, 'kkey_f041:ip:203.0.113.9') }) -test('plain first-user hash is unchanged without envelope', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('plain first-user hash is unchanged without envelope', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const a = r.extractKey( { apiKeyRecord: { id: 'key_f041' } }, { messages: [{ role: 'user', content: '同一段会话的第一句' }] }, @@ -227,8 +239,8 @@ test('plain first-user hash is unchanged without envelope', () => { assert.notEqual(a, b) }) -test('extractKey ignores x-client-request-id and hashes first user', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractKey ignores x-client-request-id and hashes first user', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const a = r.extractKey( { headers: { 'x-client-request-id': 'req-aaaa' } }, { messages: [{ role: 'user', content: '同一段会话的第一句' }] }, @@ -247,8 +259,8 @@ test('extractKey ignores x-client-request-id and hashes first user', () => { assert.equal(a, b) }) -test('later blocks of the first user message stay on one session slot', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('later blocks of the first user message stay on one session slot', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_f041' }, headers: {} } const first = { messages: [ @@ -284,8 +296,8 @@ test('later blocks of the first user message stay on one session slot', () => { assert.notEqual(a, r.extractPoolKey(req, other, { platform: 'anthropic' })) }) -test('a shared system-reminder opener does not merge different conversations', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('a shared system-reminder opener does not merge different conversations', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_f041' }, headers: {} } const opener = (question, extra = []) => ({ messages: [ @@ -310,19 +322,19 @@ test('a shared system-reminder opener does not merge different conversations', ( assert.equal(r.extractPoolKey(req, next, { platform: 'anthropic' }), a) }) -test('extractKey mode=ip uses forwarded address', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }) +test('extractKey mode=ip uses forwarded address', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'ip' } } }).init() const key = r.extractKey({ headers: { 'x-forwarded-for': '203.0.113.9, 10.0.0.1' } }, {}) assert.equal(key, 'ip:203.0.113.9') }) -test('extractKey mode=session isolates by API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'session' } } }) +test('extractKey mode=session isolates by API key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, mode: 'session' } } }).init() assert.equal(r.extractKey({ apiKeyRecord: { id: 4 } }, {}), 'k4:login') }) -test('extractOfficialFamilyKey binds parent and child hops by device_id', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractOfficialFamilyKey binds parent and child hops by device_id', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const parent = r.extractOfficialFamilyKey( { headers: { 'x-claude-code-session-id': 'parent-sess' } }, { metadata: { user_id: { device_id: 'aabbcc', session_id: 'parent-sess' } } }, @@ -342,8 +354,8 @@ test('extractOfficialFamilyKey binds parent and child hops by device_id', () => ) }) -test('parent and child session ids stay on separate slots', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('parent and child session ids stay on separate slots', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const parentReq = { headers: { 'user-agent': 'claude-cli/2.1.241 (external, sdk-cli)', 'x-claude-code-session-id': 'parent-sess' }, } @@ -388,8 +400,8 @@ function companionHaiku(sessionId, deviceId = '998dad9c1e3eccf11cd192c3919d4d1f' } } -test('haiku subagent keeps its own session and does not reuse the parent key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('haiku subagent keeps its own session and does not reuse the parent key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const device = '8cd2bdff61fcd056' const req = { apiKeyRecord: { id: 'key_f0419454c00d' }, headers: { 'user-agent': 'Go-http-client/1.1' } } const parentBody = { @@ -409,8 +421,8 @@ test('haiku subagent keeps its own session and does not reuse the parent key', ( assert.match(String(childKey), /1334bab2-94bb-4429-a694-b2fa3434b1f5/) }) -test('explicit parent session shares a family VM and keeps a separate session key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('explicit parent session shares a family VM and keeps a separate session key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_family' }, headers: {} } const parentBody = { model: 'claude-opus-5-5', @@ -436,8 +448,8 @@ test('explicit parent session shares a family VM and keeps a separate session ke assert.notEqual(r.resolve(parentKey).sessionId, r.resolve(childKey).sessionId) }) -test('companion haiku sessions stay independent without an explicit parent', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('companion haiku sessions stay independent without an explicit parent', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_side' }, headers: {} } const a = r.extractPoolKey(req, companionHaiku('sess-a'), { platform: 'anthropic' }) const b = r.extractPoolKey(req, companionHaiku('sess-b'), { platform: 'anthropic' }) @@ -446,8 +458,11 @@ test('companion haiku sessions stay independent without an explicit parent', () assert.doesNotMatch(String(b), /fam:/) }) -test('a stale parent and a real haiku chat do not absorb the companion rule', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 3600 } } }) +test('a stale parent and a real haiku chat do not absorb the companion rule', async () => { + const r = await new StickyRouter({ + dataDir: tmpDir(), + config: { sticky: { enabled: true, ttl_seconds: 3600 } }, + }).init() const device = 'dev-old' const req = { apiKeyRecord: { id: 'key_stale' }, headers: {} } const parentBody = { @@ -486,8 +501,8 @@ test('a stale parent and a real haiku chat do not absorb the companion rule', () assert.match(r.extractPoolKey(req, chat, { platform: 'anthropic' }), /chat-sess$/) }) -test('anthropic and openai sticky keys do not share a session', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('anthropic and openai sticky keys do not share a session', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { headers: { 'x-session-id': 'same-session' } } const body = { metadata: { user_id: { session_id: 'same-session' } } } const claude = r.extractPoolKey(req, body, { platform: 'anthropic' }) @@ -500,8 +515,8 @@ test('anthropic and openai sticky keys do not share a session', () => { assert.equal(r.resolve(r.extractPoolKey(req, body, { platform: 'anthropic' })).vmId, 'vm-claude') }) -test('a caller session does not stick through a content fingerprint', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('a caller session does not stick through a content fingerprint', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const firstUser = [{ role: 'user', content: '同一段跨协议会话的首条消息' }] const anthropic = { metadata: { user_id: { session_id: 'anthropic-session' } }, @@ -521,8 +536,8 @@ test('a caller session does not stick through a content fingerprint', () => { assert.equal(r.resolve(r.extractPoolKey(req, openai)), null) }) -test('explicit session is the stable lock when concurrent turns have different first messages', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('explicit session is the stable lock when concurrent turns have different first messages', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const req = { apiKeyRecord: { id: 'key_lock' }, headers: { 'x-session-id': 'shared-session' } } const first = { messages: [{ role: 'user', content: 'first visible turn' }] } const second = { messages: [{ role: 'user', content: 'trimmed current turn' }] } @@ -532,23 +547,23 @@ test('explicit session is the stable lock when concurrent turns have different f assert.deepEqual(r.collectPoolKeys(req, second), ['kkey_lock:shared-session']) }) -test('provisional bind does not increment hits', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('provisional bind does not increment hits', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-p', { accountId: 'acc', vmId: 'vm-1' }, { countHit: false }) assert.equal(r.stats().sessions['conv-p'].hits, 0) r.bind('conv-p', { accountId: 'acc', vmId: 'vm-1' }) assert.equal(r.stats().sessions['conv-p'].hits, 1) }) -test('extractKey isolates the same session per API key', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('extractKey isolates the same session per API key', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const raw = { headers: { 'x-session-id': 'same-session' } } assert.equal(r.extractKey(raw, {}), 'same-session') assert.equal(r.extractKey({ ...raw, apiKeyRecord: { id: 7 } }, {}), 'k7:same-session') }) -test('canonical identity keys are not scoped to API keys', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('canonical identity keys are not scoped to API keys', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const reqA = { apiKeyRecord: { id: 'key-a' } } const reqB = { apiKeyRecord: { id: 'key-b' } } @@ -566,8 +581,8 @@ test('canonical identity keys are not scoped to API keys', () => { assert.equal(r.canonicalDeviceKey(''), null) }) -test('canonical identity keys ignore account_uuid changes', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }) +test('canonical identity keys ignore account_uuid changes', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true } } }).init() const first = { device_id: 'device-same', account_uuid: 'account-a', session_id: 'session-same' } const second = { device_id: 'device-same', account_uuid: 'account-b', session_id: 'session-same' } @@ -575,8 +590,8 @@ test('canonical identity keys ignore account_uuid changes', () => { assert.equal(r.canonicalDeviceKey(first.device_id), r.canonicalDeviceKey(second.device_id)) }) -test('device affinity bind carries no session seat and can move to another VM', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('device affinity bind carries no session seat and can move to another VM', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const key = r.canonicalDeviceKey('device-affinity') const first = r.bindDeviceAffinity(key, { accountId: 'acc-1', vmId: 'vm-01' }) assert.equal(first.generation, 1) @@ -596,8 +611,8 @@ test('device affinity bind carries no session seat and can move to another VM', assert.equal(hit.slotIndex, null) }) -test('migrateLegacyIdentity lazily writes canonical keys on a legacy hit', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity lazily writes canonical keys on a legacy hit', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_legacy' }, headers: {} } const legacyKey = r.isolateKey('legacy-session', req) r.bind(legacyKey, { accountId: 'acc-legacy', vmId: 'vm-legacy', sessionId: 'out-legacy' }) @@ -623,8 +638,8 @@ test('migrateLegacyIdentity lazily writes canonical keys on a legacy hit', () => assert.equal(legacyHit.vmId, 'vm-legacy') }) -test('migrateLegacyIdentity does not overwrite an existing canonical session binding on conflict', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity does not overwrite an existing canonical session binding on conflict', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_conflict' }, headers: {} } const sessionKey = r.canonicalSessionKey('conflict-session') // A prior request already bound the canonical session key to vm-a. @@ -649,8 +664,8 @@ test('migrateLegacyIdentity does not overwrite an existing canonical session bin assert.equal(legacyHit.accountId, 'acc-b') }) -test('migrateLegacyIdentity is a no-op without a resolvable legacy hit and deletes nothing', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity is a no-op without a resolvable legacy hit and deletes nothing', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('unrelated-conv', { accountId: 'acc-unrelated', vmId: 'vm-unrelated' }) const before = r.stats().active_sessions @@ -668,8 +683,8 @@ test('migrateLegacyIdentity is a no-op without a resolvable legacy hit and delet assert.equal(r.resolve('unrelated-conv').vmId, 'vm-unrelated') }) -test('migrateLegacyIdentity without a trusted device/session identity mints nothing', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity without a trusted device/session identity mints nothing', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_notrust' }, headers: {} } const legacyKey = r.isolateKey('untrusted-session', req) r.bind(legacyKey, { accountId: 'acc-untrusted', vmId: 'vm-untrusted' }) @@ -679,8 +694,8 @@ test('migrateLegacyIdentity without a trusted device/session identity mints noth assert.equal(result.deviceKey, null) }) -test('migrateLegacyIdentity keeps a live device home VM', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('migrateLegacyIdentity keeps a live device home VM', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const req = { apiKeyRecord: { id: 'key_home' }, headers: {} } r.bindDeviceAffinity(r.canonicalDeviceKey('device-home'), { accountId: 'acc-home', vmId: 'vm-home' }) const legacyKey = r.isolateKey('old-session', req) @@ -691,8 +706,8 @@ test('migrateLegacyIdentity keeps a live device home VM', () => { assert.equal(r.resolve('dev2:device-home').vmId, 'vm-home') }) -test('sessionPoolKeys uses one canonical key across API keys and keeps legacy aliases', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('sessionPoolKeys uses one canonical key across API keys and keeps legacy aliases', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() const body = { metadata: { user_id: JSON.stringify({ device_id: 'dev-s', session_id: 'sess-s' }) } } const reqA = { apiKeyRecord: { id: 'a' }, headers: {} } const reqB = { apiKeyRecord: { id: 'b' }, headers: {} } @@ -712,8 +727,8 @@ test('sessionPoolKeys uses one canonical key across API keys and keeps legacy al assert.match(r.familyPoolKey(reqA, 'sess-s'), /ka:family:sess-s$/) }) -test('unbind and unbindByAccount drop dead bindings', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('unbind and unbindByAccount drop dead bindings', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('conv-dead', { accountId: 'acc-x', vmId: 'vm-02' }) r.bind('conv-other', { accountId: 'acc-y', vmId: 'vm-04' }) r.unbind('conv-dead') @@ -724,9 +739,9 @@ test('unbind and unbindByAccount drop dead bindings', () => { assert.equal(r.resolve('conv-other').vmId, 'vm-04') }) -test('proxy pool import/bind/config persist across re-open', () => { +test('proxy pool import/bind/config persist across re-open', async () => { const dir = tmpDir('kin-proxy-') - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() const res = pool.importLines('socks5://user:pass@10.0.0.1:1080\n10.0.0.2:1080\nbadline:xx\n10.0.0.1:1080:user:pass') assert.equal(res.added, 2) const id = pool.snapshot().proxies[0].id @@ -734,7 +749,7 @@ test('proxy pool import/bind/config persist across re-open', () => { pool.updateConfig({ probe_interval_min: 30, max_failures: 3 }) pool.stopScheduler() - const pool2 = new ProxyPool({ dataDir: dir }) + const pool2 = await new ProxyPool({ dataDir: dir }).init() const snap = pool2.snapshot() assert.equal(snap.totals.total, 2) assert.equal(snap.config.probe_interval_min, 30) @@ -745,9 +760,9 @@ test('proxy pool import/bind/config persist across re-open', () => { pool2.stopScheduler() }) -test('proxy remove + unbindVm persist', () => { +test('proxy remove + unbindVm persist', async () => { const dir = tmpDir('kin-proxy-') - const pool = new ProxyPool({ dataDir: dir }) + const pool = await new ProxyPool({ dataDir: dir }).init() pool.importLines('10.1.1.1:1080\n10.1.1.2:1080') const [a, b] = pool.snapshot().proxies.map((p) => p.id) pool.bind(a, 'vm-z') @@ -755,22 +770,22 @@ test('proxy remove + unbindVm persist', () => { pool.remove(b) pool.stopScheduler() - const pool2 = new ProxyPool({ dataDir: dir }) + const pool2 = await new ProxyPool({ dataDir: dir }).init() const snap = pool2.snapshot() assert.equal(snap.totals.total, 1) assert.equal(snap.proxies[0].bound_vm_id, null) pool2.stopScheduler() }) -test('disconnect_on_error config persists and runtime failure disables slot', () => { +test('disconnect_on_error config persists and runtime failure disables slot', async () => { const dir = tmpDir('kin-proxy-') const disabled = [] const disconnected = [] - const pool = new ProxyPool({ + const pool = await new ProxyPool({ dataDir: dir, onDisableVm: (vmId, reason, proxyId) => disabled.push({ vmId, reason, proxyId }), onDisconnectVm: (vmId, reason, proxyId) => disconnected.push({ vmId, reason, proxyId }), - }) + }).init() pool.importLines('10.2.2.2:1080') const id = pool.snapshot().proxies[0].id pool.bind(id, 'vm-err') @@ -779,16 +794,16 @@ test('disconnect_on_error config persists and runtime failure disables slot', () assert.equal(skipped.skipped, true) assert.equal(disconnected.length, 0) - const updated = pool.updateConfig({ disconnect_on_error: true, max_failures: 1 }) + const updated = pool.updateConfig({ enabled: false, disconnect_on_error: true, max_failures: 1 }) assert.equal(updated.ok, true) assert.equal(updated.config.disconnect_on_error, true) pool.stopScheduler() - const pool2 = new ProxyPool({ + const pool2 = await new ProxyPool({ dataDir: dir, onDisableVm: (vmId, reason, proxyId) => disabled.push({ vmId, reason, proxyId }), onDisconnectVm: (vmId, reason, proxyId) => disconnected.push({ vmId, reason, proxyId }), - }) + }).init() assert.equal(pool2.snapshot().config.disconnect_on_error, true) const reported = pool2.reportRuntimeFailure('vm-err', 'proxy_transport_failure') assert.equal(reported.ok, true) @@ -842,8 +857,8 @@ test('isShortProbeRequest leaves conversations alone', () => { assert.equal(isShortProbeRequest({ messages: [{ role: 'user', content: ' ' }] }), false) }) -test('replaceSession remints the outbound session on the same VM', () => { - const r = new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }) +test('replaceSession remints the outbound session on the same VM', async () => { + const r = await new StickyRouter({ dataDir: tmpDir(), config: { sticky: { enabled: true, ttl_seconds: 60 } } }).init() r.bind('sess:a', { accountId: 'acc', vmId: 'vm-05', sessionId: 'old-session', slotIndex: 0 }) r.bind( 'sess:a', @@ -855,5 +870,5 @@ test('replaceSession remints the outbound session on the same VM', () => { r.bind('sess:a', { accountId: 'acc', vmId: 'vm-05' }, { countHit: false, replaceSession: true, clearSlot: true }) assert.equal(r.resolve('sess:a').sessionId, null) assert.equal(r.resolve('sess:a').slotIndex, null) - r.db?.close?.() + await r.db?.close?.() }) diff --git a/test/unit/thinking-owner.test.mjs b/test/unit/thinking-owner.test.mjs index ff2faccc0..3384c2966 100644 --- a/test/unit/thinking-owner.test.mjs +++ b/test/unit/thinking-owner.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' +import { getDb } from '../../src/lib/db/database.mjs' import { createHandleProtocol } from '../../src/lib/protocol/handle-protocol.mjs' import { ThinkingOwnerStore, @@ -18,7 +18,7 @@ const UNKNOWN = 'sig-never-recorded-0123456789abcd' const SHORT = 'too-short' function store() { - return new ThinkingOwnerStore({ db: new DatabaseSync(':memory:') }) + return new ThinkingOwnerStore({ db: getDb() }) } function history() { @@ -40,12 +40,12 @@ function history() { } } -test('a signature stays with the account that first returned it', () => { +test('a signature stays with the account that first returned it', async () => { const owners = store() const response = { content: [{ type: 'thinking', thinking: 'own chain', signature: OWN }] } - assert.equal(recordThinkingOwners(response, 'account-a', owners), 1) - recordThinkingOwners(response, 'account-b', owners) - const out = filterThinkingForAccount(history(), 'account-a', owners) + assert.equal(await recordThinkingOwners(response, 'account-a', owners), 1) + await recordThinkingOwners(response, 'account-b', owners) + const out = await filterThinkingForAccount(history(), 'account-a', owners) const blocks = out.body.messages[1].content assert.equal(blocks[0].type, 'thinking') assert.equal(blocks[0].signature, OWN) @@ -62,15 +62,15 @@ test('a signature stays with the account that first returned it', () => { ) }) -test('the same foreign signature converts to the same text on a later hop', () => { +test('the same foreign signature converts to the same text on a later hop', async () => { const owners = store() - recordThinkingOwners( + await recordThinkingOwners( { content: [{ type: 'thinking', thinking: 'foreign chain', signature: FOREIGN }] }, 'account-b', owners, ) - const first = filterThinkingForAccount(history(), 'account-a', owners) - const second = filterThinkingForAccount(history(), 'account-a', owners) + const first = await filterThinkingForAccount(history(), 'account-a', owners) + const second = await filterThinkingForAccount(history(), 'account-a', owners) assert.deepEqual(first.body.messages[1].content[1], second.body.messages[1].content[1]) assert.equal( thinkingSignatureKey({ type: 'thinking', signature: FOREIGN }), @@ -78,23 +78,23 @@ test('the same foreign signature converts to the same text on a later hop', () = ) }) -test('an unreachable database leaves the history untouched', () => { +test('an unreachable database leaves the history untouched', async () => { const body = history() - const closed = new DatabaseSync(':memory:') - closed.close() + const closed = getDb() + await closed.close() const owners = new ThinkingOwnerStore({ db: closed }) - const out = filterThinkingForAccount(body, 'account-a', owners) + const out = await filterThinkingForAccount(body, 'account-a', owners) assert.equal(out.body, body) assert.equal(out.kept, 4) assert.equal(out.converted, 0) - assert.equal(recordThinkingOwners(body.messages[1], 'account-a', owners), 4) + assert.equal(await recordThinkingOwners(body.messages[1], 'account-a', owners), 4) }) const SERVED = 'sig-served-by-account-a-0123456789' test('a handler records the served signature and the next account replays it as text', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-thinking-owner-')) - const owners = new ThinkingOwnerStore({ db: new DatabaseSync(':memory:') }) + const owners = new ThinkingOwnerStore({ db: getDb() }) const seen = [] const handler = createHandleProtocol({ json: (_res, status) => status, @@ -182,10 +182,10 @@ test('a handler records the served signature and the next account replays it as fs.rmSync(root, { recursive: true, force: true }) } }) -test('dropping the only signed block merges the neighbouring turns', () => { +test('dropping the only signed block merges the neighbouring turns', async () => { const owners = store() - recordThinkingOwners({ content: [{ type: 'thinking', thinking: '', signature: FOREIGN }] }, 'account-b', owners) - const out = filterThinkingForAccount( + await recordThinkingOwners({ content: [{ type: 'thinking', thinking: '', signature: FOREIGN }] }, 'account-b', owners) + const out = await filterThinkingForAccount( { messages: [ { role: 'user', content: 'before' }, diff --git a/test/unit/usage-profile.test.mjs b/test/unit/usage-profile.test.mjs index 7c37a1719..609b664cd 100644 --- a/test/unit/usage-profile.test.mjs +++ b/test/unit/usage-profile.test.mjs @@ -75,13 +75,13 @@ test('relay restores the shipped quota and risk defaults only', () => { assert.equal(risk.refusal.enabled, true) }) -test('applyUsageProfile writes routing, distill, and risk settings', () => { +test('applyUsageProfile writes routing, distill, and risk settings', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-profile-')) const distillFile = path.join(dir, 'distill-rules.json') const settings = memorySettings() const seen = [] try { - const applied = applyUsageProfile({ + const applied = await applyUsageProfile({ mode: 'personal', persistRouting: (patch) => { seen.push(patch) @@ -97,7 +97,7 @@ test('applyUsageProfile writes routing, distill, and risk settings', () => { assert.equal(readJevConfig((key, fallback) => settings.get(key, fallback)).hard_regex_enabled, false) assert.equal(refusalGuardPolicy((key, fallback) => settings.get(key, fallback)).enabled, false) - applyUsageProfile({ + await applyUsageProfile({ mode: 'relay', persistRouting: (patch) => { seen.push(patch) @@ -115,11 +115,16 @@ test('applyUsageProfile writes routing, distill, and risk settings', () => { } }) -test('unknown profile is rejected and a saved profile survives complete', () => { +test('unknown profile is rejected and a saved profile survives complete', async () => { assert.equal(routingPatchForProfile('nope'), null) - assert.throws( - () => - applyUsageProfile({ mode: 'station', persistRouting() {}, settings: memorySettings(), distillFile: '/tmp/x' }), + await assert.rejects( + async () => + await applyUsageProfile({ + mode: 'station', + persistRouting() {}, + settings: memorySettings(), + distillFile: '/tmp/x', + }), (error) => error.code === 'invalid_profile', ) const saved = setOnboardingProfile({}, 'admin', 'personal', '2026-10-13T00:00:00.000Z') diff --git a/test/unit/vm-db-sync.test.mjs b/test/unit/vm-db-sync.test.mjs index 4d5bbadf9..d9d91d94d 100644 --- a/test/unit/vm-db-sync.test.mjs +++ b/test/unit/vm-db-sync.test.mjs @@ -7,7 +7,14 @@ import { openDatabase, closeDatabase, getDb } from '../../src/lib/db/database.mj import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' import { UsersRepo } from '../../src/lib/db/repos/users-repo.mjs' import { encryptString, decryptString, isEncrypted } from '../../src/lib/db/secure.mjs' -import { initVmDbSync, stopVmWatch, syncVmFile, removeVmFromDb, reconcileVms } from '../../src/lib/vm/vm-db-sync.mjs' +import { + initVmDbSync, + stopVmWatch, + syncVmFile, + removeVmFromDb, + reconcileVms, + flushVmDbSync, +} from '../../src/lib/vm/vm-db-sync.mjs' import { setVmWriteHook, atomicWriteJson } from '../../src/lib/vm/vm-file.mjs' import { persistOauthToVm } from '../../src/lib/oauth/oauth-credentials.mjs' import { setVmSchedulable, setActiveVm } from '../../src/lib/vm/vm-registry.mjs' @@ -37,26 +44,27 @@ function seedVmFile(id = 'vm-t1', extra = {}) { return vm } -beforeEach(() => { +beforeEach(async () => { project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vmsync-')) - openDatabase({ dataDir: path.join(project, 'data') }) + await openDatabase({ dataDir: path.join(project, 'data') }) }) -afterEach(() => { +afterEach(async () => { stopVmWatch() setVmWriteHook(null) - closeDatabase() + await closeDatabase() fs.rmSync(project, { recursive: true, force: true }) delete process.env.KIN_DB_SECRET }) -test('initVmDbSync reconciles existing vm files into DB (credentials included)', () => { +test('initVmDbSync reconciles existing vm files into DB (credentials included)', async () => { seedVmFile('vm-t1') setActiveVm(project, 'vm-t1') - const res = initVmDbSync(project, { watch: false }) + const res = await initVmDbSync(project, { watch: false }) assert.equal(res.upserted, 1) const repo = new VmsRepo(getDb()) - const row = repo.get('vm-t1') + await flushVmDbSync() + const row = await repo.get('vm-t1') assert.equal(row.access_token, null) assert.equal(row.refresh_token, null) assert.equal(row.session_key, null) @@ -64,16 +72,16 @@ test('initVmDbSync reconciles existing vm files into DB (credentials included)', assert.equal(row.vm.claude.access_token, undefined) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.has_refresh, true) - assert.equal(repo.getActiveVmId(), 'vm-t1') + assert.equal(await repo.getActiveVmId(), 'vm-t1') }) -test('startup reclaims deleted users assigned VMs, including DB-only restored records', () => { +test('startup reclaims deleted users assigned VMs, including DB-only restored records', async () => { process.env.KIN_DB_SECRET = 'reclaim-test-secret' const users = new UsersRepo(getDb()) for (const id of ['deleted-owner', 'live-owner', 'disabled-owner']) { - users.insert({ id, username: id, password_hash: 'unused', role: 'user', enabled: id !== 'disabled-owner' }) + await users.insert({ id, username: id, password_hash: 'unused', role: 'user', enabled: id !== 'disabled-owner' }) } - users.remove('deleted-owner') + await users.remove('deleted-owner') seedVmFile('assigned', { owner_user_id: 'deleted-owner', origin: 'admin_assigned' }) seedVmFile('legacy', { owner_user_id: 'deleted-owner' }) seedVmFile('private', { owner_user_id: 'deleted-owner', origin: 'user_created' }) @@ -82,18 +90,19 @@ test('startup reclaims deleted users assigned VMs, including DB-only restored re seedVmFile('unknown', { owner_user_id: 'unknown-owner', origin: 'admin_assigned' }) const restored = seedVmFile('restored', { owner_user_id: 'deleted-owner', origin: 'admin_assigned' }) const repo = new VmsRepo(getDb()) - repo.upsertFromVmJson(restored) + await repo.upsertFromVmJson(restored) fs.rmSync(path.join(project, 'vms', 'restored.json')) - const result = initVmDbSync(project, { watch: false }) + const result = await initVmDbSync(project, { watch: false }) assert.deepEqual(result.reclaimed_vm_ids.sort(), ['assigned', 'legacy', 'restored']) for (const id of result.reclaimed_vm_ids) { const vm = JSON.parse(fs.readFileSync(path.join(project, 'vms', `${id}.json`), 'utf8')) assert.equal(vm.owner_user_id, null) assert.equal(vm.origin, 'platform') - const row = repo.get(id) + await flushVmDbSync() + const row = await repo.get(id) assert.equal(row.vm.owner_user_id, null) - const raw = getDb().prepare('SELECT owner_user_id, origin FROM vms WHERE id = ?').get(id) + const raw = await getDb().prepare('SELECT owner_user_id, origin FROM vms WHERE id = $1').get(id) assert.equal(raw.owner_user_id, null) assert.equal(raw.origin, 'platform') assert.equal(row.vm.claude.has_access, true) @@ -105,31 +114,31 @@ test('startup reclaims deleted users assigned VMs, including DB-only restored re ['disabled', 'disabled-owner'], ['unknown', 'unknown-owner'], ]) { - assert.equal(repo.get(id).vm.owner_user_id, owner) + assert.equal((await repo.get(id)).vm.owner_user_id, owner) } const before = fs.readFileSync(path.join(project, 'vms', 'assigned.json'), 'utf8') - assert.deepEqual(initVmDbSync(project, { watch: false }).reclaimed_vm_ids, []) + assert.deepEqual((await initVmDbSync(project, { watch: false })).reclaimed_vm_ids, []) assert.equal(fs.readFileSync(path.join(project, 'vms', 'assigned.json'), 'utf8'), before) }) -test('startup ownership recovery skips an unreadable slot and continues with valid assignments', (t) => { +test('startup ownership recovery skips an unreadable slot and continues with valid assignments', async (t) => { const users = new UsersRepo(getDb()) - users.insert({ id: 'deleted-owner', username: 'deleted', password_hash: 'unused', role: 'user' }) - users.remove('deleted-owner') + await users.insert({ id: 'deleted-owner', username: 'deleted', password_hash: 'unused', role: 'user' }) + await users.remove('deleted-owner') seedVmFile('assigned', { owner_user_id: 'deleted-owner', origin: 'admin_assigned' }) fs.writeFileSync(path.join(project, 'vms', 'broken.json'), '{') const warnings = [] t.mock.method(console, 'warn', (message) => warnings.push(message)) - const result = initVmDbSync(project, { watch: false }) + const result = await initVmDbSync(project, { watch: false }) assert.deepEqual(result.reclaimed_vm_ids, ['assigned']) assert.ok(warnings.some((message) => /ownership recovery skipped broken.json/.test(message))) - assert.equal(new VmsRepo(getDb()).get('assigned').vm.owner_user_id, null) + assert.equal((await new VmsRepo(getDb()).get('assigned')).vm.owner_user_id, null) }) -test('write-through hook mirrors persistOauthToVm into DB', () => { +test('write-through hook mirrors persistOauthToVm into DB', async () => { seedVmFile('vm-t2') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) const vmPath = path.join(project, 'vms', 'vm-t2.json') persistOauthToVm(vmPath, { access_token: 'sk-ant-oat01-NEW', @@ -137,7 +146,8 @@ test('write-through hook mirrors persistOauthToVm into DB', () => { expires_at: 1893460000, }) const repo = new VmsRepo(getDb()) - const row = repo.get('vm-t2') + await flushVmDbSync() + const row = await repo.get('vm-t2') assert.equal(row.access_token, null) assert.equal(row.refresh_token, null) assert.equal(row.vm.claude.has_access, true) @@ -145,18 +155,19 @@ test('write-through hook mirrors persistOauthToVm into DB', () => { assert.equal(row.vm.claude.access_token, undefined) }) -test('vm-registry writes are mirrored (setVmSchedulable)', () => { +test('vm-registry writes are mirrored (setVmSchedulable)', async () => { seedVmFile('vm-t3') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) setVmSchedulable(project, 'vm-t3', false, 'test-reason') - const row = new VmsRepo(getDb()).get('vm-t3') + await flushVmDbSync() + const row = await new VmsRepo(getDb()).get('vm-t3') assert.equal(row.schedulable, false) assert.equal(row.vm.schedule_disabled_reason, 'test-reason') }) -test('setVmSchedulable preserveStatus keeps running when operator disables', () => { +test('setVmSchedulable preserveStatus keeps running when operator disables', async () => { seedVmFile('vm-t3c') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) setVmSchedulable(project, 'vm-t3c', false, 'disabled', { preserveStatus: true }) const off = JSON.parse(fs.readFileSync(path.join(project, 'vms', 'vm-t3c.json'), 'utf8')) assert.equal(off.status, 'running') @@ -164,9 +175,9 @@ test('setVmSchedulable preserveStatus keeps running when operator disables', () assert.equal(off.schedule_disabled_reason, 'disabled') }) -test('setVmSchedulable(true) restores soft-paused status to running', () => { +test('setVmSchedulable(true) restores soft-paused status to running', async () => { seedVmFile('vm-t3b') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) setVmSchedulable(project, 'vm-t3b', false, 'proxy_rebind_worker_restart') const paused = JSON.parse(fs.readFileSync(path.join(project, 'vms', 'vm-t3b.json'), 'utf8')) assert.equal(paused.status, 'paused') @@ -178,12 +189,12 @@ test('setVmSchedulable(true) restores soft-paused status to running', () => { assert.equal(ready.schedule_disabled_reason, null) }) -test('reconcile rebuilds missing vm file from DB (restore pull-up)', () => { +test('reconcile rebuilds missing vm file from DB (restore pull-up)', async () => { const vm = seedVmFile('vm-t4') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) const vmPath = path.join(project, 'vms', 'vm-t4.json') fs.rmSync(vmPath) - const res = reconcileVms(project) + const res = await reconcileVms(project) assert.equal(res.rebuilt, 1) assert.ok(fs.existsSync(vmPath)) const back = JSON.parse(fs.readFileSync(vmPath, 'utf8')) @@ -193,24 +204,24 @@ test('reconcile rebuilds missing vm file from DB (restore pull-up)', () => { assert.equal(back.claude.email, vm.claude.email) }) -test('removeVmFromDb drops the row', () => { +test('removeVmFromDb drops the row', async () => { seedVmFile('vm-t5') - initVmDbSync(project, { watch: false }) - assert.equal(removeVmFromDb('vm-t5'), true) - assert.equal(new VmsRepo(getDb()).get('vm-t5'), null) + await initVmDbSync(project, { watch: false }) + assert.equal(await removeVmFromDb('vm-t5'), true) + assert.equal(await new VmsRepo(getDb()).get('vm-t5'), null) }) -test('KIN_DB_SECRET encrypts credentials at rest, decrypts on read', () => { +test('KIN_DB_SECRET encrypts credentials at rest, decrypts on read', async () => { process.env.KIN_DB_SECRET = 'test-secret-123' seedVmFile('vm-t6') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) // raw row must be ciphertext - const raw = getDb().prepare('SELECT access_token, vm_json, encrypted FROM vms WHERE id = ?').get('vm-t6') + const raw = await getDb().prepare('SELECT access_token, vm_json, encrypted FROM vms WHERE id = $1').get('vm-t6') assert.equal(raw.encrypted, 1) assert.equal(raw.access_token, null) assert.ok(isEncrypted(raw.vm_json), 'vm_json should be encrypted') assert.doesNotMatch(String(raw.vm_json), /sk-ant-oat01/) - const row = new VmsRepo(getDb()).get('vm-t6') + const row = await new VmsRepo(getDb()).get('vm-t6') assert.equal(row.access_token, null) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.refresh_token, undefined) @@ -224,28 +235,89 @@ test('secure round-trip + tamper detection', () => { assert.equal(decryptString('plain-value', 's3cret'), 'plain-value') }) -test('syncVmFile no-ops safely when DB is closed', () => { +test('syncVmFile no-ops safely when DB is closed', async () => { seedVmFile('vm-t7') - closeDatabase() - assert.equal(syncVmFile(path.join(project, 'vms', 'vm-t7.json')), false) - assert.equal(removeVmFromDb('vm-t7'), false) + await closeDatabase() + assert.equal(await syncVmFile(path.join(project, 'vms', 'vm-t7.json')), false) + assert.equal(await removeVmFromDb('vm-t7'), false) // atomicWriteJson still works without a hook/DB atomicWriteJson(path.join(project, 'vms', 'vm-t7.json'), { id: 'vm-t7' }) }) test('file newer than DB wins on reconcile (mtime compare)', async () => { seedVmFile('vm-t8') - initVmDbSync(project, { watch: false }) + await initVmDbSync(project, { watch: false }) // simulate external edit with a newer mtime await new Promise((r) => setTimeout(r, 20)) const vmPath = path.join(project, 'vms', 'vm-t8.json') const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) vm.claude.access_token = 'sk-ant-oat01-EXTERNAL' fs.writeFileSync(vmPath, JSON.stringify(vm, null, 2)) - const res = reconcileVms(project) + const res = await reconcileVms(project) assert.equal(res.upserted, 1) - const row = new VmsRepo(getDb()).get('vm-t8') + const row = await new VmsRepo(getDb()).get('vm-t8') assert.equal(row.access_token, null) assert.equal(row.vm.claude.has_access, true) assert.equal(row.vm.claude.access_token, undefined) }) + +test('queued VM upserts finish before deletion and cannot rebuild the deleted VM', async () => { + seedVmFile('vm-ordered') + await initVmDbSync(project, { watch: false }) + const file = path.join(project, 'vms', 'vm-ordered.json') + const vm = JSON.parse(fs.readFileSync(file, 'utf8')) + atomicWriteJson(file, { ...vm, name: 'first update' }) + atomicWriteJson(file, { ...vm, name: 'last update' }) + fs.unlinkSync(file) + assert.equal(await removeVmFromDb('vm-ordered'), true) + await flushVmDbSync() + const restored = await reconcileVms(project) + assert.equal(restored.rebuilt, 0) + assert.equal(await new VmsRepo(getDb()).get('vm-ordered'), null) + assert.equal(fs.existsSync(file), false) +}) + +test('reconciliation cannot overwrite a newer queued VM snapshot', async () => { + seedVmFile('vm-race', { name: 'old snapshot' }) + await initVmDbSync(project, { watch: false }) + const db = getDb() + await db.prepare('UPDATE vms SET file_mtime_ms = 0 WHERE id = $1').run('vm-race') + const query = db.query.bind(db) + let enteredResolve + let releaseResolve + const entered = new Promise((resolve) => { + enteredResolve = resolve + }) + const release = new Promise((resolve) => { + releaseResolve = resolve + }) + let intercepted = false + db.query = async (sql, values) => { + const result = await query(sql, values) + if (!intercepted && sql === 'SELECT * FROM vms WHERE id = $1') { + intercepted = true + enteredResolve() + await release + } + return result + } + const reconcile = reconcileVms(project) + let flush + try { + await entered + atomicWriteJson(path.join(project, 'vms', 'vm-race.json'), { id: 'vm-race', name: 'new snapshot' }) + let flushed = false + flush = flushVmDbSync().then(() => { + flushed = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(flushed, false, 'flush must wait for the reconciliation preceding the new snapshot') + } finally { + releaseResolve() + await reconcile + await flush + db.query = query + } + const row = await new VmsRepo(db).get('vm-race') + assert.equal(row.vm.name, 'new snapshot') +}) diff --git a/test/unit/vm-package.test.mjs b/test/unit/vm-package.test.mjs index 7f7dc1bef..3217b9a32 100644 --- a/test/unit/vm-package.test.mjs +++ b/test/unit/vm-package.test.mjs @@ -83,12 +83,12 @@ function makeHandler(project, body, { pool, role = 'admin', panelUsers = null } return { handlePanel, response } } -function makePool(t) { - const db = createDatabase({ dbPath: ':memory:' }) - const pool = new ProxyPool({ db }) - t.after(() => { +async function makePool(t) { + const db = await createDatabase({ dbPath: ':memory:' }) + const pool = await new ProxyPool({ db }).init() + t.after(async () => { pool.stopScheduler() - db.close() + await db.close() }) return pool } @@ -134,7 +134,7 @@ test('package export keeps a window whose stored fields are partly invalid', () test('import creates the slot and rolls a colliding id forward', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const doc = claudePackage('vm-pkg') const first = makeHandler(root, doc, { pool }) await first.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) @@ -174,7 +174,7 @@ test('import creates the slot and rolls a colliding id forward', async (t) => { test('write-back updates the current slot and leaves runtime counters', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-put-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const created = makeHandler(root, claudePackage('vm-pkg'), { pool }) await created.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) assert.equal(created.response.status, 201, created.response.body?.error?.message) @@ -204,7 +204,7 @@ test('write-back updates the current slot and leaves runtime counters', async (t test('local egress imports as px-local and codex tokens stay out of vm.json', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-kind-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const local = claudePackage('vm-local') local.proxy = { kind: 'local' } const localRes = makeHandler(root, local, { pool }) @@ -249,7 +249,7 @@ test('local egress imports as px-local and codex tokens stay out of vm.json', as test('user cannot download a package and cannot import past the create quota', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-user-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const admin = makeHandler(root, claudePackage('vm-pkg'), { pool }) await admin.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) assert.equal(admin.response.status, 201, admin.response.body?.error?.message) @@ -275,7 +275,7 @@ test('user cannot download a package and cannot import past the create quota', a test('tenant import cannot bind px-local or another owner SOCKS and drops seed_policy', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-tenant-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const panelUsers = { getById: () => ({ vm_create_quota: 5 }) } const importAs = async (doc) => { const h = makeHandler(root, doc, { pool, role: 'user', panelUsers }) @@ -315,7 +315,7 @@ test('tenant import cannot bind px-local or another owner SOCKS and drops seed_p test('admin export returns the SOCKS password', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-pkg-get-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) - const pool = makePool(t) + const pool = await makePool(t) const created = makeHandler(root, claudePackage('vm-pkg'), { pool }) await created.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) const got = makeHandler(root, null, { pool }) diff --git a/test/unit/vm-quota-override.test.mjs b/test/unit/vm-quota-override.test.mjs index 781f6bd33..2b71f6ac0 100644 --- a/test/unit/vm-quota-override.test.mjs +++ b/test/unit/vm-quota-override.test.mjs @@ -6,8 +6,8 @@ import path from 'node:path' import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' import { parseVmQuotaOverride, vmQuotaOverrideOf } from '../../src/lib/pool/vm-quota-override.mjs' -function quota() { - return new AccountQuota({ +async function quota() { + return await new AccountQuota({ dataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'kin-vm-quota-')), config: { quota: { block_on_5h: true, block_on_7d: true }, @@ -17,7 +17,7 @@ function quota() { { account_id: 'acc-a', vm_id: 'vm-a' }, { account_id: 'acc-b', vm_id: 'vm-b' }, ], - }) + }).init() } function setUtil(q, accountId, u5, u7 = 0.1) { @@ -32,8 +32,8 @@ function setUtil(q, accountId, u5, u7 = 0.1) { ) } -test('VM 5h threshold override only moves that VM gate', () => { - const q = quota() +test('VM 5h threshold override only moves that VM gate', async () => { + const q = await quota() q.setVmQuotaOverride('vm-a', { limit_5h: 0.95 }) setUtil(q, 'acc-a', 0.9) setUtil(q, 'acc-b', 0.9) @@ -41,8 +41,8 @@ test('VM 5h threshold override only moves that VM gate', () => { assert.equal(q.canAccept('acc-b').reason, 'quota_5h_safety') }) -test('VM block switch off lets the slot run past its threshold; clearing restores global', () => { - const q = quota() +test('VM block switch off lets the slot run past its threshold; clearing restores global', async () => { + const q = await quota() setUtil(q, 'acc-a', 0.9) q.setVmQuotaOverride('vm-a', { block_on_5h: false }) assert.equal(q.canAccept('acc-a').ok, true) diff --git a/test/unit/vm-test-chat-billing.test.mjs b/test/unit/vm-test-chat-billing.test.mjs index 40d979103..23e7189cc 100644 --- a/test/unit/vm-test-chat-billing.test.mjs +++ b/test/unit/vm-test-chat-billing.test.mjs @@ -70,7 +70,7 @@ test('runVmTestChat without project/vm does not invent a panel request_log', asy const result = await runVmTestChat({ vmId: 'vm-01', requestLog: store }) assert.equal(result.ok, false) assert.equal(result.error?.code, 'invalid_request') - assert.equal(store.listNormal({ limit: 5 }).length, 0) + assert.equal((await store.listNormal({ limit: 5 })).length, 0) }) test('runVmTestChat loopbacks /v1/messages with x-kin-vm pin and never calls a worker', async (t) => { diff --git a/test/unit/vm-test-chat-codex.test.mjs b/test/unit/vm-test-chat-codex.test.mjs index b29995a6a..9777f891b 100644 --- a/test/unit/vm-test-chat-codex.test.mjs +++ b/test/unit/vm-test-chat-codex.test.mjs @@ -66,9 +66,9 @@ async function startLoopbackServer(t, respond) { return { baseUrl: `http://127.0.0.1:${server.address().port}`, calls } } -test('listTestableModels(openai) hides Claude ids', () => { - const gpt = listTestableModels('openai') - const claude = listTestableModels('anthropic') +test('listTestableModels(openai) hides Claude ids', async () => { + const gpt = await listTestableModels('openai') + const claude = await listTestableModels('anthropic') assert.ok(gpt.length > 0) assert.ok(claude.length > 0) assert.ok(gpt.every((m) => /^(gpt-|codex-)/i.test(m.id) || m.family === 'codex')) @@ -294,7 +294,7 @@ test('runVmTestChat Codex ENOENT is not rewritten as wrap cli-hop', async (t) => t.after(() => fs.rmSync(root, { recursive: true, force: true })) seedCodexVm(root) seedRouting(root) - const model = listTestableModels('openai')[0]?.id || 'gpt-5.5' + const model = (await listTestableModels('openai'))[0]?.id || 'gpt-5.5' const { baseUrl } = await startLoopbackServer(t, () => ({ status: 503, body: { diff --git a/test/unit/vm-test-schedule.test.mjs b/test/unit/vm-test-schedule.test.mjs index 8217717ad..bb231a2ae 100644 --- a/test/unit/vm-test-schedule.test.mjs +++ b/test/unit/vm-test-schedule.test.mjs @@ -3,8 +3,8 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { DatabaseSync } from 'node:sqlite' -import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' +import { createDatabase } from '../../src/lib/db/database.mjs' +import { loadSettingsState, RuntimeSettings as SettingsRepo } from '../../src/lib/db/settings-state.mjs' import { compileStructuredSchedule, normalizeScheduleStructure, @@ -94,36 +94,36 @@ test('legacy cron input must match structured patterns', () => { ) }) -test('SettingsRepo schedules reload after restart without an immediate catch-up run', (t) => { +test('SettingsRepo schedules reload after restart without an immediate catch-up run', async (t) => { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-test-schedule-')) - const dbPath = path.join(dataDir, 'settings.db') - let db = new DatabaseSync(dbPath) - db.exec('CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT NOT NULL)') + let db = await createDatabase() + await loadSettingsState(db) let settings = new SettingsRepo(db) let first let restored - t.after(() => { + t.after(async () => { first?.stop() restored?.stop() - db.close() + await db.close() fs.rmSync(dataDir, { recursive: true, force: true }) }) first = runner({ settings }).schedule - first.save('vm-01', claudeConfig) + await first.save('vm-01', claudeConfig) assert.deepEqual( Object.keys(settings.all()).filter((key) => key.startsWith(VM_TEST_SCHEDULE_PREFIX)), [`${VM_TEST_SCHEDULE_PREFIX}vm-01`], ) - first.start() + await first.start() assert.equal(first.jobs.has('vm-01'), true) assert.equal(first.get('vm-01').running, false) assert.ok(first.get('vm-01').next_run_at) assert.equal(first.get('vm-01').cron, undefined) first.stop() - db.close() - db = new DatabaseSync(dbPath) + await db.close() + db = await createDatabase() + await loadSettingsState(db) settings = new SettingsRepo(db) restored = new VmTestScheduleRunner({ projectRoot: '/missing-project', @@ -131,7 +131,7 @@ test('SettingsRepo schedules reload after restart without an immediate catch-up list: () => [claudeVm], runTest: async () => ({ ok: true }), }) - restored.start() + await restored.start() assert.equal(restored.jobs.has('vm-01'), true) assert.equal(restored.get('vm-01').enabled, true) restored.stop() @@ -148,8 +148,8 @@ test('runs do not overlap per VM and failures persist only compact result fields }) }, }) - schedule.save('vm-01', claudeConfig) - schedule.start() + await schedule.save('vm-01', claudeConfig) + await schedule.start() const job = schedule.jobs.get('vm-01') const first = job.trigger() @@ -175,11 +175,11 @@ test('runs do not overlap per VM and failures persist only compact result fields schedule.stop() }) -test('sync overwrites schedule only and creates disabled plans with default dialog params', () => { +test('sync overwrites schedule only and creates disabled plans with default dialog params', async () => { const codexVm = { id: 'vm-02', platform: 'openai' } const { settings, schedule } = runner({ list: () => [claudeVm, codexVm] }) - schedule.save('vm-01', { ...claudeConfig, prompt: 'keep-me', enabled: true }) - const result = schedule.syncSchedules({ + await schedule.save('vm-01', { ...claudeConfig, prompt: 'keep-me', enabled: true }) + const result = await schedule.syncSchedules({ scope: 'all', schedule: normalizeScheduleStructure({ frequency: 'hourly', @@ -202,9 +202,9 @@ test('sync overwrites schedule only and creates disabled plans with default dial assert.equal(created.reasoning_effort, 'medium') }) -test('default template persists schedule structure only', () => { +test('default template persists schedule structure only', async () => { const { schedule } = runner() - const saved = schedule.saveDefaultTemplate({ + const saved = await schedule.saveDefaultTemplate({ frequency: 'weekly', time: '10:15', weekdays: [5], @@ -212,7 +212,7 @@ test('default template persists schedule structure only', () => { }) assert.equal(saved.frequency, 'weekly') assert.deepEqual(schedule.getDefaultTemplate(), saved) - schedule.clearDefaultTemplate() + await schedule.clearDefaultTemplate() assert.equal(schedule.getDefaultTemplate(), null) }) @@ -231,12 +231,12 @@ test('unparseable stored cron surfaces parse_error without exposing cron string' assert.equal(view.cron, undefined) }) -test('sync with test copies switch and dialog to every slot of the same kind', () => { +test('sync with test copies switch and dialog to every slot of the same kind', async () => { const codexVm = { id: 'vm-02', platform: 'openai' } const otherClaude = { id: 'vm-03', platform: 'anthropic' } const { settings, schedule } = runner({ list: () => [claudeVm, codexVm, otherClaude] }) - schedule.save('vm-01', { ...claudeConfig, enabled: false }) - schedule.save('vm-02', { + await schedule.save('vm-01', { ...claudeConfig, enabled: false }) + await schedule.save('vm-02', { enabled: false, frequency: 'daily', time: '08:00', @@ -246,7 +246,7 @@ test('sync with test copies switch and dialog to every slot of the same kind', ( max_tokens: null, reasoning_effort: 'low', }) - const result = schedule.syncSchedules({ + const result = await schedule.syncSchedules({ scope: 'all', schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), test: { enabled: true, model: 'claude-haiku-5-5', prompt: 'ping', max_tokens: 32 }, @@ -267,7 +267,7 @@ test('sync with test copies switch and dialog to every slot of the same kind', ( assert.equal(codex.reasoning_effort, 'low') assert.equal(codex.max_tokens, null) - schedule.syncSchedules({ + await schedule.syncSchedules({ scope: 'all', schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), test: { enabled: false, model: 'claude-haiku-5-5', prompt: 'ping', max_tokens: 32 }, @@ -277,21 +277,22 @@ test('sync with test copies switch and dialog to every slot of the same kind', ( } }) -test('sync with an invalid test writes nothing', () => { +test('sync with an invalid test writes nothing', async () => { const { settings, schedule } = runner({ list: () => [claudeVm, { id: 'vm-03', platform: 'anthropic' }] }) - schedule.save('vm-01', { ...claudeConfig, enabled: false }) + await schedule.save('vm-01', { ...claudeConfig, enabled: false }) const before = settings.all() for (const test of [ { enabled: 'yes', model: 'm', prompt: 'p', max_tokens: 1 }, { enabled: true, model: '', prompt: 'p', max_tokens: 1 }, { enabled: true, model: 'm', prompt: 'p' }, ]) { - assert.throws(() => - schedule.syncSchedules({ - scope: 'all', - schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), - test, - }), + await assert.rejects( + async () => + await schedule.syncSchedules({ + scope: 'all', + schedule: normalizeScheduleStructure({ frequency: 'hourly', minute: 0, timezone: 'Asia/Shanghai' }), + test, + }), ) } assert.deepEqual(settings.all(), before) @@ -303,8 +304,12 @@ test('flush waits until a scheduled result finishes persisting', async () => { schedule.started = true let release let entered - const gate = new Promise((resolve) => { release = resolve }) - const writing = new Promise((resolve) => { entered = resolve }) + const gate = new Promise((resolve) => { + release = resolve + }) + const writing = new Promise((resolve) => { + entered = resolve + }) const originalSet = settings.set.bind(settings) settings.set = async (...args) => { entered() @@ -314,7 +319,9 @@ test('flush waits until a scheduled result finishes persisting', async () => { const run = schedule._run('vm-01') await writing let flushed = false - const flush = schedule.flush().then(() => { flushed = true }) + const flush = schedule.flush().then(() => { + flushed = true + }) try { await new Promise((resolve) => setImmediate(resolve)) assert.equal(flushed, false, 'the scheduled result is still being written') @@ -333,8 +340,12 @@ test('flush reports a failed scheduled result write', async () => { const failure = new Error('synthetic scheduled result write failure') let release let entered - const gate = new Promise((resolve) => { release = resolve }) - const writing = new Promise((resolve) => { entered = resolve }) + const gate = new Promise((resolve) => { + release = resolve + }) + const writing = new Promise((resolve) => { + entered = resolve + }) settings.set = async () => { entered() await gate diff --git a/test/unit/weekly-split.test.mjs b/test/unit/weekly-split.test.mjs index 41c894357..2998eb3fc 100644 --- a/test/unit/weekly-split.test.mjs +++ b/test/unit/weekly-split.test.mjs @@ -97,12 +97,12 @@ test('misaligned 7d vs 7d_oi uses the difference as regular usage', () => { assert.equal(s.mode, 'open') }) -test('AccountQuota.weeklySplitOf reads unified windows and config', () => { +test('AccountQuota.weeklySplitOf reads unified windows and config', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-wsplit-')) - const q = new AccountQuota({ + const q = await new AccountQuota({ dataDir: dir, config: { quota: { weekly_split: { enabled: true, fable_share: 0.5 } } }, - }) + }).init() q.ingestHeaders('acc-1', { 'anthropic-ratelimit-unified-7d-utilization': '0.50', 'anthropic-ratelimit-unified-7d_oi-utilization': '0', diff --git a/test/unit/write-queue.test.mjs b/test/unit/write-queue.test.mjs new file mode 100644 index 000000000..3db293572 --- /dev/null +++ b/test/unit/write-queue.test.mjs @@ -0,0 +1,38 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { WriteQueue } from '../../src/lib/pool/write-queue.mjs' + +test('flush waits for ordered writes even when the first write is delayed', async () => { + const queue = new WriteQueue() + const writes = [] + let release + const gate = new Promise((resolve) => { + release = resolve + }) + queue.enqueue(async () => { + await gate + writes.push('first') + }) + queue.enqueue(() => writes.push('second')) + const flushed = queue.flush() + await Promise.resolve() + assert.deepEqual(writes, []) + release() + await flushed + assert.deepEqual(writes, ['first', 'second']) +}) + +test('failed persistence is reported by flush and later writes cannot report success', async () => { + const queue = new WriteQueue() + const failure = Object.assign(new Error('synthetic persistence failure'), { code: 'test_failure' }) + let wroteAfterFailure = false + queue.enqueue(() => { + throw failure + }) + queue.enqueue(() => { + wroteAfterFailure = true + }) + await assert.rejects(queue.flush(), (error) => error === failure) + assert.equal(wroteAfterFailure, false) + await assert.rejects(queue.flush(), (error) => error === failure) +}) From fad3e77a1a2cd2b0151c5de3d97b1808b0e2223f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 12:59:48 +0800 Subject: [PATCH 023/473] test(migration): retain fractional API endpoint priorities --- test/unit/sqlite-import.test.mjs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/unit/sqlite-import.test.mjs b/test/unit/sqlite-import.test.mjs index aa94247af..bc61d5f36 100644 --- a/test/unit/sqlite-import.test.mjs +++ b/test/unit/sqlite-import.test.mjs @@ -85,3 +85,19 @@ test('fractional weights in the preserved legacy runtime table survive import', .get('legacy-fractional') assert.equal(row.weight, 1.25) }) + +test('fractional API endpoint priorities survive import and store mutations', async () => { + const old = sqlite({ dbPath: source }) + old + .prepare('INSERT INTO api_endpoints(id,name,base_url,priority) VALUES(?,?,?,?)') + .run('fractional-endpoint', 'fractional', 'https://example.invalid', 1.25) + old.close() + const db = getDb() + await importSqlite({ source, db }) + const { ApiEndpointStore } = await import('../../src/lib/admin/api-endpoints.mjs') + const store = new ApiEndpointStore({ db }) + assert.equal((await store.getRaw('fractional-endpoint')).priority, 1.25) + const created = await store.create({ base_url: 'https://example.invalid', priority: 2.75 }) + assert.equal(created.priority, 2.75) + assert.equal((await store.update(created.id, { priority: 0.25 })).priority, 0.25) +}) From 71ea797b480ff786c17717740d8d4f7c69059333 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 13:00:18 +0800 Subject: [PATCH 024/473] fix(migration): preserve supported fractional scheduling values --- src/lib/db/postgres-migrations/001_baseline.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/db/postgres-migrations/001_baseline.sql b/src/lib/db/postgres-migrations/001_baseline.sql index 348169650..20978b75f 100644 --- a/src/lib/db/postgres-migrations/001_baseline.sql +++ b/src/lib/db/postgres-migrations/001_baseline.sql @@ -53,7 +53,7 @@ CREATE TABLE "account_runtime_states_legacy" ( "vm_id" text COLLATE "C" NOT NULL, "status" text COLLATE "C" NOT NULL DEFAULT 'unknown', "priority" bigint NOT NULL DEFAULT 0, - "weight" bigint NOT NULL DEFAULT 1, + "weight" double precision NOT NULL DEFAULT 1, "cooldown_until" bigint, "cooldown_reason" text COLLATE "C", "model_states_json" text COLLATE "C", @@ -177,7 +177,7 @@ CREATE TABLE "api_endpoints" ( "base_url" text COLLATE "C" NOT NULL, "headers_json" text COLLATE "C", "disable_cooling" bigint NOT NULL DEFAULT 0, - "priority" bigint NOT NULL DEFAULT 0, + "priority" double precision NOT NULL DEFAULT 0, "created_at" text COLLATE "C", "updated_at" text COLLATE "C", "kind" text COLLATE "C" NOT NULL DEFAULT 'custom', From 4ad444d71e32e938c589c647850a0a795a9b84d1 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 13:00:34 +0800 Subject: [PATCH 025/473] fix(backup): drain scheduler checks before shutdown --- src/lib/admin/backup-service.mjs | 28 +++++++++++++++++++--------- test/unit/backup-service.test.mjs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index ae1da8455..f8b19643e 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -35,6 +35,8 @@ export class BackupService { this._restoreRunning = false this._maintenance = null this._running = false + this._schedulerGeneration = 0 + this._scheduledChecks = new Set() this._timer = null this._bootTimer = null this._rebindStores = null // set via onRestored() @@ -127,7 +129,7 @@ export class BackupService { // ---------- create ---------- async flush() { - await Promise.all([this._pendingBackup, this._pendingRestore].filter(Boolean)) + await Promise.all([...this._scheduledChecks, this._pendingBackup, this._pendingRestore].filter(Boolean)) } async createBackup(options = {}) { @@ -534,15 +536,22 @@ export class BackupService { /** Auto-backup checker: every checkIntervalMs + a catch-up run at boot. */ startScheduler() { this.stopScheduler() - const check = async () => { - try { - if (await this.shouldRunScheduled()) { - const rec = await this.createBackup({ kind: 'scheduled' }) - console.log(`[backup] scheduled backup completed (${Math.round((rec.size_bytes || 0) / 1048576)} MiB)`) + const generation = this._schedulerGeneration + const check = () => { + if (generation !== this._schedulerGeneration) return + const pending = (async () => { + try { + if ((await this.shouldRunScheduled()) && generation === this._schedulerGeneration) { + const rec = await this.createBackup({ kind: 'scheduled' }) + console.log(`[backup] scheduled backup completed (${Math.round((rec.size_bytes || 0) / 1048576)} MiB)`) + } + } catch (e) { + console.warn('[backup] scheduled backup failed:', String(e.message || e)) } - } catch (e) { - console.warn('[backup] scheduled backup failed:', String(e.message || e)) - } + })() + this._scheduledChecks.add(pending) + pending.finally(() => this._scheduledChecks.delete(pending)) + return pending } this._timer = setInterval(check, this.checkIntervalMs) if (typeof this._timer.unref === 'function') this._timer.unref() @@ -560,6 +569,7 @@ export class BackupService { } stopScheduler() { + this._schedulerGeneration += 1 if (this._timer) { clearInterval(this._timer) this._timer = null diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 87ef351fe..351966339 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -276,6 +276,37 @@ test('stopping scheduling drains an in-flight check without starting a backup af assert.deepEqual({ flushedDuringCheck, backups }, { flushedDuringCheck: false, backups: 0 }) }) +test('restarting scheduling does not reactivate a check from the previous schedule', async (t) => { + let check + t.mock.method(globalThis, 'setInterval', (callback) => { + check = callback + return { unref() {} } + }) + t.mock.method(globalThis, 'clearInterval', () => {}) + let releaseCheck + const gate = new Promise((resolve) => { + releaseCheck = resolve + }) + svc.shouldRunScheduled = async () => { + await gate + return true + } + let backups = 0 + svc.createBackup = async () => { + backups += 1 + return { size_bytes: 0 } + } + svc.startScheduler() + const previous = check() + svc.startScheduler() + releaseCheck() + await previous + assert.equal(backups, 0) + await check() + await svc.flush() + assert.equal(backups, 1) +}) + test('remove deletes record + file', async () => { seedVm() const rec = await svc.createBackup({ kind: 'manual' }) From 6aa877de792bbcaa0fd02134c821bf938c295c42 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 13:07:13 +0800 Subject: [PATCH 026/473] fix(backup): distinguish maintenance and isolate unrelated panel scenarios --- src/server.mjs | 4 ++-- test/e2e/panel-api.e2e.test.mjs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/server.mjs b/src/server.mjs index fb1490e7a..1bb90387d 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1119,8 +1119,8 @@ const server = http.createServer(async (req, res) => { 503, makeError({ type: ErrorType.OVERLOADED, - code: 'restore_in_progress', - message: 'Gateway is restoring from backup; retry shortly', + code: backupService.isRestoring ? 'restore_in_progress' : 'maintenance_in_progress', + message: 'Database maintenance in progress; retry shortly', status: 503, }).body, ) diff --git a/test/e2e/panel-api.e2e.test.mjs b/test/e2e/panel-api.e2e.test.mjs index 7ffb26552..2f5174323 100644 --- a/test/e2e/panel-api.e2e.test.mjs +++ b/test/e2e/panel-api.e2e.test.mjs @@ -202,7 +202,8 @@ test('sessionKey import without SOCKS5 is rejected', async () => { }) test('sessionKey import rejected when bound proxy is dead', async () => { - const gw = await startGateway() + // Slow proxy probing must not overlap the independent boot backup timer. + const gw = await startGateway({ env: { KIN_BACKUP_DISABLED: '1' } }) try { const added = await api(gw, 'POST', '/api/panel/proxies/import', { body: { text: '10.9.9.9:1080' }, @@ -304,7 +305,7 @@ test('generate-auth-url then exchange-code writes fake oauth', async () => { }) test('local egress slot generates auth url and exchanges code', async () => { - const gw = await startGateway({ oauth: false }) + const gw = await startGateway({ oauth: false, env: { KIN_BACKUP_DISABLED: '1' } }) try { const bind = await api(gw, 'POST', '/api/panel/proxies/px-local/bind', { body: { vm_id: 'vm-sim-01' } }) assert.equal(bind.status, 200, bind.text) From 3054bfa08500c43f81eeb0a84804c4a63e4bd022 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 13:13:02 +0800 Subject: [PATCH 027/473] test(pg): isolate restores by schema without database checkpoints --- .github/workflows/test.yml | 3 -- test/pg-setup.mjs | 48 ++++++------------------------- test/unit/backup-service.test.mjs | 16 +++++++++++ 3 files changed, 24 insertions(+), 43 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69782f761..aff32a07a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,9 +42,6 @@ jobs: with: go-version: '1.25.x' cache-dependency-path: worker/go.sum - - name: Keep disposable database cleanup immediate - run: | - docker exec ${{ job.services.postgres.id }} psql -U vm2api -d vm2api_test -v ON_ERROR_STOP=1 -c 'ALTER SYSTEM SET checkpoint_completion_target=0' -c 'SELECT pg_reload_conf()' - run: npm ci - name: PostgreSQL 17 backup clients run: | diff --git a/test/pg-setup.mjs b/test/pg-setup.mjs index dd06ca9af..2f561fdf9 100644 --- a/test/pg-setup.mjs +++ b/test/pg-setup.mjs @@ -1,5 +1,5 @@ /** Per-test PostgreSQL isolation for the Node test runner. Never use a deployment URL. */ -import { beforeEach, afterEach, after } from 'node:test' +import { beforeEach, afterEach } from 'node:test' import crypto from 'node:crypto' import pg from 'pg' import { openDatabase, closeDatabase, closeDatabasesForSchema } from '../src/lib/db/database.mjs' @@ -9,43 +9,23 @@ if (!url || !/_test$/.test(new URL(url).pathname)) { throw new Error('tests require VM2API_TEST_DATABASE_URL pointing to a dedicated *_test database') } let schema -let dedicated -let testUrl = url -let fileUrl = url -const backupTest = process.argv.some((arg) => - /(?:^|[/\\])(?:backup-service\.test\.mjs|backup\.e2e\.test\.mjs)$/.test(arg), -) beforeEach(async () => { await closeDatabase() schema = `test_${process.pid}_${crypto.randomBytes(8).toString('hex')}` - testUrl = fileUrl - if (backupTest && !dedicated) { - dedicated = `vm2api_${crypto.randomBytes(8).toString('hex')}_test` - const admin = new pg.Client({ connectionString: url }) - await admin.connect() - try { - await admin.query(`CREATE DATABASE "${dedicated}"`) - } finally { - await admin.end() - } - const target = new URL(url) - target.pathname = '/' + dedicated - testUrl = target.toString() - fileUrl = testUrl - } - process.env.VM2API_TEST_DATABASE_URL = testUrl + // Restore is restricted to this schema. Avoid DROP DATABASE's global checkpoint. + process.env.VM2API_TEST_DATABASE_URL = url process.env.KIN_DATABASE_SCHEMA = schema - process.env.KIN_DATABASE_URL = testUrl - process.env.VM2API_DATABASE_URL = testUrl - const db = await openDatabase({ connectionString: testUrl, schema }) + process.env.KIN_DATABASE_URL = url + process.env.VM2API_DATABASE_URL = url + const db = await openDatabase({ connectionString: url, schema }) const { loadSettingsState } = await import('../src/lib/db/settings-state.mjs') await loadSettingsState(db) }) afterEach(async () => { await closeDatabase() if (!schema) return - await closeDatabasesForSchema(schema, testUrl) - const client = new pg.Client({ connectionString: testUrl }) + await closeDatabasesForSchema(schema, url) + const client = new pg.Client({ connectionString: url }) await client.connect() try { const schemas = (await client.query('SELECT nspname FROM pg_namespace WHERE left(nspname,length($1))=$1', [schema])) @@ -56,15 +36,3 @@ afterEach(async () => { } delete process.env.KIN_DATABASE_SCHEMA }) - -after(async () => { - if (!dedicated) return - const admin = new pg.Client({ connectionString: url }) - await admin.connect() - try { - await admin.query(`DROP DATABASE "${dedicated}" WITH (FORCE)`) - } finally { - await admin.end() - } - process.env.VM2API_TEST_DATABASE_URL = url -}) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 351966339..28709cb93 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -60,6 +60,22 @@ test('tar is available in this environment', () => { assert.equal(tarAvailable(), true) }) +test('restoring the application schema preserves later writes in a neighboring schema', async () => { + const db = getDb() + const neighbor = `"${db.schema}_neighbor"` + await db.exec( + `CREATE SCHEMA ${neighbor}; CREATE TABLE ${neighbor}.sentinel(value text); INSERT INTO ${neighbor}.sentinel VALUES('before')`, + ) + try { + const rec = await svc.createBackup() + await db.exec(`UPDATE ${neighbor}.sentinel SET value='after'`) + await svc.restoreBackup(rec.id) + assert.equal((await db.prepare(`SELECT value FROM ${neighbor}.sentinel`).get()).value, 'after') + } finally { + await db.exec(`DROP SCHEMA ${neighbor} CASCADE`) + } +}) + test('createBackup produces tar.gz with manifest + db + vms + config, records ok', async () => { seedVm() await initVmDbSync(root, { watch: false }) From cf9462d5501c323efa95c254feb3821f7cf656f0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 13:17:56 +0800 Subject: [PATCH 028/473] docs(pg): record migration validation and baseline comparisons --- ...2026-10-21-postgresql-migration-results.md | 80 +++ docs/plans/2026-10-21-postgresql-migration.md | 36 +- .../postgresql/baseline-comparison.json | 656 ++++++++++++++++++ .../evidence/postgresql/verification.json | 100 +++ 4 files changed, 867 insertions(+), 5 deletions(-) create mode 100644 docs/plans/2026-10-21-postgresql-migration-results.md create mode 100644 docs/plans/evidence/postgresql/baseline-comparison.json create mode 100644 docs/plans/evidence/postgresql/verification.json diff --git a/docs/plans/2026-10-21-postgresql-migration-results.md b/docs/plans/2026-10-21-postgresql-migration-results.md new file mode 100644 index 000000000..dc73878f5 --- /dev/null +++ b/docs/plans/2026-10-21-postgresql-migration-results.md @@ -0,0 +1,80 @@ +# PostgreSQL 迁移实施与验证 + +日期：2026-10-21。分支：`codex/postgresql-migration`。 + +运行数据库已改为 PostgreSQL 17。异步数据访问、业务调用链、离线导入、新备份恢复、部署配置和数据库页面已实现。未推送分支，未对现有部署执行数据迁移、配置变更、重启或发布。 + +原始提交：`eee8f6ecb6ca04455e6edef0c8e498fb3be21f21`。运行代码提交：`6aa877de`，测试隔离提交：`3054bfa0`。完整 SHA 见随附 [验证摘要](evidence/postgresql/verification.json)。后续文档提交不改变被验证的运行代码。 + +## 实现范围 + +- `pg.Pool`、原生 `$n` SQL、固定事务连接、savepoint、迁移互斥和校验和。PostgreSQL 建立独立基线；SQLite 的 41 个历史迁移保持原样。 +- repository、用户与密钥、请求日志、计费统计、调度、集群、VM 镜像和面板调用均适配异步访问。同步调度使用已初始化的工作集，写入按序排队，维护和退出前排空。面板在写入成功后发送 JSON 响应。 +- 64 位累计整数在安全范围内返回 Number，超范围返回精确十进制字符串。已失精的 Number 被拒绝。金额、平均值、比例保留原浮点语义。保留现有合法小数权重和端点优先级。 +- 旧 SQLite 通过只读连接取得一致性快照，只在私有副本升级历史版本。空目标检查、逐表摘要、历史表、运行时表、密文、sequence 高水位和完成标记在同一导入事务内处理。 +- PG custom dump 使用导出快照，VM/配置文件在相同暂停写入窗口复制。恢复验证文件摘要、路径及数据库对象范围，以单事务恢复本应用 schema，重新加载内存状态。失败保留维护状态和持久标记。 +- Compose 支持内置或外部 PostgreSQL。应用角色没有超级用户、建库、建角色或复制权限。两种控制面 Dockerfile 安装 PG 17 客户端。升级在替换旧容器前检查连接、版本、已应用迁移前缀、校验和和旧数据导入标记。 + +SQLite 只保留在离线迁移、旧备份转换和历史 fixture 中。运行入口不再依赖 SQLite，也不提供双写或降级路径。 + +VM 数据库记录保留元数据及凭据存在标记。活凭据仍位于既有槽位和 CLI-home 存储中。顶层 JSON 备份及数据库导入不能替代这些目录；切换时必须保留原目录和业务加密密钥。 + +## 验证结果 + +测试只使用本任务创建的 PostgreSQL 17.11 和合成数据。没有读取现有部署的连接或业务数据。 + +| 检查 | 被验证提交 | 实际结果 | +| --- | --- | --- | +| 全量单元测试 | `4ad444d7` | 281 文件；3325 通过、7 失败、10 跳过。2 个失败文件的 7 个断言与原始提交相同 | +| 全量端到端测试 | `3054bfa0` | 26 文件；96 通过、17 失败、1 跳过。11 个失败文件的 17 个断言与原始提交相同，无清理超时 | +| 测试隔离与恢复范围 | `3054bfa0` | 驱动、导入、备份核心/服务/E2E 共 40/40，通过实际恢复确认相邻 schema 的后续写入保留 | +| Node 22 容器兼容性 | `4ad444d7` | 93/93 通过，覆盖驱动、SQLite 导入、大整数、账号额度和重启持久化 | +| 前端测试、构建及格式 | `4ad444d7` | 60 文件、473/473 测试通过；TypeScript/Vite 生产构建和格式检查通过 | +| Go | `4ad444d7` | worker 与 api-kernel 的 race 测试通过 | +| 部署配置 | `4ad444d7` | Python 25/25；PostgreSQL 业务角色的实际认证、DDL 和权限拒绝检查通过 | +| 语法与格式 | `4ad444d7`，末次维护改动另查 | Node source/scripts 语法、Biome、Shell、前端格式及 `git diff --check` 通过 | +| amd64 控制面镜像 | `6aa877de` | 构建通过；实际容器 HTTP、PG 指标、密钥写入、备份和恢复后内存状态检查通过 | + +Node 22、前端、Go 和部署检查后的运行代码只增加普通维护提示修正；该修正由最终端到端测试、独立维护时序复现及最终镜像验证覆盖。 + +全量测试仍有原始提交的失败。独立原始 checkout 使用相同预编译 payload 和测试依赖重跑，并逐项比较测试名称和断言正文。详细清单见 [基线对照](evidence/postgresql/baseline-comparison.json)。没有把这些失败改为跳过或通过，也没有为本次数据库迁移修改相关业务行为。 + +失败输出示例（两份源码结果相同）： + +- `cli-hop-body.test.mjs:34`：实际 `ttl='1h'`，预期 `undefined`。 +- `convert-cache-shape.test.mjs:155`：实际 `{type:'ephemeral', ttl:'1h'}`，预期 `{type:'ephemeral'}`。 +- `auto-mode.e2e.test.mjs:608`：实际 thinking 包含 `display:'summarized'`，预期对象没有该字段。 +- `account-pool-failover.e2e.test.mjs:66`：`TypeError: Cannot read properties of undefined (reading 'account_id')`。 + +第一次最终 E2E 在 `4ad444d7` 有 12/26 文件失败、94 通过、19 失败、1 跳过。其中两项慢代理场景在绑定结束后立即收到 503。独立复现确认：绑定耗时约 10–16 秒，5 秒启动备份定时器进入维护并等待绑定完成；后继请求短时拒绝，约 52–54 毫秒后恢复 200。禁用自动备份时同链请求全部 200。这两项业务 fixture 已显式禁用无关后台备份；独立自动备份 E2E 仍使用默认开启设置。 + +后续全量 E2E 的 5 个备份场景全部通过，但文件清理的 `DROP DATABASE` 等待 PostgreSQL 全实例 checkpoint 超过测试文件 90 秒限制。只读采样为 `IPC/CheckpointDone`、无阻塞连接；独立复现主体 487 毫秒、进程清理总耗时约 109 秒。测试现统一使用专用 `_test` 基库内的独立 schema，移除建库/删库及 CI 的 checkpoint 参数调整。生产恢复始终限定 schema。修正后相关 40 场景 9.8 秒结束。 + +新增回归先于修复独立提交，覆盖异步集群编辑、平台持久化、日志清理、整数精度、面板处理返回值、小数导入、定时探测结果写入，以及备份调度关闭竞态。 + +测试进程的完整日志保留在本地忽略目录 `.tmp/pg-evidence/`。关键文件为 `unit-final.log`、`e2e-complete.log`、`schema-isolation-final.log`、`node22-final.log`、`image-smoke-final.log` 和 `docker-build-final.log`。摘要和页面对照图随分支提交，不包含连接信息或凭据。 + +## 镜像与页面证据 + +最终实际运行镜像 ID：`sha256:d93e36d953db42aa80acd562a2af63cf0a969789d154d063610c0e6e060dc33e`。本地标签：`vm2api:pg-migration-check`。容器内版本为 Node `v22.23.2`、`pg_dump` / `pg_restore` `17.11`。临时容器没有挂载现有数据或宿主 Docker socket，验证结束后删除。 + +本机没有 buildx。使用旧 builder 时，只在临时 Dockerfile 顶部补 `ARG BUILDPLATFORM=linux/amd64`，其余构建指令与仓库 Dockerfile 相同。首次默认网络的 Go 依赖下载超时；使用 host 构建网络重试成功，未修改依赖版本或现有网络配置。 + +数据库页面复用既有组件和样式，用合成数据在 1440×900 窗口对照。用户已授权独立 Playwright。 + +| SQLite 基线 | PostgreSQL 实现 | +| --- | --- | +| ![SQLite](evidence/postgresql/database-sqlite-1440.png) | ![PostgreSQL](evidence/postgresql/database-postgresql-1440.png) | + +## 操作入口与限制 + +部署与切换步骤见 [PostgreSQL 部署说明](../deploy/postgresql.md)。连接和原业务加密密钥通过受控环境提供，不能放到命令参数或报告中。 + +```sh +npm run db:import:sqlite -- --source +npm run db:convert:sqlite-backup -- --source --output --vm-dir --config-dir +``` + +导入目标必须为空。转换旧备份使用专用空转换库，不能指向运行库。旧 `.db.gz` 缺少 VM 和配置文件，需要对应快照。目标开始接受新写入后，原 SQLite 不再是可直接切回的最新副本。 + +ARM64 的 Dockerfile、部署脚本、配置测试和 CI 冒烟入口已更新；本机未执行原生 ARM64 镜像构建及运行。现有环境的数据规模、停服切换、实际槽位凭据、真实外部上游、生产备份恢复和发布均未验证。 diff --git a/docs/plans/2026-10-21-postgresql-migration.md b/docs/plans/2026-10-21-postgresql-migration.md index 555cf5fda..556b5e28f 100644 --- a/docs/plans/2026-10-21-postgresql-migration.md +++ b/docs/plans/2026-10-21-postgresql-migration.md @@ -3,11 +3,11 @@ 日期：2026-10-21（北京时间） 调研基线：`eee8f6ecb6ca04455e6edef0c8e498fb3be21f21` 实施分支：`codex/postgresql-migration` -状态：计划已制定，业务代码尚未迁移；各任务完成后填写实际提交和验证结果。 +状态：迁移实现已完成；全量回归与原始提交的失败对照见 [实施与验证报告](2026-10-21-postgresql-migration-results.md)。 ## 1. 目标与边界 -将控制面的运行数据库统一为 PostgreSQL，包括用户、API 密钥、分组、账号、调度状态、代理、集群节点、虚拟机凭据镜像、请求日志、计费、备份记录和思考签名归属。 +将控制面的运行数据库统一为 PostgreSQL，包括用户、API 密钥、分组、账号、调度状态、代理、集群节点、虚拟机元数据镜像、请求日志、计费、备份记录和思考签名归属。 应用运行、单元测试、端到端测试、运维查询和新备份都使用 PostgreSQL。SQLite 仅用于离线读取旧数据库、构造历史迁移测试数据及转换旧备份。运行时不保留双写或 SQLite 降级路径。 @@ -24,7 +24,7 @@ | 版本迁移 | `src/lib/db/migrations/001_init.sql` 至 `041_usage_logs_inbound_device.sql`，逐文件事务和 SHA-256 校验 | PostgreSQL 建立独立迁移序列；原 SQLite 文件保持不变 | | 最终表结构 | 在临时 SQLite 执行 41 个迁移后，有 29 张非 SQLite 内部表，包含 `schema_migrations` 和 3 张 `_legacy` 表 | 建立逐表清单，核对列、索引、默认值、外键及保留数据 | | 额外表 | `src/lib/pool/thinking-owner.mjs` 在运行时创建 `thinking_signature_owners` | 纳入 PostgreSQL 正式迁移与存量导入 | -| VM 凭据镜像 | `src/lib/vm/vm-db-sync.mjs` 通过写钩子、启动校对及文件监听镜像到数据库 | 异步写入必须可等待、可排空，删除顺序不能改变 | +| VM 元数据镜像 | `src/lib/vm/vm-db-sync.mjs` 通过写钩子、启动校对及文件监听镜像到数据库；`VmsRepo` 移除活凭据，只保留元数据和凭据存在标记 | 异步写入必须可等待、可排空，删除顺序不能改变 | | 备份 | `src/lib/admin/backup-core.mjs` 使用 `VACUUM INTO`，归档含 `db/kin.db`、VM 和配置文件 | 更换数据库快照格式，保留原归档的其他内容 | | 恢复 | `backup-service.mjs` 关闭 SQLite、替换文件、重新绑定 store | PostgreSQL 恢复需维护模式、排空写入及恢复后重新加载 | | 恢复入口隔离 | `src/server.mjs` 的恢复标志目前主要拦截推理入口 | 需要同时控制面板和后台数据库写入 | @@ -147,7 +147,7 @@ T2/T3 可按文件范围并行。T5 在数据库契约稳定后并行。T4 与 T 选 PostgreSQL 17 作为首次实现和 CI 的固定主版本，匹配相同主版本客户端。使用专用本地测试数据库和合成数据，不读取现有部署的凭据或业务数据。测试配置独立于生产配置，名字与连接目标都必须通过 `_test` / 专用测试库保护检查。 -普通测试文件和派生网关实例有独立 schema；测试辅助工具向所有连接和子进程传播固定 search_path，并在结束后关闭连接再删除对应 schema。备份/恢复、数据库关闭和恢复维护模式测试使用独立的专用 `_test` 数据库，应用 schema 与归档保持一致。`pg_restore` 不承担跨 schema 重命名。测试用账号的建库权限仅限隔离测试环境，不作为生产账号要求。只清理由本测试实例创建的 schema/数据库。集成测试缺少 PG 时明确失败或由单独命令报告缺失，不静默改用 SQLite。 +所有测试只连接预建的专用 `_test` 数据库。每个测试和派生网关使用独立 schema；测试辅助工具向所有连接和子进程传播固定 search_path，并在结束后关闭连接再删除本测试的 schema。备份恢复显式限定应用 schema，应用 schema 与归档保持一致，`pg_restore` 不承担跨 schema 重命名。通过相邻 schema 的 sentinel 恢复测试验证范围。测试辅助工具不创建或删除数据库，不需要建库权限。集成测试缺少 PG 时明确失败，不静默改用 SQLite。 ### 5.2 必须通过的场景 @@ -200,7 +200,7 @@ node --test test/unit/database.test.mjs test/unit/db-migration-sub2api.test.mjs 使用临时目录执行全部现有迁移并只读查询 schema：41 个迁移，29 张非 SQLite 内部表。临时目录已删除；未读取现有业务数据库。 -本机 Node.js 为 `v24.19.0`。调研时 `psql`、`postgres`、`initdb` 不在 PATH；Docker 查询返回 `permission denied while trying to connect to the docker API at unix:///var/run/docker.sock`。PostgreSQL 运行、迁移、备份恢复及业务验证均未执行。实施前需提供有权限的隔离测试环境，不能据此推断 PostgreSQL 代码或业务失败。 +本机 Node.js 为 `v24.19.0`。调研阶段 `psql`、`postgres`、`initdb` 不在 PATH；Docker 首次查询返回 `permission denied while trying to connect to the docker API at unix:///var/run/docker.sock`。实施阶段已通过工具权限审核创建本任务专用 PostgreSQL 17 容器，并取得匹配的客户端。该容器只含合成数据，不使用现有部署的连接或业务数据。 参考官方资料： @@ -211,3 +211,29 @@ node --test test/unit/database.test.mjs test/unit/db-migration-sub2api.test.mjs - [PostgreSQL：Debian PGDG 客户端安装](https://www.postgresql.org/download/linux/debian/) 计划已由两个只读审查任务分别核对数据层和运维/测试范围。已补充 schema 备份隔离、升级前检查、导出快照同步、客户端主版本、LIKE 语义、大整数源读取和自增高水位要求。该审查仅验证计划覆盖与源码事实，不代表迁移实现已经通过验证。 + +## 7. 实施记录 + +- `fa0decc2`：PostgreSQL 驱动、独立基线和离线 SQLite 模块。 +- `3225b073`：基础 repository 与异步 store。 +- `893cb3ce`：日志、统计和计费查询。 +- `720a0474`：只读 SQLite 快照导入、事务校验和旧备份转换。 +- `1a5a7520`：导出快照备份、受控恢复和失败后维护状态。 +- `c2e56a2d`：PostgreSQL 部署、升级预检、指标与页面。 +- `38f6db6e`：启动、请求、调度、VM 镜像、恢复与关闭的异步调用链。 +- `3fbdc4bc`：独立 PG 测试 schema、CI 和重启持久化验证。 +- `6cb132b6`、`71ea797b`：整数精度及合法小数权重、端点优先级修正。 +- `4ad444d7`：关闭时等待已进入的备份调度检查。 +- `6aa877de`：区分恢复与普通维护提示，隔离慢代理测试的自动备份时序干扰。 + +具体实施时采用 `AsyncLocalStorage` 保持事务连接，savepoint 回滚同步丢弃未提交的缓存更新。保留 SQLite 字符串排序和 ASCII 大小写语义。SQLite 中声明为 INTEGER 的 `accounts.load_factor` 实际接受小数，PG 对应列使用 double precision，已用 1.25 权重导入回归验证。累计整数超出安全范围时以精确十进制字符串返回；余额、费用、平均值和比率保留原 double precision 语义。 + +VM 活凭据继续保留在既有槽位和 CLI-home 存储中。数据库导入和顶层 JSON 归档不能重建这些凭据。迁移操作必须原样保留对应目录及原业务加密密钥。 + +页面对照使用合成指标、相同组件依赖与样式、1440×900 窗口。用户已授权独立 Playwright 截图： + +| 原 SQLite 页面 | PostgreSQL 页面 | +| --- | --- | +| ![SQLite 原页面](evidence/postgresql/database-sqlite-1440.png) | ![PostgreSQL 页面](evidence/postgresql/database-postgresql-1440.png) | + +验证对象、实际结果和未验证项记录在同目录实施报告中。未对现有部署执行迁移、重启或发布，未推送分支。 diff --git a/docs/plans/evidence/postgresql/baseline-comparison.json b/docs/plans/evidence/postgresql/baseline-comparison.json new file mode 100644 index 000000000..60a841133 --- /dev/null +++ b/docs/plans/evidence/postgresql/baseline-comparison.json @@ -0,0 +1,656 @@ +{ + "verified_commit": "3054bfa0", + "production_code_commit": "6aa877de", + "verification_started_commit": "4ad444d7", + "baseline_commit": "eee8f6e", + "normalization": "Exact error/assertion body comparison after removing blank lines and stack frames; timing is outside the compared body.", + "suites": { + "unit": { + "verified_commit": "4ad444d7", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/unit-final.log", + "log_complete": true, + "assertion_failed_file_count": 2, + "runner_summary": [ + "2", + "281" + ], + "infrastructure_failures": [], + "failed_file_count": 2, + "failed_test_count": 7, + "assertion_failed_test_count": 7, + "runner_failed_record_count": 7, + "same_assertion_count": 7, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": true, + "files": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-cli-hop-body.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "test_line": 16, + "current_log_line": 767, + "baseline_log_line": 38, + "same_assertion": true, + "assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181", + "baseline_assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181" + }, + { + "name": "every cli-hop system becomes the identity plus the caller body", + "test_line": 41, + "current_log_line": 786, + "baseline_log_line": 57, + "same_assertion": true, + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b" + }, + { + "name": "cli-hop normalizes mixed caller markers to the official template", + "test_line": 63, + "current_log_line": 811, + "baseline_log_line": 82, + "same_assertion": true, + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b" + }, + { + "name": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "test_line": 357, + "current_log_line": 836, + "baseline_log_line": 107, + "same_assertion": true, + "assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11", + "baseline_assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11" + }, + { + "name": "cli-hop short history only marks the last message", + "test_line": 381, + "current_log_line": 861, + "baseline_log_line": 132, + "same_assertion": true, + "assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d", + "baseline_assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d" + }, + { + "name": "cli-hop uses the inbound session TTL on both message anchors", + "test_line": 421, + "current_log_line": 891, + "baseline_log_line": 162, + "same_assertion": true, + "assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca", + "baseline_assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca" + } + ] + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-convert-cache-shape.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "openai.chat multi-turn cli-hop marks last and penultimate user", + "test_line": 141, + "current_log_line": 1416, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b" + } + ] + } + ] + }, + "e2e": { + "verified_commit": "3054bfa0", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/e2e-complete.log", + "log_complete": true, + "assertion_failed_file_count": 11, + "runner_summary": [ + "11", + "26" + ], + "infrastructure_failures": [], + "failed_file_count": 11, + "failed_test_count": 17, + "assertion_failed_test_count": 17, + "runner_failed_record_count": 17, + "same_assertion_count": 17, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": true, + "files": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-account-pool-failover.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "test_line": 10, + "current_log_line": 15, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "baseline_assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811" + } + ] + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-auto-mode.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "test_line": 497, + "current_log_line": 54, + "baseline_log_line": 18, + "same_assertion": true, + "assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "baseline_assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15" + } + ] + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-completions.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/completions old OpenAI prompt format", + "test_line": 8, + "current_log_line": 116, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-credentials.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "reset destroys home and recreates the same slot without oauth", + "test_line": 70, + "current_log_line": 167, + "baseline_log_line": 16, + "same_assertion": true, + "assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "baseline_assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477" + } + ] + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-crs-default.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "default POST /v1/messages uses Go worker pool, not CLI", + "test_line": 8, + "current_log_line": 198, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-errors.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "unknown model is rejected without hop", + "test_line": 32, + "current_log_line": 231, + "baseline_log_line": 17, + "same_assertion": true, + "assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "baseline_assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3" + } + ] + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-health.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "harness /health 200 and honest capabilities", + "test_line": 5, + "current_log_line": 262, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "baseline_assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb" + } + ] + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hermes.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "test_line": 15, + "current_log_line": 292, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "baseline_assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390" + }, + { + "name": "Hermes system blob without UA still classifies as hermes", + "test_line": 53, + "current_log_line": 355, + "baseline_log_line": 77, + "same_assertion": true, + "assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "baseline_assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b" + } + ] + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hop-meta.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "CRS default keeps thinking + max_tokens + temperature in official body", + "test_line": 8, + "current_log_line": 431, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + }, + { + "name": "CRS keeps a long caller system in the structured official prompt", + "test_line": 31, + "current_log_line": 449, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "baseline_assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027" + }, + { + "name": "legacy CLI selector cannot drop HTTP request parameters", + "test_line": 61, + "current_log_line": 467, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + } + ] + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-messages.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "HTTP worker preserves thinking, sampling and max_tokens", + "test_line": 114, + "current_log_line": 524, + "baseline_log_line": 20, + "same_assertion": true, + "assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "baseline_assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b" + }, + { + "name": "HTTP path preserves long system without CLI argv truncation", + "test_line": 189, + "current_log_line": 548, + "baseline_log_line": 44, + "same_assertion": true, + "assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "baseline_assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049" + } + ] + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-openai-responses.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/responses non-stream shape: object + output_text + usage", + "test_line": 7, + "current_log_line": 593, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + }, + { + "name": "POST /v1/responses stream includes [DONE]", + "test_line": 32, + "current_log_line": 611, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "baseline_assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833" + }, + { + "name": "POST /v1/responses tools → function_call in output", + "test_line": 52, + "current_log_line": 629, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + } + ] + } + ] + } + }, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": true, + "previous_attempts": [ + { + "verified_commit": "4ad444d7", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/e2e-final.log", + "failed_file_count": 12, + "failed_test_count": 19, + "baseline_matched_file_count": 11, + "baseline_matched_test_count": 17, + "additional_failures": [ + { + "file": "test/e2e/panel-api.e2e.test.mjs", + "name": "sessionKey import rejected when bound proxy is dead", + "actual_status": 503, + "expected_status": 200, + "classification": "scheduled_backup_maintenance_interference" + }, + { + "file": "test/e2e/panel-api.e2e.test.mjs", + "name": "local egress slot generates auth url and exchanges code", + "actual_status": 503, + "expected_status": 200, + "classification": "scheduled_backup_maintenance_interference" + } + ], + "diagnostic_logs": [ + "/tmp/vm2api-pg-implementation/panel-maintenance-diagnostic.log", + "/tmp/vm2api-pg-implementation/panel-local-maintenance-diagnostic.log" + ] + }, + { + "verified_commit": "6aa877de", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/e2e-final-isolated.log", + "log_complete": true, + "assertion_failed_file_count": 11, + "runner_summary": [ + "12", + "26" + ], + "infrastructure_failures": [ + { + "file": "test/e2e/backup.e2e.test.mjs", + "name": "test/e2e/backup.e2e.test.mjs", + "current_log_line": 96, + "classification": "file_level_timeout_after_business_cases_passed", + "cause": "under_investigation", + "error": " 'Promise resolution is still pending but the event loop has already resolved'" + } + ], + "failed_file_count": 11, + "failed_test_count": 17, + "assertion_failed_test_count": 17, + "runner_failed_record_count": 18, + "same_assertion_count": 17, + "all_assertion_failures_match_baseline": true, + "all_failures_match_baseline": false, + "files": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-account-pool-failover.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "test_line": 10, + "current_log_line": 15, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "baseline_assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811" + } + ] + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-auto-mode.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "test_line": 497, + "current_log_line": 54, + "baseline_log_line": 18, + "same_assertion": true, + "assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "baseline_assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15" + } + ] + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-completions.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/completions old OpenAI prompt format", + "test_line": 8, + "current_log_line": 128, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-credentials.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "reset destroys home and recreates the same slot without oauth", + "test_line": 70, + "current_log_line": 179, + "baseline_log_line": 16, + "same_assertion": true, + "assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "baseline_assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477" + } + ] + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-crs-default.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "default POST /v1/messages uses Go worker pool, not CLI", + "test_line": 8, + "current_log_line": 210, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4" + } + ] + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-errors.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "unknown model is rejected without hop", + "test_line": 32, + "current_log_line": 243, + "baseline_log_line": 17, + "same_assertion": true, + "assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "baseline_assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3" + } + ] + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-health.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "harness /health 200 and honest capabilities", + "test_line": 5, + "current_log_line": 274, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "baseline_assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb" + } + ] + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hermes.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "test_line": 15, + "current_log_line": 304, + "baseline_log_line": 14, + "same_assertion": true, + "assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "baseline_assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390" + }, + { + "name": "Hermes system blob without UA still classifies as hermes", + "test_line": 53, + "current_log_line": 367, + "baseline_log_line": 77, + "same_assertion": true, + "assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "baseline_assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b" + } + ] + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-hop-meta.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "CRS default keeps thinking + max_tokens + temperature in official body", + "test_line": 8, + "current_log_line": 443, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + }, + { + "name": "CRS keeps a long caller system in the structured official prompt", + "test_line": 31, + "current_log_line": 461, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "baseline_assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027" + }, + { + "name": "legacy CLI selector cannot drop HTTP request parameters", + "test_line": 61, + "current_log_line": 479, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4" + } + ] + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-messages.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "HTTP worker preserves thinking, sampling and max_tokens", + "test_line": 114, + "current_log_line": 536, + "baseline_log_line": 20, + "same_assertion": true, + "assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "baseline_assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b" + }, + { + "name": "HTTP path preserves long system without CLI argv truncation", + "test_line": 189, + "current_log_line": 560, + "baseline_log_line": 44, + "same_assertion": true, + "assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "baseline_assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049" + } + ] + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "baseline_log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/baseline-openai-responses.e2e.test.mjs.log", + "same_assertion": true, + "tests": [ + { + "name": "POST /v1/responses non-stream shape: object + output_text + usage", + "test_line": 7, + "current_log_line": 605, + "baseline_log_line": 15, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + }, + { + "name": "POST /v1/responses stream includes [DONE]", + "test_line": 32, + "current_log_line": 623, + "baseline_log_line": 33, + "same_assertion": true, + "assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "baseline_assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833" + }, + { + "name": "POST /v1/responses tools → function_call in output", + "test_line": 52, + "current_log_line": 641, + "baseline_log_line": 51, + "same_assertion": true, + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7" + } + ] + } + ] + } + ], + "test_isolation_fix": { + "commit": "3054bfa0", + "log": "/home/lb/IdeaProjects/vm2api/.tmp/pg-evidence/schema-isolation-final.log", + "log_sha256": "95aa62bbd19cb45c9692e7085220cfafb460b7249bfaf7d969f02282b356e3d3", + "tests": 40, + "passed": 40, + "failed": 0, + "scope": "Dedicated *_test database with per-test schemas; adjacent schema sentinel retained and writable after restore." + } +} diff --git a/docs/plans/evidence/postgresql/verification.json b/docs/plans/evidence/postgresql/verification.json new file mode 100644 index 000000000..f4b06b0f5 --- /dev/null +++ b/docs/plans/evidence/postgresql/verification.json @@ -0,0 +1,100 @@ +{ + "baseline_commit": "eee8f6ecb6ca04455e6edef0c8e498fb3be21f21", + "production_code_commit": "6aa877de792bbcaa0fd02134c821bf938c295c42", + "final_test_commit": "3054bfa08500c43f81eeb0a84804c4a63e4bd022", + "database": "PostgreSQL 17.11", + "unit": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "tests": 3342, + "pass": 3325, + "fail": 7, + "skipped": 10, + "cancelled": 0, + "failed_files": 2, + "files": 281, + "log": ".tmp/pg-evidence/unit-final.log" + }, + "e2e": { + "verified_commit": "3054bfa08500c43f81eeb0a84804c4a63e4bd022", + "tests": 114, + "pass": 96, + "fail": 17, + "skipped": 1, + "cancelled": 0, + "failed_files": 11, + "files": 26, + "log": ".tmp/pg-evidence/e2e-complete.log" + }, + "node22": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "tests": 93, + "pass": 93, + "fail": 0, + "skipped": 0, + "cancelled": 0, + "log": ".tmp/pg-evidence/node22-final.log" + }, + "schema_isolation": { + "verified_commit": "3054bfa08500c43f81eeb0a84804c4a63e4bd022", + "tests": 40, + "pass": 40, + "fail": 0, + "skipped": 0, + "cancelled": 0, + "log": ".tmp/pg-evidence/schema-isolation-final.log" + }, + "web": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "files": 60, + "tests": 473, + "pass": 473, + "fail": 0, + "build": "pass", + "format": "pass" + }, + "go": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "worker_race": "pass", + "api_kernel_race": "pass" + }, + "deployment": { + "verified_commit": "4ad444d71e32e938c589c647850a0a795a9b84d1", + "python_tests": 25, + "pass": 25, + "postgres_role_smoke": "pass" + }, + "image": { + "verified_commit": "6aa877de792bbcaa0fd02134c821bf938c295c42", + "result": "PASS", + "image": "sha256:d93e36d953db42aa80acd562a2af63cf0a969789d154d063610c0e6e060dc33e", + "versions": [ + "v22.23.2", + "pg_dump (PostgreSQL) 17.11 (Debian 17.11-1.pgdg12+2)", + "pg_restore (PostgreSQL) 17.11 (Debian 17.11-1.pgdg12+2)" + ], + "checks": [ + "HTTP health", + "PostgreSQL metrics", + "API key mutations", + "snapshot backup", + "restore and memory rebind" + ] + }, + "screenshots": { + "viewport": { + "width": 1440, + "height": 900 + }, + "baseline": "database-sqlite-1440.png", + "implementation": "database-postgresql-1440.png", + "data": "synthetic" + }, + "limitations": [ + "Existing baseline failures remain; see baseline-comparison.json", + "Native ARM64 build/runtime not performed locally", + "No existing deployment, live credentials, customer data, or external inference upstream was migrated or tested" + ], + "git": { + "pushed": false + } +} From bbe124a51293c0518508ed9c5af2d6c148d1b084 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 14:45:54 +0800 Subject: [PATCH 029/473] test(k8s): specify readiness and bounded WebSocket draining --- test/unit/control-plane-lifecycle.test.mjs | 61 ++++++++++++++++++++++ test/unit/responses-ws.test.mjs | 59 +++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 test/unit/control-plane-lifecycle.test.mjs diff --git a/test/unit/control-plane-lifecycle.test.mjs b/test/unit/control-plane-lifecycle.test.mjs new file mode 100644 index 000000000..cf8dca674 --- /dev/null +++ b/test/unit/control-plane-lifecycle.test.mjs @@ -0,0 +1,61 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' +import { controlPlaneMode, shutdownTimeoutMs } from '../../src/lib/core/control-plane-mode.mjs' +import { ControlPlaneLifecycle, withTimeout } from '../../src/lib/core/control-plane-lifecycle.mjs' + +test('Kubernetes mode is explicit and shutdown timeout accepts positive integer milliseconds', () => { + assert.equal(controlPlaneMode({}), 'standalone') + assert.equal(controlPlaneMode({ VM2API_CONTROL_PLANE_MODE: 'kubernetes' }), 'kubernetes') + assert.throws(() => controlPlaneMode({ VM2API_CONTROL_PLANE_MODE: 'kubernete' }), /CONTROL_PLANE_MODE/) + assert.equal(shutdownTimeoutMs({}), 60_000) + assert.equal(shutdownTimeoutMs({ VM2API_SHUTDOWN_TIMEOUT_MS: '300' }), 300) + assert.throws(() => shutdownTimeoutMs({ VM2API_SHUTDOWN_TIMEOUT_MS: '-1' }), /SHUTDOWN_TIMEOUT_MS/) +}) + +test('readiness requires initialization, database, control lock, and no maintenance or drain', async () => { + let database = true + let locked = true + let maintaining = false + const lifecycle = new ControlPlaneLifecycle({ + probe: async () => { if (!database) throw new Error('private connection details') }, + lockHealthy: () => locked, + isMaintaining: () => maintaining, + }) + assert.equal(await lifecycle.isReady(), false) + lifecycle.markReady() + assert.equal(await lifecycle.isReady(), true) + maintaining = true + assert.equal(await lifecycle.isReady(), false) + maintaining = false + locked = false + assert.equal(await lifecycle.isReady(), false) + locked = true + database = false + assert.equal(await lifecycle.isReady(), false) + database = true + lifecycle.beginDrain() + assert.equal(await lifecycle.isReady(), false) +}) + +test('HTTP drain waits for both the handler and the response to finish', async () => { + const lifecycle = new ControlPlaneLifecycle() + const req = new EventEmitter() + const res = new EventEmitter() + const handlerFinished = lifecycle.trackHttp(req, res) + let drained = false + const pending = lifecycle.waitForIdle().then(() => { drained = true }) + handlerFinished() + await new Promise((resolve) => setTimeout(resolve, 15)) + assert.equal(drained, false, 'a returned SSE handler still owns its open response') + res.emit('finish') + await pending + assert.equal(drained, true) +}) + +test('readiness and draining waits have bounded deadlines', async () => { + const lifecycle = new ControlPlaneLifecycle({ probe: () => new Promise(() => {}), probeTimeoutMs: 20 }) + lifecycle.markReady() + assert.equal(await lifecycle.isReady(), false) + await assert.rejects(withTimeout(new Promise(() => {}), 20), { code: 'shutdown_timeout' }) +}) diff --git a/test/unit/responses-ws.test.mjs b/test/unit/responses-ws.test.mjs index 13ab88ac4..6d228e517 100644 --- a/test/unit/responses-ws.test.mjs +++ b/test/unit/responses-ws.test.mjs @@ -53,6 +53,7 @@ async function listen({ routing, requireAuth, handleProtocol, maxBodyBytes }) { await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() return { + handler, server, port, httpHits, @@ -953,3 +954,61 @@ test('unexpected handler failures do not expose internal exception details', asy await live.close() } }) + +test('draining closes idle Responses sockets and rejects new upgrades', async () => { + const live = await listen({ routing: ENABLED, requireAuth: allowAuth, handleProtocol: async () => {} }) + const opened = await openWs(live.url) + try { + const closed = once(opened.ws, 'close') + await live.handler.drain() + assert.equal((await closed)[0], 1001) + const rejected = await openWs(live.url) + assert.equal(rejected.status, 503) + assert.equal(rejected.body.error.code, 'control_plane_draining') + } finally { + opened.ws.terminate() + await live.close() + } +}) + +test('draining waits for an active Responses hop and refuses the next request', async () => { + let started + let release + let hops = 0 + const running = new Promise((resolve) => { started = resolve }) + const gate = new Promise((resolve) => { release = resolve }) + const live = await listen({ + routing: ENABLED, + requireAuth: allowAuth, + async handleProtocol(_req, res) { + hops += 1 + started() + await gate + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.end('data: {"type":"response.completed","response":{"id":"drained","output":[]}}\n\n') + }, + }) + const opened = await openWs(live.url) + try { + opened.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'first' })) + await running + let drained = false + const pending = live.handler.drain().then(() => { drained = true }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(drained, false) + const rejected = nextFrame(opened.ws, (frame) => frame.error?.code === 'control_plane_draining') + opened.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'second' })) + await rejected + const completed = nextFrame(opened.ws, (frame) => frame.type === 'response.completed') + const closed = once(opened.ws, 'close') + release() + assert.equal((await completed).response.id, 'drained') + await pending + assert.equal((await closed)[0], 1001) + assert.equal(hops, 1) + } finally { + release() + opened.ws.terminate() + await live.close() + } +}) From d157fd4c4d49fcf11522bf719c1e1a83dacc2853 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 14:48:49 +0800 Subject: [PATCH 030/473] test(kubernetes): cover readiness session persistence and shutdown deadlines --- .../e2e/kubernetes-control-plane.e2e.test.mjs | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 test/e2e/kubernetes-control-plane.e2e.test.mjs diff --git a/test/e2e/kubernetes-control-plane.e2e.test.mjs b/test/e2e/kubernetes-control-plane.e2e.test.mjs new file mode 100644 index 000000000..c70b5c3e6 --- /dev/null +++ b/test/e2e/kubernetes-control-plane.e2e.test.mjs @@ -0,0 +1,186 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import crypto from 'node:crypto' +import { startGateway, api } from '../harness.mjs' + +const mode = { VM2API_CONTROL_PLANE_MODE: 'kubernetes', KIN_BACKUP_DISABLED: '1' } + +function exitOf(child) { + if (child.exitCode != null) return Promise.resolve(child.exitCode) + return new Promise((resolve) => child.once('exit', resolve)) +} + +async function slowMessages(gw) { + const payload = Buffer.from( + JSON.stringify({ + model: 'claude-haiku-4-5-20251001', + max_tokens: 8, + stream: true, + messages: [{ role: 'user', content: 'finish the accepted request during shutdown' }], + }), + ) + let request + const response = new Promise((resolve, reject) => { + request = http.request( + `${gw.baseUrl}/v1/messages`, + { + method: 'POST', + agent: false, + headers: { + authorization: `Bearer ${gw.apiKey}`, + 'content-type': 'application/json', + 'content-length': payload.length, + }, + }, + (res) => { + let text = '' + res.on('data', (chunk) => { + text += chunk + }) + res.on('end', () => resolve({ status: res.statusCode, text })) + res.on('error', reject) + }, + ) + request.on('error', reject) + }) + response.catch(() => {}) + request.flushHeaders() + const split = Math.floor(payload.length / 2) + request.write(payload.subarray(0, split)) + await new Promise((resolve) => setTimeout(resolve, 100)) + return { request, response, finish: () => request.end(payload.subarray(split)) } +} + +test('Kubernetes control-plane lock rejects a second process before initialization and releases on exit', async () => { + const first = await startGateway({ env: mode }) + const project = first.project + let unexpectedSecond + try { + assert.equal((await fetch(`${first.baseUrl}/readyz`)).status, 200) + await assert.rejects(async () => { + unexpectedSecond = await startGateway({ project, env: mode }) + }, /control.plane.*(?:already|held|active|in.use)/i) + assert.equal((await fetch(`${first.baseUrl}/readyz`)).status, 200) + } finally { + await unexpectedSecond?.stop() + await first.stop() + } + const replacement = await startGateway({ project, env: mode }) + try { + assert.equal((await fetch(`${replacement.baseUrl}/readyz`)).status, 200) + } finally { + await replacement.stop() + } +}) + +test('Kubernetes panel sessions survive process replacement and logout revokes the stored token', async () => { + let gw = await startGateway({ env: mode }) + const project = gw.project + let cookie + try { + const login = await fetch(`${gw.baseUrl}/api/panel/login`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'admin', password: 'testpass' }), + }) + assert.equal(login.status, 200) + cookie = login.headers.get('set-cookie').split(';')[0] + assert.equal((await fetch(`${gw.baseUrl}/api/panel/me`, { headers: { cookie } })).status, 200) + const token = decodeURIComponent(cookie.slice(cookie.indexOf('=') + 1)) + const stored = await gw.db.prepare('SELECT token_hash FROM panel_sessions WHERE username=$1').all('admin') + assert.deepEqual( + stored.map((row) => row.token_hash), + [crypto.createHash('sha256').update(token).digest('hex')], + ) + } finally { + await gw.stop() + } + gw = await startGateway({ project, env: mode }) + try { + assert.equal((await fetch(`${gw.baseUrl}/api/panel/me`, { headers: { cookie } })).status, 200) + assert.equal((await fetch(`${gw.baseUrl}/api/panel/logout`, { method: 'POST', headers: { cookie } })).status, 200) + assert.equal((await fetch(`${gw.baseUrl}/api/panel/me`, { headers: { cookie } })).status, 401) + } finally { + await gw.stop() + } +}) + +test('readiness rejects backup maintenance while liveness and accepted SSE requests continue', async () => { + const gw = await startGateway({ env: mode }) + let slow + let backup + try { + slow = await slowMessages(gw) + backup = api(gw, 'POST', '/api/panel/backups') + const deadline = Date.now() + 3000 + let ready + do { + ready = await fetch(`${gw.baseUrl}/readyz`) + if (ready.status === 503) break + await new Promise((resolve) => setTimeout(resolve, 10)) + } while (Date.now() < deadline) + assert.equal(ready.status, 503) + assert.equal((await fetch(`${gw.baseUrl}/healthz`)).status, 200) + slow.finish() + const streamed = await slow.response + assert.equal(streamed.status, 200) + assert.match(streamed.text, /message_stop/) + assert.equal((await backup).status, 201) + assert.equal((await fetch(`${gw.baseUrl}/readyz`)).status, 200) + } finally { + slow?.request.destroy() + await backup?.catch(() => {}) + await gw.stop() + } +}) + +test('SIGTERM waits for an accepted request to finish streaming and persist its log', async () => { + const gw = await startGateway({ env: mode }) + let slow + try { + slow = await slowMessages(gw) + const exit = exitOf(gw.child) + gw.child.kill('SIGTERM') + await new Promise((resolve) => setTimeout(resolve, 75)) + assert.equal(gw.child.exitCode, null) + slow.finish() + const streamed = await slow.response + assert.equal(streamed.status, 200) + assert.match(streamed.text, /message_stop/) + assert.equal(await exit, 0) + assert.equal((await gw.db.prepare('SELECT count(*) AS n FROM usage_logs WHERE status=200').get()).n, 1) + } finally { + slow?.request.destroy() + await gw.stop() + } +}) + +test('shutdown deadline interrupts an unfinished request and exits unsuccessfully', async () => { + const gw = await startGateway({ env: { ...mode, VM2API_SHUTDOWN_TIMEOUT_MS: '300' } }) + let slow + try { + slow = await slowMessages(gw) + const exit = exitOf(gw.child) + const started = Date.now() + gw.child.kill('SIGTERM') + let timeout + try { + await Promise.race([ + (async () => { + await assert.rejects(slow.response) + assert.equal(await exit, 1) + })(), + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error('shutdown deadline was not enforced')), 6000) + }), + ]) + } finally { + clearTimeout(timeout) + } + assert.ok(Date.now() - started < 6000, 'shutdown must have a bounded deadline') + } finally { + slow?.request.destroy() + await gw.stop() + } +}) From 17fee3b7b4881bec3adf2db48e42ea6700c13583 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 14:50:46 +0800 Subject: [PATCH 031/473] feat(db): guard a single control plane and persist panel sessions --- src/lib/db/control-plane-lock.mjs | 114 ++++++++++++++++++ .../002_control_plane_sessions.sql | 9 ++ src/lib/db/repos/panel-sessions-repo.mjs | 68 +++++++++++ test/unit/control-plane-lock.test.mjs | 81 +++++++++++++ test/unit/panel-sessions-repo.test.mjs | 84 +++++++++++++ 5 files changed, 356 insertions(+) create mode 100644 src/lib/db/control-plane-lock.mjs create mode 100644 src/lib/db/postgres-migrations/002_control_plane_sessions.sql create mode 100644 src/lib/db/repos/panel-sessions-repo.mjs create mode 100644 test/unit/control-plane-lock.test.mjs create mode 100644 test/unit/panel-sessions-repo.test.mjs diff --git a/src/lib/db/control-plane-lock.mjs b/src/lib/db/control-plane-lock.mjs new file mode 100644 index 000000000..467a66602 --- /dev/null +++ b/src/lib/db/control-plane-lock.mjs @@ -0,0 +1,114 @@ +/** Single active controller guard. This does not fence remote execution or provide failover. */ + +const LOCK_SQL = 'SELECT pg_try_advisory_lock(hashtext(current_database()), hashtext($1)) AS acquired' +const UNLOCK_SQL = 'SELECT pg_advisory_unlock(hashtext(current_database()), hashtext($1))' + +function lockError(code, message) { + return Object.assign(new Error(message), { code }) +} + +export class ControlPlaneLock { + constructor(db, { onLost = null } = {}) { + this.db = db + this.onLost = onLost + this.resource = `vm2api:control-plane:${db.schema}` + this._healthy = false + this._client = null + this._starting = null + this._stopping = null + this._closing = false + this._lost = false + this._onError = (error) => this._lose(error) + this._onEnd = () => this._lose(lockError('control_plane_lock_lost', 'Control-plane lock connection ended')) + } + + get healthy() { + return this._healthy + } + + async start() { + if (this._healthy) return this + if (this._starting) return this._starting + if (this._lost) throw lockError('control_plane_lock_lost', 'Control-plane lock was lost; restart is required') + if (this._stopping) await this._stopping + this._closing = false + const pending = this._acquire() + this._starting = pending + try { + return await pending + } finally { + if (this._starting === pending) this._starting = null + } + } + + async _acquire() { + const client = await this.db.pool.connect() + if (this._closing) { + client.release() + throw lockError('control_plane_lock_stopped', 'Control-plane lock startup was stopped') + } + this._client = client + client.on('error', this._onError) + client.on('end', this._onEnd) + try { + const result = await client.query(LOCK_SQL, [this.resource]) + if (!result.rows[0]?.acquired) { + throw lockError('control_plane_already_active', 'Another control plane is active for this database and schema') + } + if (this._lost || this._closing) { + throw lockError('control_plane_lock_lost', 'Control-plane lock was lost during startup') + } + this._healthy = true + return this + } catch (error) { + this._healthy = false + this._client = null + this._detach(client) + client.release(error.code === 'control_plane_already_active' ? undefined : true) + throw error + } + } + + _lose(error) { + if (this._closing || this._lost || !this._client) return + this._healthy = false + this._lost = true + // The hook must synchronously stop admission and own any asynchronous cleanup. + this.onLost?.(error) + } + + _detach(client) { + client.removeListener('error', this._onError) + client.removeListener('end', this._onEnd) + } + + async stop() { + if (this._stopping) return this._stopping + this._closing = true + this._healthy = false + const pending = this._release() + this._stopping = pending + try { + await pending + } finally { + if (this._stopping === pending) this._stopping = null + } + } + + async _release() { + if (this._starting) await this._starting.catch(() => {}) + const client = this._client + if (!client) return + this._client = null + this._detach(client) + let failed = this._lost + try { + if (!this._lost) await client.query(UNLOCK_SQL, [this.resource]) + } catch (error) { + failed = true + throw error + } finally { + client.release(failed) + } + } +} diff --git a/src/lib/db/postgres-migrations/002_control_plane_sessions.sql b/src/lib/db/postgres-migrations/002_control_plane_sessions.sql new file mode 100644 index 000000000..4a5799853 --- /dev/null +++ b/src/lib/db/postgres-migrations/002_control_plane_sessions.sql @@ -0,0 +1,9 @@ +CREATE TABLE panel_sessions ( + token_hash text COLLATE "C" PRIMARY KEY, + username text COLLATE "C" NOT NULL, + role text COLLATE "C" NOT NULL, + expires_at bigint NOT NULL +); + +CREATE INDEX idx_panel_sessions_username ON panel_sessions (username); +CREATE INDEX idx_panel_sessions_expires_at ON panel_sessions (expires_at); diff --git a/src/lib/db/repos/panel-sessions-repo.mjs b/src/lib/db/repos/panel-sessions-repo.mjs new file mode 100644 index 000000000..f5dc0741c --- /dev/null +++ b/src/lib/db/repos/panel-sessions-repo.mjs @@ -0,0 +1,68 @@ +/** Shared panel sessions. Callers provide only SHA-256 token hashes. */ +import { getDb } from '../database.mjs' + +const NOW_MS = 'floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint' + +function session(row) { + return row ? { user: row.username, role: row.role, exp: row.expires_at } : null +} + +function ttl(value) { + if (!Number.isSafeInteger(value) || value <= 0) throw new RangeError('session ttlMs must be a positive safe integer') + return value +} + +export class PanelSessionsRepo { + constructor(db = getDb()) { + this.db = db + } + + async create({ tokenHash, username, role, ttlMs }) { + return session( + await this.db + .prepare(` + INSERT INTO panel_sessions (token_hash, username, role, expires_at) + VALUES ($1, $2, $3, ${NOW_MS} + $4::bigint) + RETURNING username, role, expires_at + `) + .get(tokenHash, username, role, ttl(ttlMs)), + ) + } + + async get(tokenHash) { + return session( + await this.db + .prepare(` + SELECT username, role, expires_at FROM panel_sessions + WHERE token_hash = $1 AND expires_at > ${NOW_MS} + `) + .get(tokenHash), + ) + } + + async renew(tokenHash, { expectedExp, ttlMs }) { + return session( + await this.db + .prepare(` + UPDATE panel_sessions SET expires_at = ${NOW_MS} + $3::bigint + WHERE token_hash = $1 AND expires_at = $2 AND expires_at > ${NOW_MS} + RETURNING username, role, expires_at + `) + .get(tokenHash, expectedExp, ttl(ttlMs)), + ) + } + + async remove(tokenHash) { + return (await this.db.prepare('DELETE FROM panel_sessions WHERE token_hash = $1').run(tokenHash)).changes > 0 + } + + async removeForUser(username, { exceptHash = null } = {}) { + return ( + await this.db + .prepare(` + DELETE FROM panel_sessions WHERE username = $1 AND ($2::text IS NULL OR token_hash <> $2) + `) + .run(username, exceptHash) + ).changes + } +} diff --git a/test/unit/control-plane-lock.test.mjs b/test/unit/control-plane-lock.test.mjs new file mode 100644 index 000000000..5f7999081 --- /dev/null +++ b/test/unit/control-plane-lock.test.mjs @@ -0,0 +1,81 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { ControlPlaneLock } from '../../src/lib/db/control-plane-lock.mjs' + +test('one controller owns each database and schema until explicit stop', async () => { + const db = getDb() + let lost = 0 + const first = new ControlPlaneLock(db, { + onLost: () => { + lost += 1 + }, + }) + const second = new ControlPlaneLock(db) + try { + assert.equal(first.healthy, false) + assert.equal(await first.start(), first) + assert.equal(first.healthy, true) + await Promise.all([first.start(), first.start()]) + await assert.rejects(second.start(), { code: 'control_plane_already_active' }) + assert.equal(second.healthy, false) + await first.stop() + await first.stop() + assert.equal(first.healthy, false) + assert.equal(lost, 0) + await second.start() + assert.equal(second.healthy, true) + } finally { + await first.stop() + await second.stop() + } +}) + +test('independent schemas do not block each other and stop releases the pool client', async () => { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema + '_other', max: 1 }) + const first = new ControlPlaneLock(db) + const second = new ControlPlaneLock(other) + try { + await Promise.all([first.start(), second.start()]) + assert.equal(first.healthy, true) + assert.equal(second.healthy, true) + await second.stop() + assert.equal((await other.prepare('SELECT 1 AS n').get()).n, 1) + await other.close() + } finally { + await first.stop() + await second.stop() + await other.close() + } +}) + +test('a terminated lock connection fails closed and reports loss only once', async () => { + const db = getDb() + let signalLost + const lost = new Promise((resolve) => { + signalLost = resolve + }) + let calls = 0 + const lock = new ControlPlaneLock(db, { + onLost: (error) => { + calls += 1 + signalLost(error) + }, + }) + try { + await lock.start() + const ended = new Promise((resolve) => lock._client.once('end', resolve)) + const pid = lock._client.processID + assert.equal((await db.prepare('SELECT pg_terminate_backend($1) AS terminated').get(pid)).terminated, true) + await lost + await ended + assert.equal(lock.healthy, false) + assert.equal(calls, 1) + await assert.rejects(lock.start(), { code: 'control_plane_lock_lost' }) + await lock.stop() + assert.equal(calls, 1) + } finally { + await lock.stop() + } +}) diff --git a/test/unit/panel-sessions-repo.test.mjs b/test/unit/panel-sessions-repo.test.mjs new file mode 100644 index 000000000..61f8da236 --- /dev/null +++ b/test/unit/panel-sessions-repo.test.mjs @@ -0,0 +1,84 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import { getDb } from '../../src/lib/db/database.mjs' +import { PanelSessionsRepo } from '../../src/lib/db/repos/panel-sessions-repo.mjs' + +const hash = (label) => crypto.createHash('sha256').update(label).digest('hex') +const nowSql = 'floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint' + +test('panel sessions use PostgreSQL time and persist only token hashes', async (t) => { + const db = getDb() + const repo = new PanelSessionsRepo(db) + t.mock.method(Date, 'now', () => 0) + const before = (await db.prepare(`SELECT ${nowSql} AS now`).get()).now + const created = await repo.create({ tokenHash: hash('token-a'), username: 'alice', role: 'user', ttlMs: 60_000 }) + const after = (await db.prepare(`SELECT ${nowSql} AS now`).get()).now + assert.ok(created.exp >= before + 60_000 && created.exp <= after + 60_000) + assert.deepEqual(await new PanelSessionsRepo(db).get(hash('token-a')), created) + assert.equal(created.user, 'alice') + assert.equal(created.role, 'user') + const stored = await db.prepare('SELECT * FROM panel_sessions').get() + assert.deepEqual(Object.keys(stored).sort(), ['expires_at', 'role', 'token_hash', 'username']) + assert.equal(stored.token_hash, hash('token-a')) +}) + +test('expired and revoked sessions cannot be renewed or resurrected', async () => { + const db = getDb() + const repo = new PanelSessionsRepo(db) + const key = hash('token-a') + const created = await repo.create({ tokenHash: key, username: 'alice', role: 'user', ttlMs: 60_000 }) + await db.prepare('UPDATE panel_sessions SET expires_at = 0 WHERE token_hash = $1').run(key) + assert.equal(await repo.get(key), null) + assert.equal(await repo.renew(key, { expectedExp: 0, ttlMs: 60_000 }), null) + assert.equal(await repo.remove(key), true) + assert.equal(await repo.remove(key), false) + assert.equal(await repo.renew(key, { expectedExp: created.exp, ttlMs: 60_000 }), null) + assert.equal((await db.prepare('SELECT count(*) AS n FROM panel_sessions').get()).n, 0) +}) + +test('concurrent renewals compare the observed expiry and preserve the winner', async () => { + const repo = new PanelSessionsRepo(getDb()) + const key = hash('token-a') + const created = await repo.create({ tokenHash: key, username: 'alice', role: 'admin', ttlMs: 60_000 }) + const attempts = await Promise.all([ + repo.renew(key, { expectedExp: created.exp, ttlMs: 120_000 }), + repo.renew(key, { expectedExp: created.exp, ttlMs: 180_000 }), + ]) + const winners = attempts.filter(Boolean) + assert.equal(winners.length, 1) + assert.ok(winners[0].exp > created.exp) + assert.deepEqual(await repo.get(key), winners[0]) + assert.equal(await repo.renew(key, { expectedExp: created.exp, ttlMs: 240_000 }), null) +}) + +test('user revocation preserves only the excluded hash and other users', async () => { + const repo = new PanelSessionsRepo(getDb()) + for (const [label, username] of [ + ['a', 'alice'], + ['b', 'alice'], + ['c', 'alice'], + ['d', 'bob'], + ]) { + await repo.create({ tokenHash: hash(label), username, role: 'user', ttlMs: 60_000 }) + } + assert.equal(await repo.removeForUser('alice', { exceptHash: hash('b') }), 2) + assert.equal(await repo.get(hash('a')), null) + assert.equal(await repo.get(hash('c')), null) + assert.equal((await repo.get(hash('b'))).user, 'alice') + assert.equal((await repo.get(hash('d'))).user, 'bob') + assert.equal(await repo.removeForUser('alice'), 1) + assert.equal(await repo.get(hash('b')), null) +}) + +test('invalid session durations fail before writing a session', async () => { + const db = getDb() + const repo = new PanelSessionsRepo(db) + for (const ttlMs of [0, -1, 1.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) { + await assert.rejects( + repo.create({ tokenHash: hash(String(ttlMs)), username: 'alice', role: 'user', ttlMs }), + RangeError, + ) + } + assert.equal((await db.prepare('SELECT count(*) AS n FROM panel_sessions').get()).n, 0) +}) From 20551f3b55d8e9702faae95000bc79cbaef311b8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 14:54:12 +0800 Subject: [PATCH 032/473] test(db): reject SQLite import into a target with panel sessions --- test/unit/sqlite-import.test.mjs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/unit/sqlite-import.test.mjs b/test/unit/sqlite-import.test.mjs index bc61d5f36..7a8c336a8 100644 --- a/test/unit/sqlite-import.test.mjs +++ b/test/unit/sqlite-import.test.mjs @@ -43,6 +43,24 @@ test('offline import preserves source, credentials, bigint, runtime and legacy d await assert.rejects(importSqlite({ source, db }), /target is not empty/) }) +test('PostgreSQL-only session rows make the import target nonempty', async () => { + const old = sqlite({ dbPath: source }) + old.close() + const before = digest() + const db = getDb() + await db + .prepare(` + INSERT INTO panel_sessions(token_hash,username,role,expires_at) + VALUES($1,'alice','admin',floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint + 60000) + `) + .run('a'.repeat(64)) + await assert.rejects(importSqlite({ source, db }), /target is not empty \(panel_sessions\)/) + assert.equal((await db.prepare('SELECT count(*) AS n FROM panel_sessions').get()).n, 1) + assert.equal((await db.prepare('SELECT count(*) AS n FROM settings').get()).n, 0) + assert.equal((await db.prepare('SELECT count(*) AS n FROM groups').get()).n, 2) + assert.equal(digest(), before) +}) + test('unmapped source schema is rejected without writing target', async () => { const old = sqlite({ dbPath: source }) old.exec('CREATE TABLE unknown_business(id TEXT)') From 03fb157cbd29a9f5b591f6e9097e50be97012397 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 14:59:52 +0800 Subject: [PATCH 033/473] fix(db): include PostgreSQL-only tables in import preflight --- src/lib/db/sqlite-import.mjs | 15 ++++++++++++--- test/unit/database.test.mjs | 15 ++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/lib/db/sqlite-import.mjs b/src/lib/db/sqlite-import.mjs index f8da35827..9e11d1975 100644 --- a/src/lib/db/sqlite-import.mjs +++ b/src/lib/db/sqlite-import.mjs @@ -89,9 +89,18 @@ export async function importSqlite({ source, db }) { await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ `vm2api:sqlite-import:${db.schema}`, ]) - await db.exec(`LOCK TABLE ${expected.map(q).join(',')} IN ACCESS EXCLUSIVE MODE`) - for (const table of expected) { - const count = (await db.prepare(`SELECT count(*) AS n FROM ${q(table)}`).get()).n + const targetTables = ( + await db + .prepare(` + SELECT tablename FROM pg_tables + WHERE schemaname = $1 AND tablename <> 'schema_migrations' ORDER BY tablename + `) + .all(db.schema) + ).map((row) => row.tablename) + const targetName = (table) => `${q(db.schema)}.${q(table)}` + await db.exec(`LOCK TABLE ${targetTables.map(targetName).join(',')} IN ACCESS EXCLUSIVE MODE`) + for (const table of targetTables) { + const count = (await db.prepare(`SELECT count(*) AS n FROM ${targetName(table)}`).get()).n if (table === 'groups') { const rows = await db.prepare('SELECT * FROM groups ORDER BY id').all() const seedCols = columns(template, 'groups') diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index 7453be3f6..af74078be 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -34,7 +34,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 30) + assert.equal(rows.length, 31) for (const name of [ 'settings', 'users', @@ -44,6 +44,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'thinking_signature_owners', 'accounts_legacy', 'panel_users_legacy', + 'panel_sessions', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -104,23 +105,27 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 1) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 2) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) test('failed migrations rollback DDL and migration records', async () => { const dir = fs.mkdtempSync('/tmp/vm2api-pg-migrations-') - fs.writeFileSync(dir + '/002_bad.sql', 'CREATE TABLE partial_write(id text); INVALID SQL;') + fs.writeFileSync(dir + '/003_bad.sql', 'CREATE TABLE partial_write(id text); INVALID SQL;') const db = await database() await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /unknown PostgreSQL migration/) fs.copyFileSync( new URL('../../src/lib/db/postgres-migrations/001_baseline.sql', import.meta.url), dir + '/001_baseline.sql', ) - await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /002_bad.sql failed/) + fs.copyFileSync( + new URL('../../src/lib/db/postgres-migrations/002_control_plane_sessions.sql', import.meta.url), + dir + '/002_control_plane_sessions.sql', + ) + await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /003_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 1) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 2) fs.rmSync(dir, { recursive: true, force: true }) }) From c07be03a7c86fa4b33cdc12c2fd513f921d4d64a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:02:14 +0800 Subject: [PATCH 034/473] feat(deploy): add isolated single-controller Kubernetes deployment --- .github/workflows/test.yml | 11 +- deploy/k8s/configmap.yaml | 15 ++ deploy/k8s/deployment.yaml | 124 +++++++++ deploy/k8s/ingress.example.yaml | 21 ++ deploy/k8s/kustomization.yaml | 7 + deploy/k8s/persistentvolumeclaim.yaml | 10 + deploy/k8s/service.yaml | 12 + docs/deploy/kubernetes.md | 109 ++++++++ scripts/docker-entrypoint.sh | 14 +- test/deploy/smoke_kubernetes.py | 356 ++++++++++++++++++++++++++ test/deploy/test_kubernetes.py | 104 ++++++++ test/deploy/test_kubernetes_smoke.py | 64 +++++ 12 files changed, 839 insertions(+), 8 deletions(-) create mode 100644 deploy/k8s/configmap.yaml create mode 100644 deploy/k8s/deployment.yaml create mode 100644 deploy/k8s/ingress.example.yaml create mode 100644 deploy/k8s/kustomization.yaml create mode 100644 deploy/k8s/persistentvolumeclaim.yaml create mode 100644 deploy/k8s/service.yaml create mode 100644 docs/deploy/kubernetes.md create mode 100644 test/deploy/smoke_kubernetes.py create mode 100644 test/deploy/test_kubernetes.py create mode 100644 test/deploy/test_kubernetes_smoke.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aff32a07a..72a137561 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,9 +50,16 @@ jobs: . /etc/os-release echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null sudo apt-get update - sudo apt-get install -y postgresql-client-17 + sudo apt-get install -y postgresql-client-17 python3-yaml echo /usr/lib/postgresql/17/bin >> "$GITHUB_PATH" /usr/lib/postgresql/17/bin/pg_dump --version + - name: pinned Kubernetes manifest tooling + run: | + curl --fail --silent --show-error --location -o "$RUNNER_TEMP/kubectl" https://dl.k8s.io/release/v1.36.4/bin/linux/amd64/kubectl + curl --fail --silent --show-error --location -o "$RUNNER_TEMP/kubectl.sha256" https://dl.k8s.io/release/v1.36.4/bin/linux/amd64/kubectl.sha256 + python3 -c 'import hashlib,os,pathlib; root=pathlib.Path(os.environ["RUNNER_TEMP"]); assert hashlib.sha256((root/"kubectl").read_bytes()).hexdigest()==(root/"kubectl.sha256").read_text().strip()' + chmod 755 "$RUNNER_TEMP/kubectl" + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" - name: panel user and onboarding regression tests run: node --import ./test/pg-setup.mjs --test test/unit/panel-users.test.mjs test/unit/onboarding.test.mjs test/e2e/panel-users.e2e.test.mjs - name: format-check @@ -67,7 +74,7 @@ jobs: - run: npm run test:unit - run: npm run test:e2e - name: ARM64 host preparation and rollback tests - run: python3 -m unittest discover -s test/deploy -v + run: /usr/bin/python3 -m unittest discover -s test/deploy -v - name: PostgreSQL application role permissions run: python3 test/deploy/smoke_postgres_role.py diff --git a/deploy/k8s/configmap.yaml b/deploy/k8s/configmap.yaml new file mode 100644 index 000000000..9ad88e3a0 --- /dev/null +++ b/deploy/k8s/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vm2api-config +data: + VM2API_CONTROL_PLANE_MODE: kubernetes + VM2API_SHUTDOWN_TIMEOUT_MS: "60000" + VM2API_CLUSTER_SOCKET_DIR: /opt/vm2api/data/run/cluster + KIN_PROJECT_ROOT: /opt/vm2api + KIN_DATA_DIR: /opt/vm2api/data + KIN_CAPTURES_DIR: /opt/vm2api/data/captures + KIN_AUTO_SYNC_WRAP: "0" + HOST: 0.0.0.0 + PORT: "8787" + NODE_ENV: production diff --git a/deploy/k8s/deployment.yaml b/deploy/k8s/deployment.yaml new file mode 100644 index 000000000..7607bba62 --- /dev/null +++ b/deploy/k8s/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vm2api +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: vm2api + template: + metadata: + labels: + app.kubernetes.io/name: vm2api + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 75 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + initContainers: + - name: initialize-state + image: ghcr.io/dofastted/vm2api:kubernetes-draft + command: [node, -e] + args: + - "const fs=require('node:fs');for(const dir of ['vms','data','config'])fs.mkdirSync('/state/'+dir,{recursive:true,mode:0o700})" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + volumeMounts: + - name: state + mountPath: /state + containers: + - name: control-plane + image: ghcr.io/dofastted/vm2api:kubernetes-draft + ports: + - name: http + containerPort: 8787 + envFrom: + - configMapRef: + name: vm2api-config + env: + - name: VM2API_DATABASE_URL + valueFrom: + secretKeyRef: + name: vm2api-secrets + key: VM2API_DATABASE_URL + - name: VM2API_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: vm2api-secrets + key: VM2API_ADMIN_PASSWORD + - name: VM2API_API_KEY + valueFrom: + secretKeyRef: + name: vm2api-secrets + key: VM2API_API_KEY + - name: VM2API_DB_SECRET + valueFrom: + secretKeyRef: + name: vm2api-secrets + key: VM2API_DB_SECRET + startupProbe: + httpGet: + path: /readyz + port: http + periodSeconds: 2 + timeoutSeconds: 3 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /readyz + port: http + periodSeconds: 5 + timeoutSeconds: 3 + livenessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 10 + timeoutSeconds: 3 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + volumeMounts: + - name: state + mountPath: /opt/vm2api/vms + subPath: vms + - name: state + mountPath: /opt/vm2api/data + subPath: data + - name: state + mountPath: /opt/vm2api/src/config + subPath: config + - name: run + mountPath: /opt/vm2api/data/run + - name: bin + mountPath: /opt/vm2api/bin + - name: share + mountPath: /opt/vm2api/share + volumes: + - name: state + persistentVolumeClaim: + claimName: vm2api-state + - name: run + emptyDir: {} + - name: bin + emptyDir: {} + - name: share + emptyDir: {} diff --git a/deploy/k8s/ingress.example.yaml b/deploy/k8s/ingress.example.yaml new file mode 100644 index 000000000..4c11de9fd --- /dev/null +++ b/deploy/k8s/ingress.example.yaml @@ -0,0 +1,21 @@ +# Optional; set the ingress class, host and TLS secret for your installation. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vm2api +spec: + ingressClassName: traefik + tls: + - hosts: [vm2api.example.com] + secretName: vm2api-tls + rules: + - host: vm2api.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vm2api + port: + name: http diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml new file mode 100644 index 000000000..575d46b5d --- /dev/null +++ b/deploy/k8s/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - configmap.yaml + - persistentvolumeclaim.yaml + - deployment.yaml + - service.yaml diff --git a/deploy/k8s/persistentvolumeclaim.yaml b/deploy/k8s/persistentvolumeclaim.yaml new file mode 100644 index 000000000..55806cb33 --- /dev/null +++ b/deploy/k8s/persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: vm2api-state +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/deploy/k8s/service.yaml b/deploy/k8s/service.yaml new file mode 100644 index 000000000..899d8747b --- /dev/null +++ b/deploy/k8s/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: vm2api +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: vm2api + ports: + - name: http + port: 8787 + targetPort: http diff --git a/docs/deploy/kubernetes.md b/docs/deploy/kubernetes.md new file mode 100644 index 000000000..ec5e4656e --- /dev/null +++ b/docs/deploy/kubernetes.md @@ -0,0 +1,109 @@ +# Kubernetes 单实例控制面（Draft） + +本阶段运行一个活动控制面，使用 PostgreSQL 17、持久卷和远端计算节点。默认 `replicas: 1`、`Recreate` 更新策略。更新期间会有服务中断。此部署不提供水平多活或节点故障后的无损接管；不要增加副本或添加 HPA。 + +`VM2API_CONTROL_PLANE_MODE=kubernetes` 启用 PG 独占控制面锁及 PG 面板会话。第二个实例连接同一数据库和 schema 时，在业务初始化前退出。连接丢失后，控制面停止准入并退出。该锁不向远端执行器提供 fencing，不能证明失联的远端请求已终止。迁移时必须先停止旧控制面，并确认在途任务的状态。 + +## 前提 + +- 使用包含本分支运行模式、独占锁、PG 会话和探针实现的镜像。清单中的 `kubernetes-draft` 是待替换镜像标签，不能改用不支持这些功能的既有 `latest`。 +- 独立 PostgreSQL 17 业务数据库。连接角色拥有应用 schema 和迁移对象，不需要超级用户或创建数据库权限。所有部署使用同一业务加密密钥。参见 [PostgreSQL 部署与迁移](postgresql.md)。 +- 集群有可用的 RWO StorageClass，或者将清单改为已有 PVC。示例申请 10Gi；按 VM 文件和备份保留量调整。`local-path` 卷绑定节点，不能据此声明跨节点数据可用。换节点需要可迁移的持久存储和已验证的恢复流程。 +- kubectl 含 Kustomize。控制面不访问 Kubernetes API，不挂载 ServiceAccount token、宿主 Docker socket 或 hostPath，也不使用 hostNetwork、特权模式或附加 Linux capabilities。 + +只使用远端计算节点。Pod 内的 Docker CLI 用于访问已注册远端节点的连接，不管理所在 Kubernetes 节点的 Docker。入口在 Kubernetes 模式下跳过宿主镜像拉取和自动同步。该模式不把普通 Kubernetes 工作节点变成计算槽位宿主。 + +## 配置与部署 + +在安装命名空间中预建 `vm2api-secrets`。使用权限为 `0600` 的受控环境文件提供以下键，不将文件提交到 Git，也不把值放进命令参数或日志： + +```text +VM2API_DATABASE_URL +VM2API_ADMIN_PASSWORD +VM2API_API_KEY +VM2API_DB_SECRET +``` + +管理员用户名默认为 `admin`。已有 SQLite 安装应先按 PostgreSQL 文档完成离线导入，保留原业务加密密钥和对应的 VM、配置及备份文件，之后再启动 Pod。 + +```sh +kubectl create namespace vm2api +kubectl -n vm2api create secret generic vm2api-secrets --from-env-file=/secure/vm2api.env +``` + +在本地创建覆盖目录，例如仓库内 `.local/k8s-production/kustomization.yaml`： + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: vm2api +resources: + - ../../deploy/k8s +images: + - name: ghcr.io/dofastted/vm2api + newName: registry.example.com/vm2api + digest: sha256:REPLACE_WITH_VERIFIED_IMAGE_DIGEST +patches: + - target: + kind: ConfigMap + name: vm2api-config + patch: |- + - op: add + path: /data/PUBLIC_BASE_URL + value: https://vm2api.example.com +``` + +确认实际镜像和 StorageClass 后渲染、部署： + +```sh +kubectl kustomize .local/k8s-production +kubectl apply -k .local/k8s-production +kubectl -n vm2api rollout status deployment/vm2api +kubectl -n vm2api port-forward service/vm2api 8787:8787 +``` + +入口将镜像默认配置复制到空配置目录。业务目录与进程目录分别挂载： + +| 路径 | 存储 | 用途 | +| --- | --- | --- | +| `vms` | PVC | VM JSON、CLI-home 和既有凭据文件 | +| `src/config` | PVC | 路由等持久配置 | +| `data` | PVC | 备份、恢复维护标记及其他持久业务文件 | +| `data/run` | emptyDir | 当前 Pod 的 API kernel 与 cluster relay sockets | +| `bin`、`share` | emptyDir | 从当前镜像复制的二进制和工具文件 | + +Pod 使用固定 UID/GID `1000`，持久卷通过 `fsGroup: 1000` 提供访问权限。导入既有文件时，应先核实该 UID 的访问权限。删除 Deployment 或重建 Pod 会保留 PVC；删除整个 Kustomization 会同时删除示例 PVC，不能将其用作普通停止操作。 + +## 探针、终止与入口 + +`/healthz` 只用于存活检查。`/readyz` 检查初始化、PG 查询、独占锁、维护及排空状态。启动探针成功后才开始常规存活和就绪探针。SIGTERM 停止新 HTTP 请求和 WebSocket 升级，等待在途处理及持久化完成。默认排空期限为 `VM2API_SHUTDOWN_TIMEOUT_MS=60000`，Pod 的 `terminationGracePeriodSeconds=75` 留出清理余量。修改期限时同步调整 Pod 配置。超过期限的长连接可能被中断，客户端需要处理重连。 + +可选的 `deploy/k8s/ingress.example.yaml` 不包含在默认资源中。修改域名、IngressClass 和已有 TLS Secret 后单独应用，并配置匹配的 `PUBLIC_BASE_URL`。Traefik 支持 WebSocket；SSE 路径不应附加响应缓冲中间件。入口层的超时和请求大小限制需要按实际长请求验证，不能只检查 `/healthz`。 + +Kubernetes 的 [Recreate 策略](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#recreate-deployment)、[探针语义](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/)、[持久卷与 emptyDir](https://kubernetes.io/docs/concepts/storage/volumes/) 和 [ServiceAccount token 配置](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 是这些清单的依据。Traefik 配置参见 [WebSocket](https://doc.traefik.io/traefik/master/user-guides/websocket/) 和 [缓冲中间件](https://doc.traefik.io/traefik/middlewares/http/buffering/)。 + +## 验证 + +静态与入口测试需要 Python 3、PyYAML 和 kubectl；缺少工具直接失败。Debian/Ubuntu 可安装 `python3-yaml`。测试只渲染清单，并在临时目录用合成 Unix socket 和假 Node 命令执行入口分支，不启动服务器或容器。 + +```sh +python3 -m unittest test/deploy/test_kubernetes.py -v +``` + +真实集成脚本只接受 API 地址为本机 loopback 的显式 context。候选镜像应先导入本地集群，或使用集群可拉取的已验证镜像标签： + +```sh +python3 test/deploy/smoke_kubernetes.py --context cube-local --image vm2api:kubernetes-candidate --storage-class local-path +``` + +脚本创建全新 `vm2api-k8s-649-*` 命名空间，部署独立的临时 PostgreSQL 17 和业务角色，并通过 stdin 提交随机测试 Secret。它使用同一候选镜像启动 Node.js 合成上游，配置 API endpoint 和 `category=api` 的托管 key。验证范围包括： + +- Pod 就绪、Service DNS/ClusterIP、探针和控制台。 +- Cookie 登录、PG 指标及 API key 创建。 +- 经过实际 Go API kernel 的非流式和 SSE 请求，校验本次唯一响应文本、分段到达及两条成功请求日志。 +- 第二实例被独占锁拒绝。 +- Recreate 重启后 Cookie 会话、托管 key、endpoint 和持久卷文件保留。 + +测试命名空间的 egress policy 只允许同命名空间 Pod 和集群 DNS；后台使用的 `models.dev` 在测试 Pod 中映射到 loopback。测试不使用真实上游凭据。输出仅包含检查结果、合成响应文本和运行镜像 ID。退出时核对命名空间 UID 及本次唯一标签，再删除本任务创建的命名空间。它不复用或删除现有命名空间。 + +该脚本验证单实例部署和 API endpoint 推理。它不验证远端槽位的活凭据或真实供应商推理。SSE/WebSocket 排空与故障处理还需结合对应的应用集成测试。单节点 k3s 验证不等同于跨节点存储、远端执行 fencing 或水平扩容验收。 diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 617bd966a..959baa650 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -90,11 +90,13 @@ if [ -f /opt/vm2api/image-crag/kin-kernel ]; then fi fi -if [ ! -S /var/run/docker.sock ]; then - echo "vm2api: /var/run/docker.sock not mounted; slot create/start will fail." >&2 -elif [ -f /opt/vm2api/docker/kin-os/build.mjs ]; then - # 只拉预构建客体镜像；拉不到不阻塞启动，建槽时再兜底构建。 - node /opt/vm2api/docker/kin-os/build.mjs --pull-only ubuntu || true +if [ "${VM2API_CONTROL_PLANE_MODE:-}" != kubernetes ]; then + if [ ! -S /var/run/docker.sock ]; then + echo "vm2api: /var/run/docker.sock not mounted; slot create/start will fail." >&2 + elif [ -f /opt/vm2api/docker/kin-os/build.mjs ]; then + # 只拉预构建客体镜像；拉不到不阻塞启动，建槽时再兜底构建。 + node /opt/vm2api/docker/kin-os/build.mjs --pull-only ubuntu || true + fi fi env_get_file() { @@ -154,7 +156,7 @@ if [ -z "${VM2API_DB_SECRET:-}" ] && [ -z "${KIN_DB_SECRET:-}" ]; then env_upsert_if_empty VM2API_DB_SECRET "$VM2API_DB_SECRET" "$ENVF" fi -if [ "$WRAP_CHANGED" = 1 ] && [ "${KIN_AUTO_SYNC_WRAP:-0}" = 1 ]; then +if [ "${VM2API_CONTROL_PLANE_MODE:-}" != kubernetes ] && [ "$WRAP_CHANGED" = 1 ] && [ "${KIN_AUTO_SYNC_WRAP:-0}" = 1 ]; then node /opt/vm2api/scripts/sync-wrap-cli.mjs & fi diff --git a/test/deploy/smoke_kubernetes.py b/test/deploy/smoke_kubernetes.py new file mode 100644 index 000000000..70a3c9d4c --- /dev/null +++ b/test/deploy/smoke_kubernetes.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Run a candidate image only in a newly created, owned local Kubernetes namespace.""" +import argparse +import copy +from http.cookiejar import CookieJar +import json +from pathlib import Path +import re +import secrets +import selectors +import subprocess +import time +from urllib.parse import urlparse +from urllib.request import HTTPCookieProcessor, Request, build_opener, urlopen +import uuid + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +PREFIX = 'vm2api-k8s-649-' +OWNER_LABEL = 'vm2api.test/run' +MODEL = 'claude-haiku-4-5-20251001' +UPSTREAM_KEY = 'synthetic-kubernetes-upstream-key' +UPSTREAM = r''' +const http=require('node:http'); +const proof=process.env.SMOKE_PROOF; +http.createServer(async(req,res)=>{ + if(req.url==='/readyz'){res.end('ok');return} + if(!req.url.startsWith('/v1/messages')||req.headers['x-api-key']!=='synthetic-kubernetes-upstream-key'){ + res.writeHead(403);res.end();return; + } + let body='';for await(const chunk of req)body+=chunk; + const input=JSON.parse(body); + const message={id:'msg_kubernetes_smoke',type:'message',role:'assistant',model:input.model, + content:[],stop_reason:null,stop_sequence:null,usage:{input_tokens:11,output_tokens:0}}; + if(!input.stream){ + res.setHeader('content-type','application/json'); + res.end(JSON.stringify({...message,content:[{type:'text',text:proof}],stop_reason:'end_turn', + usage:{input_tokens:11,output_tokens:3}}));return; + } + res.writeHead(200,{'content-type':'text/event-stream','cache-control':'no-cache'}); + const event=(type,data)=>res.write(`event: ${type}\ndata: ${JSON.stringify({type,...data})}\n\n`); + event('message_start',{message}); + event('content_block_start',{index:0,content_block:{type:'text',text:''}}); + setTimeout(()=>{ + event('content_block_delta',{index:0,delta:{type:'text_delta',text:proof}}); + event('content_block_stop',{index:0}); + event('message_delta',{delta:{stop_reason:'end_turn',stop_sequence:null},usage:{output_tokens:3}}); + event('message_stop',{});res.end(); + },1000); +}).listen(8080,'0.0.0.0'); +''' + + +class Smoke: + def __init__(self, context, image, storage_class): + self.context = context + self.image = image + self.storage_class = storage_class + self.run_id = uuid.uuid4().hex + self.namespace = PREFIX + self.run_id[:10] + self.namespace_uid = None + self.forward = None + self.checks = [] + self.client = build_opener(HTTPCookieProcessor(CookieJar())) + self.proof = 'kubernetes-proof-' + self.run_id[:12] + + def kubectl(self, *args, namespaced=True, content=None, check=True, timeout=240): + command = ['kubectl', '--context', self.context] + if namespaced: + command += ['--namespace', self.namespace] + result = subprocess.run(command + list(args), input=content, text=True, + capture_output=True, timeout=timeout) + if check and result.returncode: + # kubectl validation errors can include submitted Secret objects. + raise RuntimeError(f'kubectl {args[0]} failed (exit {result.returncode})') + return result + + def apply(self, *resources): + self.kubectl('apply', '-f', '-', content=json.dumps({'apiVersion': 'v1', 'kind': 'List', + 'items': resources})) + + def create_namespace(self): + endpoint = self.kubectl('config', 'view', '--minify', '-o', + 'jsonpath={.clusters[0].cluster.server}', namespaced=False).stdout + if urlparse(endpoint).hostname not in ('127.0.0.1', 'localhost', '::1'): + raise RuntimeError('smoke test requires an explicit local Kubernetes context') + existing = self.kubectl('get', 'namespace', self.namespace, '-o', 'json', + namespaced=False, check=False) + if existing.returncode == 0: + raise RuntimeError('test namespace already exists; refusing to reuse it') + resource = {'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': { + 'name': self.namespace, 'labels': {OWNER_LABEL: self.run_id}}} + created = self.kubectl('create', '-f', '-', '-o', 'json', namespaced=False, + content=json.dumps(resource)) + self.namespace_uid = json.loads(created.stdout)['metadata']['uid'] + self.apply({'apiVersion': 'networking.k8s.io/v1', 'kind': 'NetworkPolicy', + 'metadata': {'name': 'test-egress'}, 'spec': {'podSelector': {}, 'policyTypes': ['Egress'], + 'egress': [{'to': [{'podSelector': {}}]}, + {'to': [{'namespaceSelector': {'matchLabels': { + 'kubernetes.io/metadata.name': 'kube-system'}}}], + 'ports': [{'protocol': 'UDP', 'port': 53}, {'protocol': 'TCP', 'port': 53}]}]}}) + + def postgres(self, password, application_secret): + self.apply(application_secret, + {'apiVersion': 'v1', 'kind': 'ConfigMap', 'metadata': {'name': 'postgres-init'}, + 'data': {'entrypoint.sh': (ROOT / 'deploy/postgres-entrypoint.sh').read_text(), + 'init.sh': (ROOT / 'deploy/postgres-init.sh').read_text()}}, + {'apiVersion': 'v1', 'kind': 'Secret', 'metadata': {'name': 'postgres-password'}, + 'stringData': {'password': password}}, + {'apiVersion': 'v1', 'kind': 'Service', 'metadata': {'name': 'postgres'}, + 'spec': {'selector': {'app': 'postgres'}, 'ports': [{'port': 5432}]}}, + {'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': {'name': 'postgres'}, + 'spec': {'replicas': 1, 'selector': {'matchLabels': {'app': 'postgres'}}, + 'template': {'metadata': {'labels': {'app': 'postgres'}}, 'spec': { + 'automountServiceAccountToken': False, + 'securityContext': {'runAsUser': 999, 'runAsGroup': 999, + 'runAsNonRoot': True, 'fsGroup': 999, + 'seccompProfile': {'type': 'RuntimeDefault'}}, + 'containers': [{'name': 'postgres', 'image': 'postgres:17-bookworm', + 'command': ['/bin/bash', '/scripts/entrypoint.sh', 'postgres'], + 'env': [{'name': 'POSTGRES_USER', 'value': 'postgres'}, + {'name': 'POSTGRES_DB', 'value': 'postgres'}, + {'name': 'POSTGRES_INITDB_ARGS', 'value': '--auth-host=scram-sha-256'}, + {'name': 'PGDATA', 'value': '/var/lib/postgresql/data/pgdata'}, + {'name': 'VM2API_POSTGRES_PASSWORD', 'valueFrom': { + 'secretKeyRef': {'name': 'postgres-password', 'key': 'password'}}}], + 'securityContext': {'allowPrivilegeEscalation': False, + 'capabilities': {'drop': ['ALL']}}, + 'readinessProbe': {'exec': {'command': ['bash', '-ec', + 'PGPASSWORD="$VM2API_POSTGRES_PASSWORD" psql -h 127.0.0.1 ' + '-U vm2api -d vm2api -Atqc "SELECT 1" >/dev/null']}, 'periodSeconds': 2}, + 'volumeMounts': [{'name': 'data', 'mountPath': '/var/lib/postgresql/data'}, + {'name': 'scripts', 'mountPath': '/scripts'}, + {'name': 'scripts', 'mountPath': '/docker-entrypoint-initdb.d/10-vm2api.sh', + 'subPath': 'init.sh'}]}], + 'volumes': [{'name': 'data', 'emptyDir': {}}, + {'name': 'scripts', 'configMap': {'name': 'postgres-init'}}]}}}}) + self.kubectl('rollout', 'status', 'deployment/postgres', '--timeout=120s') + + def deploy(self): + rendered = subprocess.check_output(['kubectl', 'kustomize', str(ROOT / 'deploy/k8s')], text=True) + resources = list(yaml.safe_load_all(rendered)) + deployment = next(item for item in resources if item['kind'] == 'Deployment') + pod = deployment['spec']['template']['spec'] + # Automatic public price-catalog refresh stays on loopback in this fixture. + pod['hostAliases'] = [{'ip': '127.0.0.1', 'hostnames': ['models.dev']}] + for container in pod['containers'] + pod['initContainers']: + container['image'] = self.image + container['imagePullPolicy'] = 'IfNotPresent' + pvc = next(item for item in resources if item['kind'] == 'PersistentVolumeClaim') + pvc['spec']['resources']['requests']['storage'] = '1Gi' + if self.storage_class: + pvc['spec']['storageClassName'] = self.storage_class + config = next(item for item in resources if item['kind'] == 'ConfigMap') + config['data']['KIN_BACKUP_DISABLED'] = '1' + self.apply(*resources) + self.kubectl('rollout', 'status', 'deployment/vm2api', '--timeout=240s', timeout=255) + self.checks.append('pod-ready') + self.kubectl('exec', 'deployment/vm2api', '-c', 'control-plane', '--', 'node', '--input-type=module', '-e', + "const r=await fetch('http://vm2api:8787/readyz');if(r.status!==200)process.exit(1)") + self.checks.append('service-dns-and-clusterip') + return pod + + def upstream(self): + self.apply({'apiVersion': 'v1', 'kind': 'ConfigMap', 'metadata': {'name': 'synthetic-upstream'}, + 'data': {'server.cjs': UPSTREAM}}, + {'apiVersion': 'v1', 'kind': 'Service', 'metadata': {'name': 'synthetic-upstream'}, + 'spec': {'selector': {'app': 'synthetic-upstream'}, 'ports': [{'port': 8080}]}}, + {'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': {'name': 'synthetic-upstream'}, + 'spec': {'replicas': 1, 'selector': {'matchLabels': {'app': 'synthetic-upstream'}}, + 'template': {'metadata': {'labels': {'app': 'synthetic-upstream'}}, 'spec': { + 'automountServiceAccountToken': False, + 'securityContext': {'runAsNonRoot': True, 'runAsUser': 1000, + 'seccompProfile': {'type': 'RuntimeDefault'}}, + 'containers': [{'name': 'upstream', 'image': self.image, + 'imagePullPolicy': 'IfNotPresent', + 'command': ['node', '/fixture/server.cjs'], + 'env': [{'name': 'SMOKE_PROOF', 'value': self.proof}], + 'securityContext': {'allowPrivilegeEscalation': False, + 'capabilities': {'drop': ['ALL']}}, + 'readinessProbe': {'httpGet': {'path': '/readyz', 'port': 8080}}, + 'volumeMounts': [{'name': 'fixture', 'mountPath': '/fixture'}]}], + 'volumes': [{'name': 'fixture', 'configMap': {'name': 'synthetic-upstream'}}]}}}}) + self.kubectl('rollout', 'status', 'deployment/synthetic-upstream', '--timeout=120s') + + def start_forward(self): + self.stop_forward() + self.forward = subprocess.Popen(['kubectl', '--context', self.context, '-n', self.namespace, + 'port-forward', 'service/vm2api', ':8787', '--address=127.0.0.1'], + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + with selectors.DefaultSelector() as selector: + selector.register(self.forward.stdout, selectors.EVENT_READ) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if selector.select(timeout=0.5): + line = self.forward.stdout.readline() + match = re.search(r'Forwarding from 127\.0\.0\.1:(\d+)', line) + if match: + self.base_url = 'http://127.0.0.1:' + match[1] + return + if self.forward.poll() is not None: + break + raise RuntimeError('service port-forward did not become ready') + + def request(self, path, body=None, token=None, use_cookie=True): + headers = {'content-type': 'application/json'} + if token: + headers['authorization'] = 'Bearer ' + token + request = Request(self.base_url + path, headers=headers, + data=None if body is None else json.dumps(body).encode()) + with (self.client.open if use_cookie else urlopen)(request, timeout=15) as response: + assert response.status == 200 or response.status == 201, 'unexpected HTTP status' + content = response.read() + return json.loads(content) if 'application/json' in response.headers.get('content-type', '') else content + + def inference(self, managed_key): + self.request('/api/panel/api-endpoints', { + 'name': 'kubernetes-synthetic-upstream', 'kind': 'custom', 'protocol': 'anthropic', + 'base_url': 'http://synthetic-upstream:8080', 'api_key_entries': [{'api_key': UPSTREAM_KEY}], + 'models': [{'name': MODEL}], + }) + body = {'model': MODEL, 'max_tokens': 32, 'messages': [{'role': 'user', 'content': self.proof}]} + for stream in (False, True): + headers = {'authorization': 'Bearer ' + managed_key, 'content-type': 'application/json', + 'x-request-id': self.proof + ('-stream' if stream else '-plain')} + request = Request(self.base_url + '/v1/messages', headers=headers, + data=json.dumps({**body, 'stream': stream}).encode()) + with urlopen(request, timeout=15) as response: + assert response.status == 200, 'inference did not succeed' + if stream: + assert 'text/event-stream' in response.headers.get('content-type', ''), 'SSE response required' + first = response.readline() + first_at = time.monotonic() + content = first + response.read() + assert time.monotonic() - first_at >= 0.5, 'SSE frames were buffered until completion' + assert b'message_stop' in content, 'SSE stream did not finish' + assert self.proof.encode() in content, 'SSE response did not come from the synthetic upstream' + else: + content = json.loads(response.read()) + assert any(item.get('text') == self.proof for item in content['content']), 'upstream proof missing' + logs = self.request('/api/panel/request-logs?q=' + self.proof + '&limit=10')['items'] + assert len(logs) == 2 and all(item['status'] == 200 for item in logs), 'two successful request logs required' + self.checks += ['managed-api-key-inference', 'go-api-kernel-json-and-sse', 'two-request-logs'] + + def duplicate(self, pod): + other = copy.deepcopy(pod) + other['restartPolicy'] = 'Never' + for index, volume in enumerate(other['volumes']): + if volume['name'] == 'state': + other['volumes'][index] = {'name': 'state', 'emptyDir': {}} + self.apply({'apiVersion': 'v1', 'kind': 'Pod', 'metadata': {'name': 'contender'}, 'spec': other}) + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + status = json.loads(self.kubectl('get', 'pod', 'contender', '-o', 'json').stdout)['status'] + containers = status.get('containerStatuses', []) + terminated = containers and containers[0]['state'].get('terminated') + if terminated: + assert terminated['exitCode'] != 0, 'second control plane was allowed to start' + logs = self.kubectl('logs', 'contender', '-c', 'control-plane').stdout + assert 'control_plane_already_active' in logs, 'contender did not fail for the exclusive lock' + self.request('/readyz') + self.kubectl('delete', 'pod', 'contender', '--wait=true', '--timeout=30s') + self.checks.append('second-control-plane-rejected') + return + time.sleep(0.5) + raise RuntimeError('second control plane did not terminate within the startup deadline') + + def restart(self, managed_key, marker): + self.stop_forward() + self.kubectl('rollout', 'restart', 'deployment/vm2api') + self.kubectl('rollout', 'status', 'deployment/vm2api', '--timeout=240s', timeout=255) + self.start_forward() + self.request('/readyz') + keys = self.request('/api/panel/api-keys') + assert any(item['name'] == marker for item in keys['keys']), 'database or panel session did not persist' + models = self.request('/v1/models', token=managed_key, use_cookie=False) + assert any(item['id'] == MODEL for item in models['data']), 'managed API key or endpoint did not persist' + self.kubectl('exec', 'deployment/vm2api', '-c', 'control-plane', '--', 'node', '-e', + "if(require('node:fs').readFileSync('/opt/vm2api/data/kubernetes-smoke.txt','utf8')!==process.argv[1])process.exit(1)", marker) + self.checks += ['recreate-restart', 'postgres-session-and-api-key-persist', 'pvc-persists'] + + def stop_forward(self): + if self.forward is not None: + self.forward.terminate() + try: + self.forward.wait(timeout=5) + except subprocess.TimeoutExpired: + self.forward.kill() + self.forward.wait(timeout=5) + self.forward.stdout.close() + self.forward = None + + def cleanup(self): + self.stop_forward() + if self.namespace_uid is None: + return + metadata = json.loads(self.kubectl('get', 'namespace', self.namespace, '-o', 'json', + namespaced=False).stdout)['metadata'] + if (not self.namespace.startswith(PREFIX) or metadata['uid'] != self.namespace_uid or + metadata.get('labels', {}).get(OWNER_LABEL) != self.run_id): + raise RuntimeError('namespace ownership changed; refusing cleanup') + options = {'apiVersion': 'v1', 'kind': 'DeleteOptions', + 'preconditions': {'uid': self.namespace_uid}} + self.kubectl('delete', '--raw=/api/v1/namespaces/' + self.namespace, '-f', '-', + namespaced=False, content=json.dumps(options)) + self.kubectl('wait', '--for=delete', 'namespace/' + self.namespace, '--timeout=180s', + namespaced=False, timeout=195) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--context', required=True) + parser.add_argument('--image', required=True, help='candidate image available to the local cluster') + parser.add_argument('--storage-class', help='otherwise use the cluster default RWO storage class') + args = parser.parse_args() + smoke = Smoke(args.context, args.image, args.storage_class) + password = secrets.token_hex(32) + admin_password = secrets.token_hex(24) + application_secret = {'apiVersion': 'v1', 'kind': 'Secret', 'metadata': {'name': 'vm2api-secrets'}, + 'stringData': {'VM2API_DATABASE_URL': f'postgresql://vm2api:{password}@postgres:5432/vm2api', + 'VM2API_ADMIN_PASSWORD': admin_password, + 'VM2API_API_KEY': secrets.token_hex(32), + 'VM2API_DB_SECRET': secrets.token_hex(32)}} + try: + smoke.create_namespace() + smoke.postgres(password, application_secret) + pod = smoke.deploy() + smoke.upstream() + smoke.start_forward() + smoke.request('/healthz') + smoke.request('/readyz') + smoke.request('/console') + smoke.request('/api/panel/login', {'username': 'admin', 'password': admin_password}) + metrics = smoke.request('/api/panel/database/metrics')['data']['database'] + assert metrics['ok'] and metrics['engine'] == 'postgresql', 'PostgreSQL metrics required' + marker = 'kubernetes-smoke-' + smoke.run_id[:10] + managed_key = smoke.request('/api/panel/api-keys', {'name': marker, 'category': 'api'})['item']['key'] + smoke.kubectl('exec', 'deployment/vm2api', '-c', 'control-plane', '--', 'node', '-e', + "require('node:fs').writeFileSync('/opt/vm2api/data/kubernetes-smoke.txt',process.argv[1])", marker) + smoke.checks += ['health-ready-console', 'panel-cookie-login-and-postgres-metrics', 'api-key-create'] + smoke.inference(managed_key) + smoke.duplicate(pod) + smoke.restart(managed_key, marker) + pods = json.loads(smoke.kubectl('get', 'pods', '-l', 'app.kubernetes.io/name=vm2api', '-o', 'json').stdout) + image_id = pods['items'][0]['status']['containerStatuses'][0]['imageID'] + result = {'ok': True, 'namespace': smoke.namespace, 'image_id': image_id, + 'proof': smoke.proof, 'checks': smoke.checks} + finally: + smoke.cleanup() + print(json.dumps(result)) + + +if __name__ == '__main__': + main() diff --git a/test/deploy/test_kubernetes.py b/test/deploy/test_kubernetes.py new file mode 100644 index 000000000..fbb2ed43f --- /dev/null +++ b/test/deploy/test_kubernetes.py @@ -0,0 +1,104 @@ +"""Check Kubernetes isolation and execute the entrypoint without running a server.""" +import json +import os +from pathlib import Path +import socket +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] + + +class KubernetesDeploymentTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + rendered = subprocess.check_output(['kubectl', 'kustomize', str(ROOT / 'deploy/k8s')], text=True) + cls.resources = list(yaml.safe_load_all(rendered)) + cls.deployment = next(item for item in cls.resources if item['kind'] == 'Deployment') + cls.config = next(item for item in cls.resources if item['kind'] == 'ConfigMap')['data'] + + def test_single_active_control_plane_has_no_host_or_kubernetes_api_access(self): + self.assertEqual(self.deployment['spec']['replicas'], 1) + self.assertEqual(self.deployment['spec']['strategy'], {'type': 'Recreate'}) + self.assertEqual(self.config['VM2API_CONTROL_PLANE_MODE'], 'kubernetes') + pod = self.deployment['spec']['template']['spec'] + self.assertFalse(pod['automountServiceAccountToken']) + self.assertFalse(pod.get('hostNetwork', False)) + self.assertTrue(pod['securityContext']['runAsNonRoot']) + for volume in pod['volumes']: + self.assertNotIn('hostPath', volume) + for container in pod['containers'] + pod['initContainers']: + self.assertFalse(container['securityContext'].get('privileged', False)) + self.assertFalse(container['securityContext']['allowPrivilegeEscalation']) + self.assertEqual(container['securityContext']['capabilities'], {'drop': ['ALL']}) + self.assertFalse({'Secret', 'Role', 'ClusterRole', 'HorizontalPodAutoscaler'} & + {item['kind'] for item in self.resources}) + + def test_runtime_sockets_and_image_helpers_are_pod_local(self): + pod = self.deployment['spec']['template']['spec'] + volumes = {item['name']: item for item in pod['volumes']} + mounts = {item['mountPath']: item for item in pod['containers'][0]['volumeMounts']} + for directory in ('data/run', 'bin', 'share'): + self.assertIn('emptyDir', volumes[mounts['/opt/vm2api/' + directory]['name']]) + self.assertTrue(self.config['VM2API_CLUSTER_SOCKET_DIR'].startswith('/opt/vm2api/data/run/')) + pvc = next(item for item in self.resources if item['kind'] == 'PersistentVolumeClaim') + self.assertEqual(pvc['spec']['accessModes'], ['ReadWriteOnce']) + + def test_readiness_and_shutdown_contract_and_required_secret_references(self): + pod = self.deployment['spec']['template']['spec'] + container = pod['containers'][0] + self.assertEqual(container['startupProbe']['httpGet']['path'], '/readyz') + self.assertEqual(container['readinessProbe']['httpGet']['path'], '/readyz') + self.assertEqual(container['livenessProbe']['httpGet']['path'], '/healthz') + self.assertGreater(pod['terminationGracePeriodSeconds'] * 1000, + int(self.config['VM2API_SHUTDOWN_TIMEOUT_MS'])) + for item in container['env']: + reference = item['valueFrom']['secretKeyRef'] + self.assertEqual(reference['name'], 'vm2api-secrets') + self.assertEqual(reference['key'], item['name']) + self.assertFalse(reference.get('optional', False)) + self.assertEqual({item['name'] for item in container['env']}, { + 'VM2API_DATABASE_URL', 'VM2API_ADMIN_PASSWORD', 'VM2API_API_KEY', 'VM2API_DB_SECRET', + }) + + def test_kubernetes_entrypoint_skips_host_docker_even_when_a_socket_exists(self): + for mode in ('standalone', 'kubernetes'): + with self.subTest(mode=mode), tempfile.TemporaryDirectory(prefix='vm2api-entrypoint-') as directory: + root = Path(directory) + for name in ('image-bin', 'docker/kin-os', 'src', 'commands'): + (root / name).mkdir(parents=True, exist_ok=True) + for name in ('kin-kernel', 'kin-api-kernel'): + (root / 'image-bin' / name).write_text('#!/bin/sh\nexit 0\n') + (root / 'docker/kin-os/build.mjs').touch() + node = root / 'commands/node' + node.write_text('#!/usr/bin/env python3\nimport json,os,sys\n' + 'with open(os.environ["TEST_NODE_LOG"],"a") as f:f.write(json.dumps(sys.argv[1:])+"\\n")\n') + node.chmod(0o755) + script = root / 'entrypoint.sh' + script.write_text((ROOT / 'scripts/docker-entrypoint.sh').read_text() + .replace('/opt/vm2api', str(root)) + .replace('/var/run/docker.sock', str(root / 'docker.sock'))) + env = {**os.environ, 'PATH': str(root / 'commands') + ':' + os.environ['PATH'], + 'KIN_PROJECT_ROOT': str(root), 'VM2API_CONTROL_PLANE_MODE': mode, + 'VM2API_DATABASE_URL': 'postgresql://synthetic.invalid/test', + 'VM2API_ADMIN_USER': 'admin', 'VM2API_ADMIN_PASSWORD': 'synthetic-password', + 'VM2API_API_KEY': 'synthetic-key', 'VM2API_DB_SECRET': 'synthetic-secret', + 'KIN_AUTO_SYNC_WRAP': '0', 'TEST_NODE_LOG': str(root / 'node.log')} + for key in ('KIN_KERNEL_BIN', 'KIN_API_KERNEL_BIN'): + env.pop(key, None) + with socket.socket(socket.AF_UNIX) as daemon: + daemon.bind(str(root / 'docker.sock')) + result = subprocess.run(['sh', str(script)], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + calls = [json.loads(line) for line in (root / 'node.log').read_text().splitlines()] + self.assertEqual(any('--pull-only' in call for call in calls), mode == 'standalone') + self.assertEqual(calls[-1], ['src/server.mjs']) + self.assertNotIn('docker.sock not mounted', result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/deploy/test_kubernetes_smoke.py b/test/deploy/test_kubernetes_smoke.py new file mode 100644 index 000000000..39f2cfae0 --- /dev/null +++ b/test/deploy/test_kubernetes_smoke.py @@ -0,0 +1,64 @@ +"""Exercise smoke-test ownership guards without accessing Kubernetes.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('vm2api_kubernetes_smoke', Path(__file__).with_name('smoke_kubernetes.py')) +smoke_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(smoke_module) +OWNER_LABEL, Smoke = smoke_module.OWNER_LABEL, smoke_module.Smoke + + +class KubernetesSmokeOwnershipTests(unittest.TestCase): + def setUp(self): + self.smoke = Smoke('synthetic-context', 'synthetic-image', None) + + def result(self, value): + return subprocess.CompletedProcess([], 0, stdout=json.dumps(value), stderr='') + + def test_remote_context_is_rejected_before_creating_a_namespace(self): + result = subprocess.CompletedProcess([], 0, stdout='https://remote.invalid:6443', stderr='') + with patch.object(self.smoke, 'kubectl', return_value=result) as kubectl: + with self.assertRaisesRegex(RuntimeError, 'local Kubernetes context'): + self.smoke.create_namespace() + self.assertEqual(kubectl.call_count, 1) + self.assertIsNone(self.smoke.namespace_uid) + + def test_existing_namespace_is_never_reused(self): + local = subprocess.CompletedProcess([], 0, stdout='https://127.0.0.1:6443', stderr='') + with patch.object(self.smoke, 'kubectl', side_effect=[local, self.result({})]) as kubectl: + with self.assertRaisesRegex(RuntimeError, 'refusing to reuse'): + self.smoke.create_namespace() + self.assertEqual(kubectl.call_count, 2) + self.assertIsNone(self.smoke.namespace_uid) + + def test_changed_namespace_ownership_prevents_deletion(self): + self.smoke.namespace_uid = 'created-uid' + for uid, label in [('replacement-uid', self.smoke.run_id), ('created-uid', 'another-run')]: + with self.subTest(uid=uid, label=label): + resource = {'metadata': {'uid': uid, 'labels': {OWNER_LABEL: label}}} + with patch.object(self.smoke, 'kubectl', return_value=self.result(resource)) as kubectl: + with self.assertRaisesRegex(RuntimeError, 'refusing cleanup'): + self.smoke.cleanup() + self.assertEqual(kubectl.call_count, 1) + + def test_cleanup_deletes_only_the_namespace_created_by_this_run(self): + self.smoke.namespace_uid = 'created-uid' + resource = {'metadata': {'uid': 'created-uid', 'labels': {OWNER_LABEL: self.smoke.run_id}}} + with patch.object(self.smoke, 'kubectl', return_value=self.result(resource)) as kubectl: + self.smoke.cleanup() + self.assertEqual(kubectl.call_count, 3) + deletion = kubectl.call_args_list[1] + self.assertEqual(deletion.args, + ('delete', '--raw=/api/v1/namespaces/' + self.smoke.namespace, '-f', '-')) + self.assertEqual(json.loads(deletion.kwargs['content'])['preconditions'], {'uid': 'created-uid'}) + self.assertFalse(deletion.kwargs['namespaced']) + kubectl.assert_called_with('wait', '--for=delete', 'namespace/' + self.smoke.namespace, + '--timeout=180s', namespaced=False, timeout=195) + + +if __name__ == '__main__': + unittest.main() From c72d31c2bfc8af099228507ab74cb909f9299132 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:05:23 +0800 Subject: [PATCH 035/473] feat(k8s): guard controller startup and drain requests before shutdown --- src/lib/admin/panel-routes.mjs | 8 +- src/lib/core/control-plane-lifecycle.mjs | 92 ++++++++++++ src/lib/core/control-plane-mode.mjs | 19 +++ src/lib/core/security.mjs | 23 ++- src/lib/protocol/responses-ws.mjs | 87 +++++++++-- src/server.mjs | 161 ++++++++++++++++++--- test/unit/control-plane-lifecycle.test.mjs | 8 +- test/unit/responses-ws.test.mjs | 12 +- test/unit/security.test.mjs | 55 ++++++- 9 files changed, 417 insertions(+), 48 deletions(-) create mode 100644 src/lib/core/control-plane-lifecycle.mjs create mode 100644 src/lib/core/control-plane-mode.mjs diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 5fa6a2d94..501f60bfe 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -1344,7 +1344,7 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && p === '/api/panel/logout') { const tok = extractPanelToken(req) - if (tok) revokePanelSession(tok) + if (tok) await revokePanelSession(tok) const secure = panelCookieSecure(req) res.setHeader('Set-Cookie', clearPanelSessionCookie({ secure })) return json(res, 200, { ok: true }) @@ -1506,9 +1506,9 @@ export function createPanelHandler(ctx) { if (!rec) return json(res, 404, { ok: false, error: { message: 'user not found' } }) // Sessions cache the role from login time, so a role or enable change must log them out. if (before && (before.role !== rec.role || !!before.enabled !== !!rec.enabled)) { - revokePanelSessionsForUser(rec.username) + await revokePanelSessionsForUser(rec.username) } else if (body?.password) { - revokePanelSessionsForUser(rec.username, { exceptToken: extractPanelToken(req) }) + await revokePanelSessionsForUser(rec.username, { exceptToken: extractPanelToken(req) }) } return json(res, 200, panel.ok({ item: publicUserView(rec) })) } catch (e) { @@ -1525,7 +1525,7 @@ export function createPanelHandler(ctx) { try { const ok = await panelUsers.remove(id, { actorId: req.panelUserId }) if (!ok) return json(res, 404, { ok: false, error: { message: 'user not found' } }) - if (existing?.username) revokePanelSessionsForUser(existing.username) + if (existing?.username) await revokePanelSessionsForUser(existing.username) return json(res, 200, { ok: true, deleted: id }) } catch (e) { const status = e.code === 'last_admin' || e.code === 'self_delete' ? 409 : 400 diff --git a/src/lib/core/control-plane-lifecycle.mjs b/src/lib/core/control-plane-lifecycle.mjs new file mode 100644 index 000000000..2c5da46a3 --- /dev/null +++ b/src/lib/core/control-plane-lifecycle.mjs @@ -0,0 +1,92 @@ +export async function withTimeout(operation, timeoutMs, code = 'shutdown_timeout') { + let timer + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timer = setTimeout(() => reject(Object.assign(new Error(code), { code })), timeoutMs) + }), + ]) + } finally { + clearTimeout(timer) + } +} + +export class ControlPlaneLifecycle { + constructor({ + probe = async () => {}, + lockHealthy = () => true, + isMaintaining = () => false, + probeTimeoutMs = 1500, + } = {}) { + this.probe = probe + this.lockHealthy = lockHealthy + this.isMaintaining = isMaintaining + this.probeTimeoutMs = probeTimeoutMs + this.initialized = false + this.draining = false + this.requests = new Set() + this.waiters = new Set() + this.probing = null + } + + markReady() { + this.initialized = true + } + + beginDrain() { + this.draining = true + } + + async isReady() { + const allowed = () => this.initialized && !this.draining && this.lockHealthy() && !this.isMaintaining() + if (!allowed()) return false + try { + this.probing ||= Promise.resolve() + .then(() => this.probe()) + .finally(() => { + this.probing = null + }) + await withTimeout(this.probing, this.probeTimeoutMs, 'readiness_timeout') + return allowed() + } catch { + return false + } + } + + trackHttp(req, res) { + const entry = { req, res, handlerDone: false, responseDone: false } + this.requests.add(entry) + const settle = () => { + if (!entry.handlerDone || !entry.responseDone) return + this.requests.delete(entry) + res.off('finish', finished) + res.off('close', finished) + if (!this.requests.size) for (const resolve of this.waiters) resolve() + } + const finished = () => { + entry.responseDone = true + settle() + } + res.once('finish', finished) + res.once('close', finished) + return () => { + entry.handlerDone = true + settle() + } + } + + waitForIdle() { + if (!this.requests.size) return Promise.resolve() + return new Promise((resolve) => this.waiters.add(resolve)).finally(() => { + if (!this.requests.size) this.waiters.clear() + }) + } + + forceClose() { + for (const { req, res } of this.requests) { + res.destroy?.() + req.destroy?.() + } + } +} diff --git a/src/lib/core/control-plane-mode.mjs b/src/lib/core/control-plane-mode.mjs new file mode 100644 index 000000000..5ec005939 --- /dev/null +++ b/src/lib/core/control-plane-mode.mjs @@ -0,0 +1,19 @@ +export function controlPlaneMode(env = process.env) { + const mode = env.VM2API_CONTROL_PLANE_MODE || 'standalone' + if (mode !== 'standalone' && mode !== 'kubernetes') { + throw new Error('VM2API_CONTROL_PLANE_MODE must be standalone or kubernetes') + } + return mode +} + +export function isKubernetesMode(env = process.env) { + return controlPlaneMode(env) === 'kubernetes' +} + +export function shutdownTimeoutMs(env = process.env) { + const value = Number(env.VM2API_SHUTDOWN_TIMEOUT_MS ?? 60_000) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error('VM2API_SHUTDOWN_TIMEOUT_MS must be a positive integer') + } + return value +} diff --git a/src/lib/core/security.mjs b/src/lib/core/security.mjs index 97d92d2d0..af188aab1 100644 --- a/src/lib/core/security.mjs +++ b/src/lib/core/security.mjs @@ -2,6 +2,8 @@ import crypto from 'node:crypto' import fs from 'node:fs' import path from 'node:path' import { atomicWriteJson } from '../vm/vm-file.mjs' +import { isKubernetesMode } from './control-plane-mode.mjs' +import { PanelSessionsRepo } from '../db/repos/panel-sessions-repo.mjs' export function hashKey(key) { return crypto.createHash('sha256').update(String(key)).digest('hex') @@ -140,7 +142,7 @@ function saveSessions() { } } -loadSessions() +if (!isKubernetesMode()) loadSessions() export async function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra = {}) { if (ttlMs && typeof ttlMs === 'object') { @@ -149,6 +151,10 @@ export async function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra } const token = 'kin-panel-' + crypto.randomBytes(24).toString('hex') const role = extra.role || (_lookupRole && (await _lookupRole(username))) || 'admin' + if (isKubernetesMode()) { + await new PanelSessionsRepo().create({ tokenHash: panelSessionKey(token), username, role, ttlMs }) + return token + } panelSessions.set(panelSessionKey(token), { user: username, role, @@ -161,6 +167,15 @@ export async function createPanelSession(username, ttlMs = SESSION_TTL_MS, extra export async function verifyPanelSession(token) { if (!token || !token.startsWith('kin-panel-')) return null const key = panelSessionKey(token) + if (isKubernetesMode()) { + const repo = new PanelSessionsRepo() + const session = await repo.get(key) + if (!session) return null + if (session.exp - Date.now() < SESSION_TTL_MS / 2) { + return (await repo.renew(key, { expectedExp: session.exp, ttlMs: SESSION_TTL_MS })) || (await repo.get(key)) + } + return session + } let s = panelSessions.get(key) if (!s) { // reload from disk once (multi-process / restart) @@ -189,16 +204,18 @@ export async function verifyPanelSession(token) { return s } -export function revokePanelSession(token) { +export async function revokePanelSession(token) { if (!token) return + if (isKubernetesMode()) return new PanelSessionsRepo().remove(panelSessionKey(token)) panelSessions.delete(panelSessionKey(token)) saveSessions() } -export function revokePanelSessionsForUser(username, { exceptToken } = {}) { +export async function revokePanelSessionsForUser(username, { exceptToken } = {}) { const name = String(username || '') if (!name) return 0 const exceptKey = exceptToken ? panelSessionKey(exceptToken) : null + if (isKubernetesMode()) return new PanelSessionsRepo().removeForUser(name, { exceptHash: exceptKey }) let n = 0 for (const [key, s] of panelSessions.entries()) { if (s.user !== name) continue diff --git a/src/lib/protocol/responses-ws.mjs b/src/lib/protocol/responses-ws.mjs index f0b44a2ce..6549b9827 100644 --- a/src/lib/protocol/responses-ws.mjs +++ b/src/lib/protocol/responses-ws.mjs @@ -455,6 +455,17 @@ export function createResponsesWsHandler({ maxBodyBytes = DEFAULT_MAX_REQUEST_BYTES, } = {}) { const wss = new WebSocketServer({ noServer: true, maxPayload: maxBodyBytes }) + const connections = new Map() + const tasks = new Set() + let draining = false + + const drainingError = () => + responsesWsErrorFrame({ + status: 503, + type: 'overloaded_error', + code: 'control_plane_draining', + message: 'Control plane is draining', + }) function handshakeRes(socket) { return { @@ -512,12 +523,27 @@ export function createResponsesWsHandler({ currentReq?.abort?.() } catch {} } + const closeIdle = () => { + if (draining && !busy && ws.readyState === ws.OPEN) ws.close(1001, 'Control plane is draining') + } + connections.set(ws, { + drain() { + if (queue.length) sendFrame(ws, drainingError()) + queue.length = 0 + queuedBytes = 0 + closeIdle() + }, + terminate() { + abortCurrent() + ws.terminate() + }, + }) const pump = async () => { if (busy) return busy = true try { - while (!closed && queue.length) { + while (!closed && !draining && queue.length) { const payload = queue.shift() queuedBytes -= Buffer.byteLength(payload) const decoded = decodeResponseCreate(payload) @@ -634,6 +660,7 @@ export function createResponsesWsHandler({ } finally { busy = false currentReq = null + closeIdle() } } @@ -674,6 +701,10 @@ export function createResponsesWsHandler({ sendFrame(ws, responseInterruptAcceptedFrame(clientMessage.value)) return } + if (draining) { + sendFrame(ws, drainingError()) + return + } const routing = typeof getRouting === 'function' ? getRouting() || {} : {} const bytes = Buffer.byteLength(raw) if (bytes + queuedBytes > inboundBodyLimit(routing, maxBodyBytes)) { @@ -699,20 +730,25 @@ export function createResponsesWsHandler({ } queue.push(raw) queuedBytes += bytes - pump().catch((error) => { - console.warn('[responses-ws] connection failed', error?.code || error?.name || 'unknown_error') - sendFrame( - ws, - responsesWsErrorFrame({ - status: 500, - type: 'api_error', - code: 'api_error', - message: 'Responses connection failed', - }), - ) - }) + const task = pump() + tasks.add(task) + task + .catch((error) => { + console.warn('[responses-ws] connection failed', error?.code || error?.name || 'unknown_error') + sendFrame( + ws, + responsesWsErrorFrame({ + status: 500, + type: 'api_error', + code: 'api_error', + message: 'Responses connection failed', + }), + ) + }) + .finally(() => tasks.delete(task)) }) ws.on('close', () => { + connections.delete(ws) closed = true queue.length = 0 queuedBytes = 0 @@ -732,13 +768,36 @@ export function createResponsesWsHandler({ return false } if (!isResponsesWsPath(pathname)) return false + if (draining) { + writeHttpSocket(socket, 503, { error: drainingError().error }) + return true + } const routing = typeof getRouting === 'function' ? getRouting() || {} : {} if (rejectNewWs(socket, routing)) return true const res = handshakeRes(socket) if (typeof requireAuth === 'function' && !(await requireAuth(req, res))) return true + if (draining) { + writeHttpSocket(socket, 503, { error: drainingError().error }) + return true + } wss.handleUpgrade(req, socket, head, (ws) => attach(ws, req)) return true } - return { handleUpgrade, wss } + function beginDrain() { + draining = true + for (const connection of connections.values()) connection.drain() + } + + async function drain() { + beginDrain() + while (tasks.size) await Promise.allSettled([...tasks]) + } + + function terminate() { + beginDrain() + for (const connection of connections.values()) connection.terminate() + } + + return { handleUpgrade, wss, beginDrain, drain, terminate } } diff --git a/src/server.mjs b/src/server.mjs index 1bb90387d..f64f934d1 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -104,6 +104,9 @@ import { createHandleProtocol } from './lib/protocol/handle-protocol.mjs' import { createResponsesWsHandler } from './lib/protocol/responses-ws.mjs' import { handleUserCountTokens, handleUserUsage } from './lib/protocol/user-count-tokens.mjs' import { createPanelHandler } from './lib/admin/panel-routes.mjs' +import { isKubernetesMode, shutdownTimeoutMs } from './lib/core/control-plane-mode.mjs' +import { ControlPlaneLifecycle, withTimeout } from './lib/core/control-plane-lifecycle.mjs' +import { ControlPlaneLock } from './lib/db/control-plane-lock.mjs' const FEATURES = [ 'passthrough', @@ -135,7 +138,8 @@ const cfg = loadConfig() if (!getPanelAdmin().password) { throw new Error('VM2API_ADMIN_PASSWORD must be set; insecure default panel credentials are disabled') } -fs.mkdirSync(cfg.paths.captures, { recursive: true }) +const kubernetesMode = isKubernetesMode() +const shutdownGraceMs = shutdownTimeoutMs() const allowRate = createRateLimiter({ capacity: cfg.limits.rate_capacity, @@ -146,7 +150,54 @@ const allowRate = createRateLimiter({ const activeRequests = new Set() let _shuttingDown = false const dataDir = cfg.paths.data || path.join(cfg.paths.root, 'data') -await openDatabase({ dataDir }) +const db = await openDatabase({ dataDir }) +let controlPlaneLock = null +let startupAborting = false +const lifecycle = new ControlPlaneLifecycle({ + probe: () => db.prepare('SELECT 1').get(), + lockHealthy: () => !kubernetesMode || controlPlaneLock?.healthy === true, + isMaintaining: () => backupService.isMaintaining, +}) +async function failStartup(error) { + if (startupAborting) return + startupAborting = true + lifecycle.beginDrain() + console.error(`[startup] failed: ${error?.code || error?.name || 'startup_failed'}`) + const deadline = setTimeout(() => process.exit(1), 5000) + try { + await controlPlaneLock?.stop() + } catch {} + try { + await closeDatabase() + } catch {} + clearTimeout(deadline) + process.exit(1) +} +if (kubernetesMode) { + process.once('uncaughtException', failStartup) + process.once('unhandledRejection', failStartup) + controlPlaneLock = new ControlPlaneLock(db, { + onLost(error) { + lifecycle.beginDrain() + console.error('[control-plane] exclusive lock lost') + if (!lifecycle.initialized) { + void failStartup(error) + return + } + void shutdown('control_plane_lock_lost', { failed: true }) + }, + }) + try { + await controlPlaneLock.start() + } catch (error) { + await controlPlaneLock.stop() + await closeDatabase() + throw Object.assign(new Error(`Control plane startup failed: ${error.code || 'lock_unavailable'}`), { + code: error.code || 'lock_unavailable', + }) + } +} +fs.mkdirSync(cfg.paths.captures, { recursive: true }) const legacyImport = await runLegacyImport({ dataDir, projectRoot: cfg.paths.project }) if (legacyImport?.imported) { console.log('[db] legacy JSON import done:', JSON.stringify(legacyImport.counts || {})) @@ -915,6 +966,11 @@ const protocolHandlers = createHandleProtocol({ }) async function dispatchProtocol(name, req, res, ...args) { + if (lifecycle.draining || (kubernetesMode && controlPlaneLock?.healthy !== true)) { + return json(res, 503, { + error: { type: 'overloaded_error', code: 'control_plane_draining', message: 'Control plane is draining' }, + }) + } if (backupService.isMaintaining) { return json(res, 503, { error: { type: 'overloaded_error', code: 'maintenance_in_progress', message: 'Database maintenance in progress' }, @@ -1089,7 +1145,10 @@ const handlePanel = createPanelHandler({ slotShell, }) +const sockets = new Set() +const terminalSockets = new Set() const server = http.createServer(async (req, res) => { + const handlerFinished = lifecycle.trackHttp(req, res) activeRequests.add(req) try { res.req = req @@ -1104,6 +1163,24 @@ const server = http.createServer(async (req, res) => { } const p = url.pathname + if ((req.method === 'GET' || req.method === 'HEAD') && p === '/readyz') { + const ready = await lifecycle.isReady() + if (req.method === 'HEAD') { + res.writeHead(ready ? 200 : 503) + return res.end() + } + return json(res, ready ? 200 : 503, { status: ready ? 'ready' : 'not_ready' }) + } + if ( + (lifecycle.draining || (kubernetesMode && (!lifecycle.initialized || controlPlaneLock?.healthy !== true))) && + p !== '/healthz' && + p !== '/health' + ) { + return json(res, 503, { + error: { type: 'overloaded_error', code: 'control_plane_draining', message: 'Control plane is draining' }, + }) + } + if (isTelemetryPath(p)) { return json(res, 200, telemetryInterceptResponse(p)) } @@ -1258,15 +1335,29 @@ const server = http.createServer(async (req, res) => { } } finally { activeRequests.delete(req) + handlerFinished() } }) +server.on('connection', (socket) => { + sockets.add(socket) + socket.once('close', () => { + sockets.delete(socket) + terminalSockets.delete(socket) + }) +}) + server.on('upgrade', async (req, socket, head) => { - if (backupService.isMaintaining) { + if ( + lifecycle.draining || + (kubernetesMode && (!lifecycle.initialized || controlPlaneLock?.healthy !== true)) || + backupService.isMaintaining + ) { socket.end('HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n') return } if (await responsesWs.handleUpgrade(req, socket, head)) return + terminalSockets.add(socket) if (clusterRoutes.handleUpgrade(req, socket, head)) return if (!slotShell.handleUpgrade(req, socket, head)) socket.destroy() }) @@ -1320,10 +1411,24 @@ process.on('uncaughtException', (e) => console.error('[uncaught]', e)) process.on('unhandledRejection', (e) => console.error('[unhandled]', e)) // Graceful shutdown: stop schedulers/watchers, drain persistence, close the pool. -async function shutdown(signal) { +async function shutdown(signal, { failed = false } = {}) { if (_shuttingDown) return _shuttingDown = true + lifecycle.beginDrain() + responsesWs.beginDrain() console.log(`[shutdown] ${signal} — closing`) + const deadline = Date.now() + shutdownGraceMs + const forceClose = () => { + lifecycle.forceClose() + responsesWs.terminate() + server.closeAllConnections?.() + for (const socket of sockets) socket.destroy() + } + const hardStop = setTimeout(() => { + console.error('[shutdown] cleanup deadline exceeded') + forceClose() + process.exit(1) + }, shutdownGraceMs + 5000) try { vmTestSchedule.stop() } catch {} @@ -1355,33 +1460,44 @@ async function shutdown(signal) { try { notifyMonitor?.stop?.() } catch {} + requestLog.stopCleanupScheduler() try { stopVmWatch() } catch {} + const serverClosed = new Promise((resolve) => server.close(resolve)) + for (const socket of terminalSockets) socket.end() try { - stopAllRustKernels() - } catch {} - try { - clusterManager.stop() - } catch {} - try { - await new Promise((resolve) => server.close(resolve)) - } catch {} + await withTimeout(Promise.all([serverClosed, lifecycle.waitForIdle(), responsesWs.drain()]), shutdownGraceMs) + } catch { + failed = true + console.error('[shutdown] request drain deadline exceeded') + forceClose() + } try { - await backupService.flush() - await pausePriceCatalog() - for (const monitor of maintenanceMonitors()) await monitor?.flush?.() - await vmTestSchedule.flush() - await flushRuntimeWrites() - await closeDatabase() + await withTimeout( + (async () => { + await backupService.flush() + await pausePriceCatalog() + for (const monitor of maintenanceMonitors()) await monitor?.flush?.() + await vmTestSchedule.flush() + await flushRuntimeWrites() + stopAllRustKernels() + await clusterManager.stop() + await controlPlaneLock?.stop() + await closeDatabase() + })(), + Math.max(1, deadline - Date.now()) + 5000, + ) } catch (error) { + failed = true console.error(`[shutdown] persistence failed: ${error.code || error.name}`) + forceClose() try { - await closeDatabase() + await withTimeout(controlPlaneLock?.stop(), 1000) } catch {} - process.exit(1) } - process.exit(0) + clearTimeout(hardStop) + process.exit(failed ? 1 : 0) } process.on('SIGTERM', async () => await shutdown('SIGTERM')) process.on('SIGINT', async () => await shutdown('SIGINT')) @@ -1475,6 +1591,9 @@ server.listen(cfg.port, cfg.host, async () => { } catch (error) { console.warn('[worker-models] fetch failed', error.message) } + lifecycle.markReady() + process.off('uncaughtException', failStartup) + process.off('unhandledRejection', failStartup) console.log( JSON.stringify({ event: 'vm2api-started', diff --git a/test/unit/control-plane-lifecycle.test.mjs b/test/unit/control-plane-lifecycle.test.mjs index cf8dca674..ff3a671a7 100644 --- a/test/unit/control-plane-lifecycle.test.mjs +++ b/test/unit/control-plane-lifecycle.test.mjs @@ -18,7 +18,9 @@ test('readiness requires initialization, database, control lock, and no maintena let locked = true let maintaining = false const lifecycle = new ControlPlaneLifecycle({ - probe: async () => { if (!database) throw new Error('private connection details') }, + probe: async () => { + if (!database) throw new Error('private connection details') + }, lockHealthy: () => locked, isMaintaining: () => maintaining, }) @@ -44,7 +46,9 @@ test('HTTP drain waits for both the handler and the response to finish', async ( const res = new EventEmitter() const handlerFinished = lifecycle.trackHttp(req, res) let drained = false - const pending = lifecycle.waitForIdle().then(() => { drained = true }) + const pending = lifecycle.waitForIdle().then(() => { + drained = true + }) handlerFinished() await new Promise((resolve) => setTimeout(resolve, 15)) assert.equal(drained, false, 'a returned SSE handler still owns its open response') diff --git a/test/unit/responses-ws.test.mjs b/test/unit/responses-ws.test.mjs index 6d228e517..f43e806c5 100644 --- a/test/unit/responses-ws.test.mjs +++ b/test/unit/responses-ws.test.mjs @@ -975,8 +975,12 @@ test('draining waits for an active Responses hop and refuses the next request', let started let release let hops = 0 - const running = new Promise((resolve) => { started = resolve }) - const gate = new Promise((resolve) => { release = resolve }) + const running = new Promise((resolve) => { + started = resolve + }) + const gate = new Promise((resolve) => { + release = resolve + }) const live = await listen({ routing: ENABLED, requireAuth: allowAuth, @@ -993,7 +997,9 @@ test('draining waits for an active Responses hop and refuses the next request', opened.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'first' })) await running let drained = false - const pending = live.handler.drain().then(() => { drained = true }) + const pending = live.handler.drain().then(() => { + drained = true + }) await new Promise((resolve) => setImmediate(resolve)) assert.equal(drained, false) const rejected = nextFrame(opened.ws, (frame) => frame.error?.code === 'control_plane_draining') diff --git a/test/unit/security.test.mjs b/test/unit/security.test.mjs index 0e5eed3a0..31d550706 100644 --- a/test/unit/security.test.mjs +++ b/test/unit/security.test.mjs @@ -3,6 +3,8 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { PanelSessionsRepo } from '../../src/lib/db/repos/panel-sessions-repo.mjs' const PANEL_SESSION_TTL_MS = 7 * 24 * 3600 * 1000 @@ -50,7 +52,7 @@ test('panel session persistence stores only token hashes', async () => { fs.readdirSync(dir).filter((name) => name.includes('.tmp')), [], ) - security.revokePanelSession(token) + await security.revokePanelSession(token) assert.equal(await security.verifyPanelSession(token), null) } finally { if (previous == null) delete process.env.KIN_DATA_DIR @@ -157,3 +159,54 @@ test('existing loose panel session file is tightened on load', async () => { fs.rmSync(dir, { recursive: true, force: true }) } }) + +test('Kubernetes sessions share PostgreSQL state and never use the local session file', async () => { + const previousMode = process.env.VM2API_CONTROL_PLANE_MODE + const previousDir = process.env.KIN_DATA_DIR + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-k8s-sessions-')) + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + process.env.KIN_DATA_DIR = dir + try { + const first = await import(`../../src/lib/core/security.mjs?k8s-first=${Date.now()}`) + const second = await import(`../../src/lib/core/security.mjs?k8s-second=${Date.now()}`) + const token = await first.createPanelSession('shared-admin') + assert.equal((await second.verifyPanelSession(token)).user, 'shared-admin') + assert.equal(fs.existsSync(path.join(dir, 'panel-sessions.json')), false) + const row = await getDb().prepare('SELECT token_hash FROM panel_sessions WHERE username = $1').get('shared-admin') + assert.match(row.token_hash, /^[a-f0-9]{64}$/) + assert.notEqual(row.token_hash, token) + await first.revokePanelSession(token) + assert.equal(await second.verifyPanelSession(token), null) + + const kept = await first.createPanelSession('shared-admin') + const removed = await second.createPanelSession('shared-admin') + assert.equal(await first.revokePanelSessionsForUser('shared-admin', { exceptToken: kept }), 1) + assert.equal(await second.verifyPanelSession(removed), null) + assert.ok(await second.verifyPanelSession(kept)) + } finally { + if (previousMode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previousMode + if (previousDir == null) delete process.env.KIN_DATA_DIR + else process.env.KIN_DATA_DIR = previousDir + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +test('Kubernetes authentication fails closed when its session database is unavailable', async () => { + const previousMode = process.env.VM2API_CONTROL_PLANE_MODE + const get = PanelSessionsRepo.prototype.get + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + try { + const security = await import(`../../src/lib/core/security.mjs?k8s-failure=${Date.now()}`) + const token = await security.createPanelSession('database-only') + assert.ok(await security.verifyPanelSession(token)) + PanelSessionsRepo.prototype.get = async () => { + throw new Error('synthetic database unavailable') + } + await assert.rejects(security.verifyPanelSession(token), /synthetic database unavailable/) + } finally { + PanelSessionsRepo.prototype.get = get + if (previousMode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previousMode + } +}) From 8702faf61a7972155f41bf526be2ae755f97f842 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:20:14 +0800 Subject: [PATCH 036/473] test(db): reject undersized control-plane connection pools --- test/unit/control-plane-lock.test.mjs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/unit/control-plane-lock.test.mjs b/test/unit/control-plane-lock.test.mjs index 5f7999081..3913445d5 100644 --- a/test/unit/control-plane-lock.test.mjs +++ b/test/unit/control-plane-lock.test.mjs @@ -3,6 +3,24 @@ import assert from 'node:assert/strict' import { getDb, createDatabase } from '../../src/lib/db/database.mjs' import { ControlPlaneLock } from '../../src/lib/db/control-plane-lock.mjs' +test('a one-connection pool rejects control-plane startup without consuming the business connection', async () => { + const db = getDb() + const single = await createDatabase({ connectionString: db.connectionString, schema: db.schema + '_single', max: 1 }) + const lock = new ControlPlaneLock(single) + try { + await assert.rejects(lock.start(), { + code: 'control_plane_pool_too_small', + message: 'Control-plane mode requires a PostgreSQL connection pool with at least 2 connections', + }) + assert.equal(lock.healthy, false) + assert.equal(single.pool.totalCount, single.pool.idleCount) + assert.equal((await single.prepare('SELECT 1 AS n').get()).n, 1) + } finally { + await lock.stop() + await single.close() + } +}) + test('one controller owns each database and schema until explicit stop', async () => { const db = getDb() let lost = 0 From 3a4c5a58ad4810ae416183512c649dfe0b33b25a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:21:02 +0800 Subject: [PATCH 037/473] fix(db): reject control-plane pools without business capacity --- src/lib/db/control-plane-lock.mjs | 6 ++++++ test/unit/control-plane-lock.test.mjs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/db/control-plane-lock.mjs b/src/lib/db/control-plane-lock.mjs index 467a66602..47a37e0a1 100644 --- a/src/lib/db/control-plane-lock.mjs +++ b/src/lib/db/control-plane-lock.mjs @@ -30,6 +30,12 @@ export class ControlPlaneLock { if (this._healthy) return this if (this._starting) return this._starting if (this._lost) throw lockError('control_plane_lock_lost', 'Control-plane lock was lost; restart is required') + if (this.db.pool.options.max < 2) { + throw lockError( + 'control_plane_pool_too_small', + 'Control-plane mode requires a PostgreSQL connection pool with at least 2 connections', + ) + } if (this._stopping) await this._stopping this._closing = false const pending = this._acquire() diff --git a/test/unit/control-plane-lock.test.mjs b/test/unit/control-plane-lock.test.mjs index 3913445d5..6e7c0da3f 100644 --- a/test/unit/control-plane-lock.test.mjs +++ b/test/unit/control-plane-lock.test.mjs @@ -51,7 +51,7 @@ test('one controller owns each database and schema until explicit stop', async ( test('independent schemas do not block each other and stop releases the pool client', async () => { const db = getDb() - const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema + '_other', max: 1 }) + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema + '_other', max: 2 }) const first = new ControlPlaneLock(db) const second = new ControlPlaneLock(other) try { From 9e8c5d79128e31c46081d37c17cc6237a1783f46 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:24:26 +0800 Subject: [PATCH 038/473] test(k8s): require terminal channel tasks to drain --- test/unit/terminal-drain.test.mjs | 87 +++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 test/unit/terminal-drain.test.mjs diff --git a/test/unit/terminal-drain.test.mjs b/test/unit/terminal-drain.test.mjs new file mode 100644 index 000000000..6cdcbc154 --- /dev/null +++ b/test/unit/terminal-drain.test.mjs @@ -0,0 +1,87 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { EventEmitter, once } from 'node:events' +import { WebSocket } from 'ws' +import { createClusterRoutes } from '../../src/lib/cluster/cluster-routes.mjs' +import { createSlotShell } from '../../src/lib/vm/slot-shell.mjs' + +test('cluster terminal drain waits for a pending SSH channel and closes it before returning', async () => { + let release + const gate = new Promise((resolve) => { + release = resolve + }) + let closed = 0 + const stream = new EventEmitter() + stream.close = () => { + closed += 1 + } + const routes = createClusterRoutes({ + manager: { + consumeShellTicket: () => true, + openShell: async () => { + await gate + return stream + }, + }, + }) + const server = http.createServer() + server.on('upgrade', (req, socket, head) => routes.handleUpgrade(req, socket, head)) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const ws = new WebSocket(`ws://127.0.0.1:${server.address().port}/api/panel/cluster/nodes/node-a/shell?ticket=t`) + try { + await once(ws, 'open') + const disconnected = once(ws, 'close') + let drained = false + const draining = routes.drain().then(() => { + drained = true + }) + await disconnected + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(drained, false, 'a channel that is still opening belongs to the drain') + release() + await draining + assert.equal(closed, 1) + } finally { + release() + ws.terminate() + await new Promise((resolve) => server.close(resolve)) + } +}) + +test('slot terminal drain waits for login readiness and rejects the pending upgrade', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'terminal-drain-')) + fs.mkdirSync(path.join(root, 'vms'), { recursive: true }) + fs.writeFileSync(path.join(root, 'vms', 'vm-02.json'), JSON.stringify({ id: 'vm-02' })) + let release + const gate = new Promise((resolve) => { + release = resolve + }) + const shell = createSlotShell({ + projectRoot: root, + authLoginReady: async () => {}, + upgradeLoginReady: () => gate, + }) + const seen = [] + const socket = { write: (data) => seen.push(String(data)), destroy() {} } + try { + const login = await shell.issueTicket('vm-02', { login: true }) + shell.handleUpgrade({ url: `/api/panel/vms/vm-02/shell?ticket=${login.ticket}&login=1` }, socket, Buffer.alloc(0)) + let drained = false + const draining = shell.drain().then(() => { + drained = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(drained, false) + release() + await draining + assert.match(seen[0], /^HTTP\/1\.1 503 /) + assert.equal((await shell.issueTicket('vm-02')).error.code, 'control_plane_draining') + } finally { + release() + fs.rmSync(root, { recursive: true, force: true }) + } +}) From 9548ccd1b9e8a2d80016dca08ff7f26ab2646908 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:32:05 +0800 Subject: [PATCH 039/473] test(k8s): expose unfinished body cancellation during forced drain --- test/unit/control-plane-lifecycle.test.mjs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/unit/control-plane-lifecycle.test.mjs b/test/unit/control-plane-lifecycle.test.mjs index ff3a671a7..4b8d240ad 100644 --- a/test/unit/control-plane-lifecycle.test.mjs +++ b/test/unit/control-plane-lifecycle.test.mjs @@ -1,6 +1,8 @@ import test from 'node:test' import assert from 'node:assert/strict' import { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import { readRawBody } from '../../src/lib/http/respond.mjs' import { controlPlaneMode, shutdownTimeoutMs } from '../../src/lib/core/control-plane-mode.mjs' import { ControlPlaneLifecycle, withTimeout } from '../../src/lib/core/control-plane-lifecycle.mjs' @@ -63,3 +65,19 @@ test('readiness and draining waits have bounded deadlines', async () => { assert.equal(await lifecycle.isReady(), false) await assert.rejects(withTimeout(new Promise(() => {}), 20), { code: 'shutdown_timeout' }) }) + +test('forced shutdown rejects an unfinished body before closing its response', async () => { + const lifecycle = new ControlPlaneLifecycle() + const req = new PassThrough() + const res = new EventEmitter() + res.destroy = () => { + req.destroy() + res.emit('close') + } + const handlerFinished = lifecycle.trackHttp(req, res) + const body = readRawBody(req, 100).finally(handlerFinished) + req.write('{') + lifecycle.forceClose() + await assert.rejects(withTimeout(body, 30), { code: 'request_stream_error' }) + await lifecycle.waitForIdle() +}) From b474c7f86b13969c1c73e3c6fc07c6e84000536b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:37:20 +0800 Subject: [PATCH 040/473] fix(k8s): drain terminal tasks before closing persistence --- src/lib/cluster/cluster-routes.mjs | 38 ++++++- src/lib/core/control-plane-lifecycle.mjs | 2 +- src/lib/vm/slot-shell.mjs | 110 +++++++++++++++++---- src/server.mjs | 26 ++++- test/unit/control-plane-lifecycle.test.mjs | 5 +- 5 files changed, 151 insertions(+), 30 deletions(-) diff --git a/src/lib/cluster/cluster-routes.mjs b/src/lib/cluster/cluster-routes.mjs index aa9961ba0..dda0e070c 100644 --- a/src/lib/cluster/cluster-routes.mjs +++ b/src/lib/cluster/cluster-routes.mjs @@ -166,6 +166,17 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { } const wss = new WebSocketServer({ noServer: true, maxPayload: 1 << 20 }) + const pending = new Set() + let draining = false + let drainError = null + function track(task) { + const tracked = Promise.resolve(task) + .catch((error) => { + drainError ||= error + }) + .finally(() => pending.delete(tracked)) + pending.add(tracked) + } function rejectUpgrade(socket, status, text) { socket.write(`HTTP/1.1 ${status} ${text}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`) @@ -177,6 +188,10 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { const url = new URL(req.url || '/', 'http://local') const sm = SHELL_RE.exec(url.pathname) if (!sm) return false + if (draining) { + rejectUpgrade(socket, 503, 'Service Unavailable') + return true + } const id = sm[1] if (!manager.consumeShellTicket(url.searchParams.get('ticket') || '', id)) { rejectUpgrade(socket, 401, 'Unauthorized') @@ -184,7 +199,10 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { } const cols = clampInt(url.searchParams.get('cols'), 10, 500, 80) const rows = clampInt(url.searchParams.get('rows'), 5, 200, 24) - wss.handleUpgrade(req, socket, head, (ws) => attachShell(ws, id, { cols, rows })) + wss.handleUpgrade(req, socket, head, (ws) => { + track(new Promise((resolve) => ws.once('close', resolve))) + track(attachShell(ws, id, { cols, rows })) + }) return true } @@ -231,5 +249,21 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { for (const msg of pending.splice(0)) apply(msg) } - return { handle, handleUpgrade } + function beginDrain() { + draining = true + for (const ws of wss.clients) ws.close(1001, 'Control plane is draining') + } + + async function drain() { + beginDrain() + while (pending.size) await Promise.all([...pending]) + if (drainError) throw drainError + } + + function terminate() { + draining = true + for (const ws of wss.clients) ws.terminate() + } + + return { handle, handleUpgrade, beginDrain, drain, terminate } } diff --git a/src/lib/core/control-plane-lifecycle.mjs b/src/lib/core/control-plane-lifecycle.mjs index 2c5da46a3..f60b5051a 100644 --- a/src/lib/core/control-plane-lifecycle.mjs +++ b/src/lib/core/control-plane-lifecycle.mjs @@ -85,8 +85,8 @@ export class ControlPlaneLifecycle { forceClose() { for (const { req, res } of this.requests) { + req.destroy?.(Object.assign(new Error('Control plane shutdown deadline exceeded'), { code: 'shutdown_timeout' })) res.destroy?.() - req.destroy?.() } } } diff --git a/src/lib/vm/slot-shell.mjs b/src/lib/vm/slot-shell.mjs index 80a7c2902..d3000fb16 100644 --- a/src/lib/vm/slot-shell.mjs +++ b/src/lib/vm/slot-shell.mjs @@ -381,8 +381,21 @@ export function createSlotShell({ const liveTokens = new Set() let sessions = 0 const wss = new WebSocketServer({ noServer: true, maxPayload: 1 << 20 }) + const pendingTasks = new Set() + const connections = new Map() + let draining = false + let drainError = null + function track(task) { + const tracked = Promise.resolve(task) + .catch((error) => { + drainError ||= error + }) + .finally(() => pendingTasks.delete(tracked)) + pendingTasks.add(tracked) + } async function issueTicket(vmId, { login = false } = {}) { + if (draining) return refuse(503, 'control_plane_draining', 'Control plane is draining') const vm = getVm(projectRoot, vmId) if (!vm) return refuse(404, 'vm_not_found', 'vm not found') if (isCodexVm(vm)) return refuse(409, 'no_container', 'GPT 槽没有容器，无法打开终端') @@ -393,6 +406,7 @@ export function createSlotShell({ return refuse(409, err.code || 'auth_login_not_ready', err.message) } } + if (draining) return refuse(503, 'control_plane_draining', 'Control plane is draining') if (sessions >= MAX_SESSIONS) return refuse(429, 'shell_limit', `终端会话已达上限 ${MAX_SESSIONS}`) const now = Date.now() for (const [t, v] of tickets) if (v.exp <= now) tickets.delete(t) @@ -420,6 +434,10 @@ export function createSlotShell({ const url = new URL(req.url || '/', 'http://local') const sm = SHELL_RE.exec(url.pathname) if (!sm) return false + if (draining) { + rejectUpgrade(socket, 503, 'Service Unavailable') + return true + } const vmId = decodeURIComponent(sm[1]) const ticket = url.searchParams.get('ticket') || '' const entry = consumeTicket(ticket, vmId) @@ -439,6 +457,10 @@ export function createSlotShell({ } const login = !!entry.login const accept = () => { + if (draining) { + rejectUpgrade(socket, 503, 'Service Unavailable') + return + } if (sessions >= MAX_SESSIONS) { rejectUpgrade(socket, 429, 'Too Many Requests') return @@ -449,20 +471,27 @@ export function createSlotShell({ rows: clampInt(url.searchParams.get('rows'), 5, 200, 24), login, } - wss.handleUpgrade(req, socket, head, (ws) => attach(ws, vm, size)) + wss.handleUpgrade(req, socket, head, (ws) => { + track(new Promise((resolve) => ws.once('close', resolve))) + track(attach(ws, vm, size)) + }) } if (!login) { accept() return true } - ;(upgradeLoginReady || authLoginReady || assertAuthLoginReady)(projectRoot, vm).then(accept, (err) => - rejectUpgrade( - socket, - 409, - String(err?.message || 'Conflict') - .replace(/[\r\n]+/g, ' ') - .slice(0, 120) || 'Conflict', - ), + track( + Promise.resolve() + .then(() => (upgradeLoginReady || authLoginReady || assertAuthLoginReady)(projectRoot, vm)) + .then(accept, (err) => + rejectUpgrade( + socket, + 409, + String(err?.message || 'Conflict') + .replace(/[\r\n]+/g, ' ') + .slice(0, 120) || 'Conflict', + ), + ), ) return true } @@ -475,6 +504,11 @@ export function createSlotShell({ let session = null let closed = false let released = false + let adoptTimer = null + connections.set(ws, () => { + clearInterval(adoptTimer) + ws.close(1001, 'Control plane is draining') + }) const release = () => { if (released) return released = true @@ -512,12 +546,16 @@ export function createSlotShell({ } ws.on('close', () => { closed = true + connections.delete(ws) + clearInterval(adoptTimer) release() if (!session) return session.stream.destroy() - reap().catch((err) => { - logger.warn?.(`[slot-shell] ${vm.id} reap failed: ${err.message}`) - }) + track( + reap().catch((err) => { + logger.warn?.(`[slot-shell] ${vm.id} reap failed: ${err.message}`) + }), + ) }) try { connect = slotHost(vm).dockerApi() @@ -542,10 +580,10 @@ export function createSlotShell({ ws.close(1011) return } - if (closed) { + if (closed || draining) { // The close handler ran before the session existed: reap here instead. session.stream.destroy() - reap().catch(() => {}) + await reap().catch(() => {}) return } // ConsoleSize on exec start is API >= 1.42; resize covers older daemons. @@ -568,17 +606,30 @@ export function createSlotShell({ if (report) send(JSON.stringify({ t: 'error', message: '官方登录凭证未能导入槽凭证' })) }, }) - const adoptTimer = size.login ? setInterval(() => void adoptLive(), 1000) : null + adoptTimer = size.login ? setInterval(() => track(adoptLive()), 1000) : null adoptTimer?.unref?.() let ended = false - const onEnd = async () => { + const onEnd = () => { if (ended) return ended = true clearInterval(adoptTimer) - if (size.login) await adoptLive(true) - const info = await inspectExec(connect, session.execId).catch(() => null) - send(JSON.stringify({ t: 'exit', code: info?.ExitCode ?? null })) - ws.close(1000) + track( + (async () => { + if (size.login) { + const adopted = await adoptLive(true) + if ( + draining && + adopted.error && + !['auth_login_incomplete', 'auth_login_scope'].includes(adopted.error.code) + ) { + throw adopted.error + } + } + const info = await inspectExec(connect, session.execId).catch(() => null) + send(JSON.stringify({ t: 'exit', code: info?.ExitCode ?? null })) + ws.close(1000) + })(), + ) } session.stream.on('data', (chunk) => send(chunk)) session.stream.on('end', onEnd) @@ -587,5 +638,22 @@ export function createSlotShell({ for (const msg of pending.splice(0)) apply(msg) } - return { issueTicket, handleUpgrade, liveShellTokens: () => [...liveTokens] } + function beginDrain() { + draining = true + tickets.clear() + for (const close of connections.values()) close() + } + + async function drain() { + beginDrain() + while (pendingTasks.size) await Promise.all([...pendingTasks]) + if (drainError) throw drainError + } + + function terminate() { + beginDrain() + for (const ws of connections.keys()) ws.terminate() + } + + return { issueTicket, handleUpgrade, liveShellTokens: () => [...liveTokens], beginDrain, drain, terminate } } diff --git a/src/server.mjs b/src/server.mjs index f64f934d1..492577815 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1146,7 +1146,6 @@ const handlePanel = createPanelHandler({ }) const sockets = new Set() -const terminalSockets = new Set() const server = http.createServer(async (req, res) => { const handlerFinished = lifecycle.trackHttp(req, res) activeRequests.add(req) @@ -1343,7 +1342,6 @@ server.on('connection', (socket) => { sockets.add(socket) socket.once('close', () => { sockets.delete(socket) - terminalSockets.delete(socket) }) }) @@ -1357,7 +1355,6 @@ server.on('upgrade', async (req, socket, head) => { return } if (await responsesWs.handleUpgrade(req, socket, head)) return - terminalSockets.add(socket) if (clusterRoutes.handleUpgrade(req, socket, head)) return if (!slotShell.handleUpgrade(req, socket, head)) socket.destroy() }) @@ -1416,11 +1413,15 @@ async function shutdown(signal, { failed = false } = {}) { _shuttingDown = true lifecycle.beginDrain() responsesWs.beginDrain() + clusterRoutes.beginDrain() + slotShell.beginDrain() console.log(`[shutdown] ${signal} — closing`) const deadline = Date.now() + shutdownGraceMs const forceClose = () => { lifecycle.forceClose() responsesWs.terminate() + clusterRoutes.terminate() + slotShell.terminate() server.closeAllConnections?.() for (const socket of sockets) socket.destroy() } @@ -1465,9 +1466,17 @@ async function shutdown(signal, { failed = false } = {}) { stopVmWatch() } catch {} const serverClosed = new Promise((resolve) => server.close(resolve)) - for (const socket of terminalSockets) socket.end() try { - await withTimeout(Promise.all([serverClosed, lifecycle.waitForIdle(), responsesWs.drain()]), shutdownGraceMs) + await withTimeout( + Promise.all([ + serverClosed, + lifecycle.waitForIdle(), + responsesWs.drain(), + clusterRoutes.drain(), + slotShell.drain(), + ]), + shutdownGraceMs, + ) } catch { failed = true console.error('[shutdown] request drain deadline exceeded') @@ -1476,6 +1485,13 @@ async function shutdown(signal, { failed = false } = {}) { try { await withTimeout( (async () => { + // An interrupted connection can still be completing its cancellation or login import. + await Promise.allSettled([ + lifecycle.waitForIdle(), + responsesWs.drain(), + clusterRoutes.drain(), + slotShell.drain(), + ]) await backupService.flush() await pausePriceCatalog() for (const monitor of maintenanceMonitors()) await monitor?.flush?.() diff --git a/test/unit/control-plane-lifecycle.test.mjs b/test/unit/control-plane-lifecycle.test.mjs index 4b8d240ad..2a3bcc923 100644 --- a/test/unit/control-plane-lifecycle.test.mjs +++ b/test/unit/control-plane-lifecycle.test.mjs @@ -78,6 +78,9 @@ test('forced shutdown rejects an unfinished body before closing its response', a const body = readRawBody(req, 100).finally(handlerFinished) req.write('{') lifecycle.forceClose() - await assert.rejects(withTimeout(body, 30), { code: 'request_stream_error' }) + await assert.rejects(withTimeout(body, 30), (error) => { + assert.equal(error.body?.error?.code, 'request_stream_error') + return true + }) await lifecycle.waitForIdle() }) From 3ce947f832dc6465d0e1eeecdb4d1425fb8aa2ec Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 15:59:32 +0800 Subject: [PATCH 041/473] test(backup): cover restoring snapshots before the latest PG migration --- test/unit/backup-service.test.mjs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 28709cb93..60b9a9698 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -4,7 +4,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { spawnSync } from 'node:child_process' -import { openDatabase, closeDatabase, getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { openDatabase, closeDatabase, getDb, createDatabase, applyMigrations } from '../../src/lib/db/database.mjs' import { BackupRepo } from '../../src/lib/db/repos/backup-repo.mjs' import { sha256FileAsync } from '../../src/lib/admin/backup-core.mjs' import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' @@ -76,6 +76,22 @@ test('restoring the application schema preserves later writes in a neighboring s } }) +test('restoring a PG 001 snapshot removes later tables before reapplying migration 002', async () => { + const db = getDb() + await db.exec("DROP TABLE panel_sessions; DELETE FROM schema_migrations WHERE version='002'") + await new SettingsRepo(db).set('older-snapshot', 'before') + const rec = await svc.createBackup() + await applyMigrations(db) + await db + .prepare('INSERT INTO panel_sessions VALUES($1,$2,$3,$4)') + .run('after-snapshot', 'admin', 'admin', Date.now() + 60000) + await new SettingsRepo(db).set('older-snapshot', 'after') + assert.equal((await svc.restoreBackup(rec.id)).ok, true) + assert.equal(await new SettingsRepo(db).get('older-snapshot'), 'before') + assert.equal((await db.prepare('SELECT count(*) AS n FROM panel_sessions').get()).n, 0) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 2) +}) + test('createBackup produces tar.gz with manifest + db + vms + config, records ok', async () => { seedVm() await initVmDbSync(root, { watch: false }) From 9b9e92551487dae84e537011a1796e4140553ddc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:00:54 +0800 Subject: [PATCH 042/473] test(k8s): reproduce late SSH terminal credential writes after drain --- test/fixtures/delayed-terminal-drain.mjs | 76 ++++++++++++++++++++++++ test/unit/terminal-drain.test.mjs | 14 +++++ 2 files changed, 90 insertions(+) create mode 100644 test/fixtures/delayed-terminal-drain.mjs diff --git a/test/fixtures/delayed-terminal-drain.mjs b/test/fixtures/delayed-terminal-drain.mjs new file mode 100644 index 000000000..f000b2a6b --- /dev/null +++ b/test/fixtures/delayed-terminal-drain.mjs @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import { EventEmitter, once } from 'node:events' +import { mock } from 'node:test' +import { WebSocket } from 'ws' + +const dockerModule = new URL('../../src/lib/cluster/docker-remote.mjs', import.meta.url).href +const actual = await import(dockerModule) +const tty = new EventEmitter() +tty.write = () => {} +// ssh2 sends EOF/CLOSE here; the peer may acknowledge the close later. +tty.destroy = () => { tty.destroyRequested = true } +mock.module(dockerModule, { + namedExports: { + ...actual, + openExecTty: async () => ({ execId: 'synthetic-exec', stream: tty }), + resizeExec: async () => {}, + execDetached: async () => {}, + inspectExec: async () => ({ ExitCode: 0 }), + }, +}) +const { createSlotShell } = await import('../../src/lib/vm/slot-shell.mjs') +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'terminal-delayed-close-')) +const home = path.join(root, 'vms', 'vm-02', 'cli-home', '.claude') +fs.mkdirSync(home, { recursive: true }) +fs.writeFileSync(path.join(root, 'vms', 'vm-02.json'), JSON.stringify({ id: 'vm-02' })) +fs.writeFileSync(path.join(home, 'credentials.json'), JSON.stringify({ claudeAiOauth: { + accessToken: 'synthetic-access', refreshToken: 'synthetic-refresh', + expiresAt: Date.now() + 3600_000, scopes: ['user:profile', 'user:inference'], +} })) +let databaseClosed = false +let imports = 0 +let writesAfterClose = 0 +const shell = createSlotShell({ + projectRoot: root, + authLoginReady: async () => {}, + upgradeLoginReady: async () => {}, + importOauth: async () => { + imports += 1 + if (databaseClosed) writesAfterClose += 1 + return { ok: true } + }, + logger: { warn() {} }, +}) +const ticket = await shell.issueTicket('vm-02', { login: true }) +const server = http.createServer() +server.on('upgrade', (req, socket, head) => shell.handleUpgrade(req, socket, head)) +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) +const ws = new WebSocket(`ws://127.0.0.1:${server.address().port}/api/panel/vms/vm-02/shell?ticket=${ticket.ticket}&login=1`) +let pending +try { + await once(ws, 'open') + await new Promise((resolve) => setImmediate(resolve)) + let drained = false + pending = shell.drain().then(() => { drained = true }) + await Promise.race([pending, new Promise((resolve) => setTimeout(resolve, 200))]) + const drainedBeforeTtyClose = drained + databaseClosed = drained + assert.equal(tty.destroyRequested, true) + tty.emit('close') + await pending + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(drainedBeforeTtyClose, false, 'drain returned before the SSH TTY close/final adoption') + assert.equal(imports, 1) + assert.equal(writesAfterClose, 0, 'OAuth import began after the drain allowed database closure') +} finally { + tty.emit('close') + await pending?.catch(() => {}) + ws.terminate() + await new Promise((resolve) => server.close(resolve)) + mock.restoreAll() + fs.rmSync(root, { recursive: true, force: true }) +} diff --git a/test/unit/terminal-drain.test.mjs b/test/unit/terminal-drain.test.mjs index 6cdcbc154..ed2e5cee6 100644 --- a/test/unit/terminal-drain.test.mjs +++ b/test/unit/terminal-drain.test.mjs @@ -5,10 +5,24 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { EventEmitter, once } from 'node:events' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' import { WebSocket } from 'ws' import { createClusterRoutes } from '../../src/lib/cluster/cluster-routes.mjs' import { createSlotShell } from '../../src/lib/vm/slot-shell.mjs' +test('slot terminal drain waits for a delayed SSH TTY close and final OAuth import', () => { + const child = spawnSync( + process.execPath, + [ + '--experimental-test-module-mocks', + fileURLToPath(new URL('../fixtures/delayed-terminal-drain.mjs', import.meta.url)), + ], + { encoding: 'utf8', timeout: 5000 }, + ) + assert.equal(child.status, 0, child.stderr || child.error?.message) +}) + test('cluster terminal drain waits for a pending SSH channel and closes it before returning', async () => { let release const gate = new Promise((resolve) => { From 468cb402d4e16a1e1a750f8f128ae4c160a5d29a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:05:38 +0800 Subject: [PATCH 043/473] fix(k8s): await terminal close and final credential import --- src/lib/vm/slot-shell.mjs | 32 ++++++++++++++++-------- test/fixtures/delayed-terminal-drain.mjs | 8 ++++++ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/lib/vm/slot-shell.mjs b/src/lib/vm/slot-shell.mjs index d3000fb16..b6167a2dd 100644 --- a/src/lib/vm/slot-shell.mjs +++ b/src/lib/vm/slot-shell.mjs @@ -608,6 +608,12 @@ export function createSlotShell({ }) adoptTimer = size.login ? setInterval(() => track(adoptLive()), 1000) : null adoptTimer?.unref?.() + let completeTerminal + track( + new Promise((resolve) => { + completeTerminal = resolve + }), + ) let ended = false const onEnd = () => { if (ended) return @@ -615,19 +621,23 @@ export function createSlotShell({ clearInterval(adoptTimer) track( (async () => { - if (size.login) { - const adopted = await adoptLive(true) - if ( - draining && - adopted.error && - !['auth_login_incomplete', 'auth_login_scope'].includes(adopted.error.code) - ) { - throw adopted.error + try { + if (size.login) { + const adopted = await adoptLive(true) + if ( + draining && + adopted.error && + !['auth_login_incomplete', 'auth_login_scope'].includes(adopted.error.code) + ) { + throw adopted.error + } } + const info = await inspectExec(connect, session.execId).catch(() => null) + send(JSON.stringify({ t: 'exit', code: info?.ExitCode ?? null })) + ws.close(1000) + } finally { + completeTerminal() } - const info = await inspectExec(connect, session.execId).catch(() => null) - send(JSON.stringify({ t: 'exit', code: info?.ExitCode ?? null })) - ws.close(1000) })(), ) } diff --git a/test/fixtures/delayed-terminal-drain.mjs b/test/fixtures/delayed-terminal-drain.mjs index f000b2a6b..5d9e3dfd8 100644 --- a/test/fixtures/delayed-terminal-drain.mjs +++ b/test/fixtures/delayed-terminal-drain.mjs @@ -34,6 +34,8 @@ fs.writeFileSync(path.join(home, 'credentials.json'), JSON.stringify({ claudeAiO let databaseClosed = false let imports = 0 let writesAfterClose = 0 +let releaseImport +const importGate = new Promise((resolve) => { releaseImport = resolve }) const shell = createSlotShell({ projectRoot: root, authLoginReady: async () => {}, @@ -41,6 +43,7 @@ const shell = createSlotShell({ importOauth: async () => { imports += 1 if (databaseClosed) writesAfterClose += 1 + await importGate return { ok: true } }, logger: { warn() {} }, @@ -61,12 +64,17 @@ try { databaseClosed = drained assert.equal(tty.destroyRequested, true) tty.emit('close') + await new Promise((resolve) => setImmediate(resolve)) + const drainedDuringImport = drained + releaseImport() await pending await new Promise((resolve) => setImmediate(resolve)) assert.equal(drainedBeforeTtyClose, false, 'drain returned before the SSH TTY close/final adoption') + assert.equal(drainedDuringImport, false, 'drain returned while the final OAuth import was pending') assert.equal(imports, 1) assert.equal(writesAfterClose, 0, 'OAuth import began after the drain allowed database closure') } finally { + releaseImport() tty.emit('close') await pending?.catch(() => {}) ws.terminate() From 1366e203b4b771542702517bc4e1bf0279e6ee8f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:09:25 +0800 Subject: [PATCH 044/473] fix(backup): remove newer tables atomically when restoring older PG snapshots --- src/lib/admin/backup-service.mjs | 40 +++++++++++++++++++++++++------ test/unit/backup-service.test.mjs | 28 ++++++++++++++++++++++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index f8b19643e..d8c8487ac 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -3,7 +3,7 @@ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' import { Worker } from 'node:worker_threads' -import { getDb, applyMigrations } from '../db/database.mjs' +import { getDb, applyMigrations, quoteIdentifier } from '../db/database.mjs' import { sha256FileAsync, runPgTool, runProcess, extractArchiveMember } from './backup-core.mjs' import { SettingsRepo } from '../db/repos/settings-repo.mjs' import { BackupRepo } from '../db/repos/backup-repo.mjs' @@ -347,6 +347,7 @@ export class BackupService { const dump = path.join(tmp, 'db', 'postgres.dump') const listing = await runPgTool('pg_restore', ['--list', dump], getDb().connectionString) const schema = getDb().schema + const tables = new Set() const allowed = /^(TABLE DATA|SEQUENCE OWNED BY|SEQUENCE SET|FK CONSTRAINT|TABLE|SEQUENCE|DEFAULT|CONSTRAINT|INDEX|SCHEMA|COMMENT) (\S+) (.+)$/ for (const line of listing.split('\n')) { @@ -362,8 +363,9 @@ export class BackupService { ) { throw new Error('backup contains objects outside the application schema allowlist') } + if (item[1] === 'TABLE') tables.add(item[3].split(' ')[0]) } - return { tmp, dump, manifest } + return { tmp, dump, manifest, tables } } catch (error) { fs.rmSync(tmp, { recursive: true, force: true }) throw error @@ -398,20 +400,44 @@ export class BackupService { } async _applyArchive(archive) { + const db = getDb() + // pg_restore --clean only removes objects present in the archive. Later + // migrations can add tables (including login sessions) that must disappear + // before restoring an older snapshot and reapplying those migrations. + const extraTables = (await db.prepare('SELECT tablename FROM pg_tables WHERE schemaname=$1').all(db.schema)) + .map(({ tablename }) => tablename) + .filter((name) => !archive.tables.has(name)) + const cleanup = path.join(archive.tmp, 'cleanup.sql') + const restore = path.join(archive.tmp, 'restore.sql') + fs.writeFileSync( + cleanup, + extraTables.length + ? `DROP TABLE ${extraTables.map((name) => `${quoteIdentifier(db.schema)}.${quoteIdentifier(name)}`).join(', ')};\n` + : '', + { mode: 0o600 }, + ) + fs.writeFileSync(restore, '', { mode: 0o600 }) await runPgTool( 'pg_restore', [ '--clean', '--if-exists', - '--single-transaction', '--no-owner', '--no-privileges', - `--schema=${getDb().schema}`, - '--dbname', - decodeURIComponent(new URL(getDb().connectionString).pathname.slice(1)), + `--schema=${db.schema}`, + '--file', + restore, archive.dump, ], - getDb().connectionString, + db.connectionString, + ) + // Both the removal and restore must commit together. RESTRICT (the default) + // also refuses dependencies outside the application schema without deleting + // neighboring objects. psql streams the potentially multi-GB SQL file. + await runPgTool( + 'psql', + ['--no-psqlrc', '--quiet', '--single-transaction', '--set=ON_ERROR_STOP=1', '--file', cleanup, '--file', restore], + db.connectionString, ) this._restoreFiles(archive.tmp) await applyMigrations(getDb()) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 60b9a9698..0b69a2714 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -92,6 +92,34 @@ test('restoring a PG 001 snapshot removes later tables before reapplying migrati assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 2) }) +test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { + const db = getDb() + await db.exec("DROP TABLE panel_sessions; DELETE FROM schema_migrations WHERE version='002'") + await new SettingsRepo(db).set('older-snapshot', 'before') + const rec = await svc.createBackup() + const archive = await svc._extractArchive(rec) + const neighbor = `"${db.schema}_neighbor"` + try { + await applyMigrations(db) + await new SettingsRepo(db).set('older-snapshot', 'after') + await db + .prepare('INSERT INTO panel_sessions VALUES($1,$2,$3,$4)') + .run('later', 'admin', 'admin', Date.now() + 60000) + await db.exec(`CREATE SCHEMA ${neighbor}; CREATE TABLE ${neighbor}.sentinel(token text REFERENCES panel_sessions(token_hash)); + INSERT INTO ${neighbor}.sentinel VALUES('later')`) + await assert.rejects( + db.withMaintenance(() => svc._applyArchive(archive)), + /psql failed/, + ) + assert.equal(await new SettingsRepo(db).get('older-snapshot'), 'after') + assert.equal((await db.prepare('SELECT token_hash FROM panel_sessions').get()).token_hash, 'later') + assert.equal((await db.prepare(`SELECT token FROM ${neighbor}.sentinel`).get()).token, 'later') + } finally { + await db.exec(`DROP SCHEMA IF EXISTS ${neighbor} CASCADE`) + fs.rmSync(archive.tmp, { recursive: true, force: true }) + } +}) + test('createBackup produces tar.gz with manifest + db + vms + config, records ok', async () => { seedVm() await initVmDbSync(root, { watch: false }) From e8e418b9a532959095a6c8aa5ccd70dc1f63a09a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:24:14 +0800 Subject: [PATCH 045/473] test(kubernetes): detect buffered SSE text deltas in runtime smoke --- test/deploy/smoke_kubernetes.py | 37 ++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/test/deploy/smoke_kubernetes.py b/test/deploy/smoke_kubernetes.py index 70a3c9d4c..27ac28863 100644 --- a/test/deploy/smoke_kubernetes.py +++ b/test/deploy/smoke_kubernetes.py @@ -43,8 +43,10 @@ const event=(type,data)=>res.write(`event: ${type}\ndata: ${JSON.stringify({type,...data})}\n\n`); event('message_start',{message}); event('content_block_start',{index:0,content_block:{type:'text',text:''}}); + const split=Math.floor(proof.length/2); + event('content_block_delta',{index:0,delta:{type:'text_delta',text:proof.slice(0,split)}}); setTimeout(()=>{ - event('content_block_delta',{index:0,delta:{type:'text_delta',text:proof}}); + event('content_block_delta',{index:0,delta:{type:'text_delta',text:proof.slice(split)}}); event('content_block_stop',{index:0}); event('message_delta',{delta:{stop_reason:'end_turn',stop_sequence:null},usage:{output_tokens:3}}); event('message_stop',{});res.end(); @@ -157,6 +159,9 @@ def deploy(self): config['data']['KIN_BACKUP_DISABLED'] = '1' self.apply(*resources) self.kubectl('rollout', 'status', 'deployment/vm2api', '--timeout=240s', timeout=255) + pods = json.loads(self.kubectl('get', 'pods', '-l', 'app.kubernetes.io/name=vm2api', '-o', 'json').stdout) + image_id = pods['items'][0]['status']['containerStatuses'][0]['imageID'] + print(json.dumps({'event': 'pod-ready', 'namespace': self.namespace, 'image_id': image_id}), flush=True) self.checks.append('pod-ready') self.kubectl('exec', 'deployment/vm2api', '-c', 'control-plane', '--', 'node', '--input-type=module', '-e', "const r=await fetch('http://vm2api:8787/readyz');if(r.status!==200)process.exit(1)") @@ -227,16 +232,34 @@ def inference(self, managed_key): 'x-request-id': self.proof + ('-stream' if stream else '-plain')} request = Request(self.base_url + '/v1/messages', headers=headers, data=json.dumps({**body, 'stream': stream}).encode()) + started = time.monotonic() with urlopen(request, timeout=15) as response: assert response.status == 200, 'inference did not succeed' if stream: assert 'text/event-stream' in response.headers.get('content-type', ''), 'SSE response required' - first = response.readline() - first_at = time.monotonic() - content = first + response.read() - assert time.monotonic() - first_at >= 0.5, 'SSE frames were buffered until completion' - assert b'message_stop' in content, 'SSE stream did not finish' - assert self.proof.encode() in content, 'SSE response did not come from the synthetic upstream' + header_ms = round((time.monotonic() - started) * 1000, 1) + frames, deltas = [], [] + for line in response: + if not line.startswith(b'data:'): + continue + frame = json.loads(line[5:]) + arrived_ms = round((time.monotonic() - started) * 1000, 1) + frames.append({'type': frame['type'], 'arrived_ms': arrived_ms}) + if frame.get('delta', {}).get('type') == 'text_delta': + deltas.append({'text': frame['delta']['text'], 'arrived_ms': arrived_ms}) + ended_ms = round((time.monotonic() - started) * 1000, 1) + print(json.dumps({'event': 'sse-timing', 'headers_ms': header_ms, + 'headers': {key: response.headers[key] for key in ( + 'content-type', 'cache-control', 'transfer-encoding', 'x-accel-buffering') + if key in response.headers}, + 'frames': frames, 'deltas': deltas, 'ended_ms': ended_ms}), flush=True) + assert len(deltas) >= 2, 'two upstream text deltas required' + assert deltas[-1]['arrived_ms'] - deltas[0]['arrived_ms'] >= 500, \ + 'SSE text deltas were buffered until completion' + assert ended_ms - deltas[0]['arrived_ms'] >= 500, 'SSE frames were buffered until completion' + assert any(frame['type'] == 'message_stop' for frame in frames), 'SSE stream did not finish' + assert ''.join(delta['text'] for delta in deltas) == self.proof, \ + 'SSE response did not come from the synthetic upstream' else: content = json.loads(response.read()) assert any(item.get('text') == self.proof for item in content['content']), 'upstream proof missing' From 4190522607e16ea08fa634481cd4a94011834d75 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:28:45 +0800 Subject: [PATCH 046/473] test(api-kernel): reproduce buffered SSE text deltas over HTTP --- api-kernel/internal/server/server_test.go | 69 +++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/api-kernel/internal/server/server_test.go b/api-kernel/internal/server/server_test.go index ef30623e4..da7cd165d 100644 --- a/api-kernel/internal/server/server_test.go +++ b/api-kernel/internal/server/server_test.go @@ -1,12 +1,14 @@ package server import ( + "bufio" "encoding/json" "io" "net/http" "net/http/httptest" "strings" "testing" + "time" "github.com/dofastted/kin-gateway/api-kernel/internal/pool" ) @@ -43,3 +45,70 @@ func TestForwardCopiesUpstream(t *testing.T) { t.Fatalf("body %s", rec.Body.String()) } } + +func TestForwardStreamsTextDeltasBeforeUpstreamCompletes(t *testing.T) { + allowSecond := make(chan struct{}, 1) + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + _, _ = io.WriteString(w, "data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"first\"}}\n\n") + w.(http.Flusher).Flush() + select { + case <-allowSecond: + case <-r.Context().Done(): + return + } + _, _ = io.WriteString(w, "data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"second\"}}\n\n") + w.(http.Flusher).Flush() + })) + defer up.Close() + + gateway := httptest.NewServer((&Server{Token: "t", Pool: pool.NewManager()}).Handler()) + defer gateway.Close() + defer close(allowSecond) + env, _ := json.Marshal(forwardEnv{URL: up.URL, Body: `{}`}) + req, _ := http.NewRequest(http.MethodPost, gateway.URL+"/internal/forward", strings.NewReader(string(env))) + req.Header.Set("X-Kin-Internal-Token", "t") + client := gateway.Client() + client.Timeout = 3 * time.Second + type firstRead struct { + response *http.Response + reader *bufio.Reader + line string + err error + } + first := make(chan firstRead, 1) + go func() { + resp, err := client.Do(req) + if err != nil { + first <- firstRead{err: err} + return + } + reader := bufio.NewReader(resp.Body) + line, err := reader.ReadString('\n') + first <- firstRead{response: resp, reader: reader, line: line, err: err} + }() + + var got firstRead + select { + case got = <-first: + if got.response != nil { + defer got.response.Body.Close() + } + if got.err != nil { + t.Fatal(got.err) + } + case <-time.After(time.Second): + t.Fatal("first nonempty text delta was buffered while the upstream awaited the second delta") + } + if got.response.StatusCode != http.StatusOK || !strings.Contains(got.line, `"text":"first"`) { + t.Fatalf("unexpected first delta: status=%d line=%q", got.response.StatusCode, got.line) + } + allowSecond <- struct{}{} + rest, err := io.ReadAll(got.reader) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(rest), `"text":"second"`) { + t.Fatalf("missing second text delta: %q", rest) + } +} From da82550c0322a51e6565aaa774ac13d2ff3a2820 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:29:38 +0800 Subject: [PATCH 047/473] test(backup): protect cross-schema partitions during restore --- test/unit/backup-service.test.mjs | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 0b69a2714..5b5b8132c 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -120,6 +120,44 @@ test('an outside dependency refuses older-snapshot cleanup without changing eith } }) +for (const included of [true, false]) { + test(`restore refuses a cross-schema partition of ${included ? 'an archived' : 'a later'} table before changing either schema`, async () => { + const db = getDb() + const neighbor = `"${db.schema}_neighbor"` + const createParent = 'CREATE TABLE restore_partitioned(id integer) PARTITION BY RANGE(id)' + if (included) await db.exec(createParent) + await new SettingsRepo(db).set('partition-marker', 'before') + const rec = await svc.createBackup() + const archive = await svc._extractArchive(rec) + try { + if (!included) await db.exec(createParent) + await new SettingsRepo(db).set('partition-marker', 'after') + await db.exec(`CREATE SCHEMA ${neighbor}; + CREATE TABLE ${neighbor}.sentinel PARTITION OF restore_partitioned FOR VALUES FROM(0) TO(100); + INSERT INTO ${neighbor}.sentinel VALUES(17)`) + const tables = () => + db + .prepare(` + SELECT c.relname, c.oid FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=$1 AND c.relkind IN ('r','p') ORDER BY c.relname + `) + .all(db.schema) + const before = await tables() + await assert.rejects( + db.withMaintenance(() => svc._applyArchive(archive)), + /psql failed/, + ) + assert.deepEqual(await tables(), before) + assert.equal(await new SettingsRepo(db).get('partition-marker'), 'after') + assert.deepEqual(await db.prepare(`SELECT id FROM ${neighbor}.sentinel`).all(), [{ id: 17 }]) + assert.deepEqual(await db.prepare('SELECT id FROM restore_partitioned').all(), [{ id: 17 }]) + } finally { + await db.exec(`DROP SCHEMA IF EXISTS ${neighbor} CASCADE`) + fs.rmSync(archive.tmp, { recursive: true, force: true }) + } + }) +} + test('createBackup produces tar.gz with manifest + db + vms + config, records ok', async () => { seedVm() await initVmDbSync(root, { watch: false }) From 11ee7dc1c1ba8bcdb513464b695322a5d79e4975 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:31:12 +0800 Subject: [PATCH 048/473] fix(api-kernel): flush forwarded SSE frames as they arrive --- api-kernel/internal/server/server.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/api-kernel/internal/server/server.go b/api-kernel/internal/server/server.go index 5e59b6772..1adb1c179 100644 --- a/api-kernel/internal/server/server.go +++ b/api-kernel/internal/server/server.go @@ -4,6 +4,7 @@ import ( "crypto/subtle" "encoding/json" "io" + "mime" "net/http" "strings" @@ -94,9 +95,31 @@ func (s *Server) forward(w http.ResponseWriter, r *http.Request) { } } w.WriteHeader(resp.StatusCode) + mediaType, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if mediaType == "text/event-stream" { + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + _, _ = io.Copy(flushingWriter{writer: w, flusher: flusher}, resp.Body) + return + } + } _, _ = io.Copy(w, resp.Body) } +// Expose only Write so io.Copy cannot bypass flushing through ReaderFrom. +type flushingWriter struct { + writer io.Writer + flusher http.Flusher +} + +func (w flushingWriter) Write(p []byte) (int, error) { + n, err := w.writer.Write(p) + if n > 0 { + w.flusher.Flush() + } + return n, err +} + func jsonQuote(s string) string { b, _ := json.Marshal(s) return string(b) From 6589eab3ca1179250a24c6ee5a5fee5e738e7f47 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:32:07 +0800 Subject: [PATCH 049/473] fix(backup): reject cross-schema partitions before restoring --- src/lib/admin/backup-service.mjs | 34 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index d8c8487ac..f1fd2045d 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -404,16 +404,31 @@ export class BackupService { // pg_restore --clean only removes objects present in the archive. Later // migrations can add tables (including login sessions) that must disappear // before restoring an older snapshot and reapplying those migrations. - const extraTables = (await db.prepare('SELECT tablename FROM pg_tables WHERE schemaname=$1').all(db.schema)) - .map(({ tablename }) => tablename) - .filter((name) => !archive.tables.has(name)) + const tables = ( + await db.prepare('SELECT tablename FROM pg_tables WHERE schemaname=$1 ORDER BY tablename').all(db.schema) + ).map(({ tablename }) => tablename) + const extraTables = tables.filter((name) => !archive.tables.has(name)) + const tableName = (name) => `${quoteIdentifier(db.schema)}.${quoteIdentifier(name)}` const cleanup = path.join(archive.tmp, 'cleanup.sql') const restore = path.join(archive.tmp, 'restore.sql') fs.writeFileSync( cleanup, - extraTables.length - ? `DROP TABLE ${extraTables.map((name) => `${quoteIdentifier(db.schema)}.${quoteIdentifier(name)}`).join(', ')};\n` - : '', + [ + tables.length ? `LOCK TABLE ${tables.map(tableName).join(', ')} IN ACCESS EXCLUSIVE MODE;` : '', + `DO $$ BEGIN + IF EXISTS ( + SELECT 1 FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class p ON p.oid=i.inhparent + JOIN pg_catalog.pg_namespace pn ON pn.oid=p.relnamespace + JOIN pg_catalog.pg_class c ON c.oid=i.inhrelid + JOIN pg_catalog.pg_namespace cn ON cn.oid=c.relnamespace + WHERE pn.nspname='${db.schema}' AND cn.nspname<>'${db.schema}' + ) THEN + RAISE EXCEPTION 'restore refuses cross-schema inheritance or partitions'; + END IF; + END $$;`, + extraTables.length ? `DROP TABLE ${extraTables.map(tableName).join(', ')};` : '', + ].join('\n'), { mode: 0o600 }, ) fs.writeFileSync(restore, '', { mode: 0o600 }) @@ -431,9 +446,10 @@ export class BackupService { ], db.connectionString, ) - // Both the removal and restore must commit together. RESTRICT (the default) - // also refuses dependencies outside the application schema without deleting - // neighboring objects. psql streams the potentially multi-GB SQL file. + // Lock parents and descendants before checking: attaching or detaching a + // partition must not race the check. RESTRICT alone still drops partitions + // automatically, including ones outside this schema. All cleanup and restore + // statements commit together; psql streams the potentially multi-GB SQL file. await runPgTool( 'psql', ['--no-psqlrc', '--quiet', '--single-transaction', '--set=ON_ERROR_STOP=1', '--file', cleanup, '--file', restore], From d71f9e2d95893d891e7a164eaca83138d4cf029f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 16:39:28 +0800 Subject: [PATCH 050/473] docs(k8s): record scope, runtime evidence and remaining horizontal scaling work --- docs/deploy/postgresql.md | 4 +- ...-10-21-kubernetes-control-plane-results.md | 49 +++++++ .../2026-10-21-kubernetes-control-plane.md | 36 +++++ .../evidence/kubernetes/baseline-e2e.json | 136 ++++++++++++++++++ .../evidence/kubernetes/baseline-unit.json | 66 +++++++++ .../evidence/kubernetes/runtime-after.json | 98 +++++++++++++ .../evidence/kubernetes/runtime-before.json | 35 +++++ .../evidence/kubernetes/verification.json | 59 ++++++++ 8 files changed, 481 insertions(+), 2 deletions(-) create mode 100644 docs/plans/2026-10-21-kubernetes-control-plane-results.md create mode 100644 docs/plans/2026-10-21-kubernetes-control-plane.md create mode 100644 docs/plans/evidence/kubernetes/baseline-e2e.json create mode 100644 docs/plans/evidence/kubernetes/baseline-unit.json create mode 100644 docs/plans/evidence/kubernetes/runtime-after.json create mode 100644 docs/plans/evidence/kubernetes/runtime-before.json create mode 100644 docs/plans/evidence/kubernetes/verification.json diff --git a/docs/deploy/postgresql.md b/docs/deploy/postgresql.md index 26248ffde..f54b6e202 100644 --- a/docs/deploy/postgresql.md +++ b/docs/deploy/postgresql.md @@ -10,7 +10,7 @@ 外部 PostgreSQL：在 `.env` 中填写 `VM2API_DATABASE_URL`，清空 `COMPOSE_PROFILES`。控制面使用 host 网络，数据库地址应为宿主可访问地址，不能使用 Compose 服务名 `postgres`。业务数据库应由本应用专用；应用登录角色需要管理应用 schema 的表、索引、函数和 sequence，无需超级用户或创建/删除数据库权限。应用默认 schema 为 `public`。 -使用 systemd 直接运行 Node.js 时，在 `/etc/vm2api.env` 中配置连接 URL，并安装 PostgreSQL 17 的 `pg_dump` 与 `pg_restore`。`docs/deploy/vm2api.service` 是启动示例。两种控制面镜像均已包含本机架构的 PG 17 客户端。 +使用 systemd 直接运行 Node.js 时，在 `/etc/vm2api.env` 中配置连接 URL，并安装 PostgreSQL 17 的 `pg_dump`、`pg_restore` 与 `psql`。`docs/deploy/vm2api.service` 是启动示例。两种控制面镜像均已包含本机架构的 PG 17 客户端。 ## 已有 SQLite 安装 @@ -33,7 +33,7 @@ 普通备份先短时暂停写入并排空在途任务，在专用事务中导出 PostgreSQL 快照，同时复制 VM 和配置文件。文件复制完成后恢复写入。worker 使用同一个导出快照执行 `pg_dump`，导出事务一直保留到 dump 完成。 -恢复校验外层与内部文件摘要，再检查 `pg_restore --list` 对象目录。恢复只允许本应用 schema 中的已支持对象。数据库使用 `pg_restore --single-transaction --clean --if-exists --no-owner --no-privileges`，不创建或删除整个数据库，也不恢复其他角色的所有权或授权。 +恢复校验外层与内部文件摘要，再检查 `pg_restore --list` 对象目录。恢复只允许本应用 schema 中的已支持对象。`pg_restore --clean --if-exists --no-owner --no-privileges` 生成恢复 SQL，`psql --single-transaction --set=ON_ERROR_STOP=1` 在同一事务内删除快照之外的新增表并执行恢复。事务提交后重新执行后续迁移，新增会话表不会保留快照之后的登录状态。存在其他 schema 的依赖时拒绝删除，不使用 `CASCADE`。此过程不创建或删除整个数据库，也不恢复其他角色的所有权或授权。 恢复期间拒绝新增业务写入，后台任务暂停，所有在途操作排空。恢复前自动创建 `pre_restore` 备份。数据库和文件都恢复后，等待内存 store、路由与调度状态重新加载，再退出维护状态。恢复失败会尝试恢复前快照；失败状态持续保持维护。`data/restore-maintenance.json` 在修改数据库/文件前写入，只有恢复和重新加载成功才删除。进程重启发现该文件时会在迁移或 VM 初始化前拒绝普通启动。进程仍运行时，管理员可以通过原恢复接口显式重试。进程退出后，应在停服状态下使用恢复前归档恢复对应 PostgreSQL、VM 与配置快照，验证一致性后才移除此文件并重新启动。不要在原因未查明时直接删除标记。 diff --git a/docs/plans/2026-10-21-kubernetes-control-plane-results.md b/docs/plans/2026-10-21-kubernetes-control-plane-results.md new file mode 100644 index 000000000..dde4564d2 --- /dev/null +++ b/docs/plans/2026-10-21-kubernetes-control-plane-results.md @@ -0,0 +1,49 @@ +# Kubernetes 控制面验证报告（#648，Draft） + +日期：2026-10-21。已验证源码：`6589eab3ca1179250a24c6ee5a5fee5e738e7f47`。分支：`codex/kubernetes-control-plane`，依赖 PostgreSQL 分支 `cf9462d5`（PR #649）。后续交付提交只补充文档和脱敏证据。 + +## 实现范围 + +已实现单活动控制面的 Kubernetes 部署基础。默认一个副本、Recreate 更新策略；第二个控制面在业务初始化前被 PostgreSQL 独占锁拒绝。面板会话持久化到 PG，只保存令牌摘要。新增就绪检查；退出前停止准入，等待 HTTP、SSE、WebSocket、终端实际关闭及最后一次持久化。容器不需要宿主 Docker socket、hostNetwork、hostPath、特权或 Kubernetes API token。 + +此阶段不开放多副本同时处理推理，不关闭 #648。账号、VM、用户和密钥的全局准入/限流、配置缓存一致性、任务唯一执行、全局维护和远端执行 fencing 尚未实现。部署说明保留单副本限制，不提供 HPA。 + +## 验证结果 + +测试使用本任务创建的专用 PostgreSQL 17.11、合成上游和唯一 Kubernetes 测试命名空间，没有迁移现有部署或使用真实供应商凭据。 + +| 检查 | 被验证提交 | 结果 | +| --- | --- | --- | +| 锁、生命周期、会话、认证/ACL、WebSocket/终端及 K8s E2E | `1366e203` | 12 文件，119/119 通过 | +| 数据库、SQLite 导入、指标和备份 | `6589eab3` | 44/44 通过，含旧 PG 快照恢复、跨 schema 外键/分区拒绝 | +| Node 全量 unit | `1366e203` | 285 文件；3350 通过、7 失败、10 跳过；7 个断言与原始 `eee8f6e` 逐摘要相同 | +| Node 全量 E2E | `6589eab3` | 27 文件；101 通过、17 失败、1 跳过；17 个断言与原始提交逐摘要相同，无清理超时 | +| Go API kernel | `11ee7dc1` | `go test -race ./...` 通过，新增真实 HTTP 流式回归 | +| Python 部署 | `6589eab3` | 33/33 通过，含清单、入口与测试资源所有权检查 | +| 格式与语法 | `6589eab3` | 本次 22 个 MJS 格式检查、Go 格式、shell 语法及 diff 检查通过 | +| 实际 amd64 容器与 k3s | `6589eab3` | 候选镜像构建成功；12 项运行检查通过，退出 0；测试命名空间已删除 | + +全量测试仍有基线失败，不能标记为全绿。例如 unit 实际 cache TTL 为 `1h`、预期 `undefined`；既有 E2E 包含 thinking 多出 `display: summarized` 的差异。原始基线记录在 PostgreSQL 证据目录；本次对照见 [unit](evidence/kubernetes/baseline-unit.json) 与 [E2E](evidence/kubernetes/baseline-e2e.json)。最后的备份边界修复另以 44 项定向测试验证；没有重跑全量 unit。 + +## 实际 Kubernetes 行为 + +本机 `cube-local` 为单节点 amd64 k3s，存储使用 local-path RWO。只创建并删除带本次唯一标签、经 UID 核对的测试命名空间。未修改其他命名空间、集群配置或已有服务。 + +唯一候选标签：`vm2api:kubernetes-6589eab3`。 + +- containerd manifest：`sha256:a2d4d0ffa17e833eac84c466bc13ccc0985872a277d092cf63cf5d95c29a2340`。 +- manifest config / 实际 Pod imageID：`sha256:2bf42f9e9d5fd7c652dca757e62e69c79364dfc5f548338eddcbcd87ef688dce`。 + +实际验证 Pod 就绪、Service DNS/ClusterIP、探针与控制台、Cookie 登录、PG 指标和托管 API key。通过真实 Go API kernel 调用合成上游，校验本次唯一响应文本及两条成功日志。第二实例因独占锁启动失败，原实例继续就绪。Recreate 重启后，原 Cookie、API key、endpoint 和 PVC 文件保留。 + +首次容器实测暴露了 Go 小 SSE 帧缓冲：上游相隔 1000ms 的两段正文，在客户端仅相隔 0.1ms。先提交失败测试 `41905226`，再提交刷新修复 `11ee7dc1`。新镜像客户端两段正文到达间隔 999.9ms，证明实际运行包含该修复，而非仅检查版本字段。前后记录见 [修复前](evidence/kubernetes/runtime-before.json) 与 [修复后](evidence/kubernetes/runtime-after.json)。 + +独立审查还复现并修复了迟到终端关闭后写库、恢复旧 PG 快照残留后续会话表，以及 DROP TABLE 默认 RESTRICT 仍删除跨 schema 分区的问题。相应失败回归均先于修复单独提交。 + +## 未验证和后续条件 + +- 多节点活动副本同时处理请求、全局容量一致性、执行 fencing 和安全接管未实现。 +- 原生 ARM64 构建运行、真实远端槽位和供应商推理、生产规模压测未验证。 +- 外部 Ingress、跨节点持久存储及节点故障接管未验证；单节点 RWO 和 Pod 重启不能证明这些能力。 + +部署入口参见 [Kubernetes 说明](../deploy/kubernetes.md)，后续多副本验收参见 [实施计划](2026-10-21-kubernetes-control-plane.md)。 diff --git a/docs/plans/2026-10-21-kubernetes-control-plane.md b/docs/plans/2026-10-21-kubernetes-control-plane.md new file mode 100644 index 000000000..f13abec2e --- /dev/null +++ b/docs/plans/2026-10-21-kubernetes-control-plane.md @@ -0,0 +1,36 @@ +# Kubernetes 控制面实现计划（#648） + +日期：2026-10-21。分支：`codex/kubernetes-control-plane`。 +依赖：`codex/postgresql-migration@cf9462d5`（#647）。 + +## 当前范围和完成条件 + +本次先提交可运行、可评审的 Kubernetes 部署基础 Draft PR。默认只允许一个活动控制面。控制面多副本同时处理推理尚未具备安全条件，本 PR 不关闭 #648，也不将单副本部署称为水平扩容。 + +完成条件： + +1. 提供原生 Kubernetes/Kustomize 资源和部署说明；不用宿主 Docker socket、hostNetwork、hostPath 或特权能力，控制面连接外部 PostgreSQL。业务文件使用持久卷，本地二进制和进程 socket 使用临时目录。 +2. 使用显式 Kubernetes 运行模式。部署默认一个副本和 Recreate 策略。启动前取得 PG 独占控制面锁，误扩容时第二个实例在业务初始化之前失败；失去锁时拒绝新请求并排空退出，禁止悄悄运行多个写入者。 +3. `/healthz` 保留存活语义；新增 `/readyz` 检查初始化、PG、控制面锁、维护和排空状态。SIGTERM 先停止准入，在限时内等待 HTTP/SSE 与活动 WebSocket 请求及持久化完成，再停止本地运行组件和数据库。 +4. Kubernetes 模式的面板会话存 PG。验证跨连接登录状态、注销、改密撤销及过期行为；令牌只存摘要，故障时不回退到旧本地会话缓存。 +5. 通过针对性的独立 PG 测试、双进程独占保护测试、真实 Kubernetes Pod/Service/探针与 HTTP 验证。只创建本任务独立测试资源；不修改现有部署或其他命名空间。 +6. 推送独立 fork 分支、尝试创建依赖 #647 的上游 Draft PR，并在 #648 评论中注明实现范围、验证 SHA、PR/分支链接和剩余项。 + +## 为什么暂不开放多副本推理 + +当前 key/user/账号/VM 并发与 RPM、Claude 座位、Codex 会话、同会话请求顺序、熔断及 OAuth single-flight 仍有进程内状态。AccountQuota 和 ProxyPool 的缓存快照存在跨实例覆盖问题。RuntimeSettings、路由、模型策略和 Sticky 映射也没有跨实例一致性协议。 + +更关键的是，远端执行取消目前是 best-effort。控制面失联后，不能仅因租约过期就重新分配仍可能执行中的账号/VM 容量。安全的多活需要执行端 fencing 或可确认终止的 orphan 处理协议。活凭据、VM 文件和 Pod 本地 socket 的归属也必须先明确。 + +因此，本阶段提供独占保护和必要基础，不能通过修改 `replicas` 绕过限制。单副本 Pod 重启和跨节点数据可用性仍取决于持久卷；本机单节点验证不能证明多节点高可用。 + +## 文件分工 + +- 数据层：新的 PG 迁移、面板会话 repository、独占控制面锁及其测试。 +- 生命周期：server、security、WebSocket 排空、探针及相关测试。 +- 部署：`deploy/k8s/`、容器入口、独立集成测试和部署文档。 +- 集成：本计划、测试环境、PR 和 Issue 评论。 + +## 后续多副本验收 + +后续必须使用至少两个不同节点上的活动副本，验证全局准入/限流/座位、配置同步、认证和一次性票据、原子计数、任务唯一执行、全局维护，以及故障后的 fencing 和安全接管。不同副本须同时处理重叠推理请求；不能以将全部推理转发给 leader 代替水平扩容。 diff --git a/docs/plans/evidence/kubernetes/baseline-e2e.json b/docs/plans/evidence/kubernetes/baseline-e2e.json new file mode 100644 index 000000000..a6cdf59ec --- /dev/null +++ b/docs/plans/evidence/kubernetes/baseline-e2e.json @@ -0,0 +1,136 @@ +{ + "verified_commit": "6589eab3ca1179250a24c6ee5a5fee5e738e7f47", + "baseline_commit": "eee8f6ecb6ca04455e6edef0c8e498fb3be21f21", + "files": 27, + "exit_code": 1, + "log_sha256": "8191fee5a620d736f7d084706823d8d118664ce787db6b75329457ccfc8a8a78", + "totals": { + "tests": 119, + "pass": 101, + "fail": 17, + "skipped": 1, + "cancelled": 0 + }, + "all_failures_match_baseline": true, + "assertions": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "baseline_assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "matches_baseline": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "baseline_assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "matches_baseline": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "matches_baseline": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "baseline_assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "matches_baseline": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "matches_baseline": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "baseline_assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "matches_baseline": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "baseline_assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "matches_baseline": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "baseline_assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "matches_baseline": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "baseline_assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "matches_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "matches_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "baseline_assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "matches_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "matches_baseline": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "baseline_assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "matches_baseline": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "baseline_assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "matches_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "matches_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "baseline_assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "matches_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "matches_baseline": true + } + ] +} diff --git a/docs/plans/evidence/kubernetes/baseline-unit.json b/docs/plans/evidence/kubernetes/baseline-unit.json new file mode 100644 index 000000000..1b6cf34ad --- /dev/null +++ b/docs/plans/evidence/kubernetes/baseline-unit.json @@ -0,0 +1,66 @@ +{ + "verified_commit": "1366e203b4b771542702517bc4e1bf0279e6ee8f", + "baseline_commit": "eee8f6ecb6ca04455e6edef0c8e498fb3be21f21", + "files": 285, + "exit_code": 1, + "log_sha256": "35b1fc56c1c5f0d1349306c7436d6f3b1811438ce286cbdaa0c5bbc495e40df3", + "totals": { + "tests": 3367, + "pass": 3350, + "fail": 7, + "skipped": 10, + "cancelled": 0 + }, + "all_failures_match_baseline": true, + "assertions": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181", + "baseline_assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11", + "baseline_assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d", + "baseline_assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca", + "baseline_assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca", + "matches_baseline": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "matches_baseline": true + } + ] +} diff --git a/docs/plans/evidence/kubernetes/runtime-after.json b/docs/plans/evidence/kubernetes/runtime-after.json new file mode 100644 index 000000000..8b3f4a3fd --- /dev/null +++ b/docs/plans/evidence/kubernetes/runtime-after.json @@ -0,0 +1,98 @@ +{ + "source_commit": "6589eab3ca1179250a24c6ee5a5fee5e738e7f47", + "image": "vm2api:kubernetes-6589eab3", + "docker_reported_id": "sha256:a2d4d0ffa17e833eac84c466bc13ccc0985872a277d092cf63cf5d95c29a2340", + "containerd_manifest_digest": "sha256:a2d4d0ffa17e833eac84c466bc13ccc0985872a277d092cf63cf5d95c29a2340", + "manifest_media_type": "application/vnd.oci.image.manifest.v1+json", + "manifest_config_digest": "sha256:2bf42f9e9d5fd7c652dca757e62e69c79364dfc5f548338eddcbcd87ef688dce", + "validated_at": "2026-10-21T08:37:05.897950+00:00", + "context": "cube-local", + "storage_class": "local-path", + "smoke_test_commit": "e8e418b9a532959095a6c8aa5ccd70dc1f63a09a", + "exit_code": 0, + "observed_pod_image_id": "sha256:2bf42f9e9d5fd7c652dca757e62e69c79364dfc5f548338eddcbcd87ef688dce", + "namespace": "vm2api-k8s-649-67b4ba49e2", + "namespace_deleted": true, + "checks": [ + "pod-ready", + "service-dns-and-clusterip", + "health-ready-console", + "panel-cookie-login-and-postgres-metrics", + "api-key-create", + "managed-api-key-inference", + "go-api-kernel-json-and-sse", + "two-request-logs", + "second-control-plane-rejected", + "recreate-restart", + "postgres-session-and-api-key-persist", + "pvc-persists" + ], + "sse_timing": { + "event": "sse-timing", + "headers_ms": 12.2, + "headers": { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache, no-transform", + "transfer-encoding": "chunked", + "x-accel-buffering": "no" + }, + "frames": [ + { + "type": "message_start", + "arrived_ms": 12.2 + }, + { + "type": "content_block_start", + "arrived_ms": 12.2 + }, + { + "type": "content_block_delta", + "arrived_ms": 12.2 + }, + { + "type": "content_block_delta", + "arrived_ms": 1012.1 + }, + { + "type": "content_block_stop", + "arrived_ms": 1012.2 + }, + { + "type": "message_delta", + "arrived_ms": 1012.2 + }, + { + "type": "message_stop", + "arrived_ms": 1012.2 + } + ], + "deltas": [ + { + "text": "kubernetes-pro", + "arrived_ms": 12.2 + }, + { + "text": "of-67b4ba49e296", + "arrived_ms": 1012.1 + } + ], + "ended_ms": 1019.7 + }, + "before_fix": { + "source_commit": "1366e203b4b771542702517bc4e1bf0279e6ee8f", + "exit_code": 1, + "failure": "SSE text deltas were buffered until completion", + "first_text_delta_ms": 1012.9, + "second_text_delta_ms": 1013.0, + "delta_gap_ms": 0.1 + }, + "after_fix_delta_gap_ms": 999.9, + "runtime_log": "smoke-runtime-6589eab3.log", + "limitations": [ + "single active control plane only", + "Recreate restart is not a zero-downtime rollout", + "HTTP and SSE verified through Service port-forward; external Ingress not deployed", + "synthetic internal upstream only; no external model service traffic", + "no RWX or multi-node storage validation" + ] +} diff --git a/docs/plans/evidence/kubernetes/runtime-before.json b/docs/plans/evidence/kubernetes/runtime-before.json new file mode 100644 index 000000000..1aa6aa2ab --- /dev/null +++ b/docs/plans/evidence/kubernetes/runtime-before.json @@ -0,0 +1,35 @@ +{ + "source_commit": "1366e203b4b771542702517bc4e1bf0279e6ee8f", + "smoke_test_commit": "e8e418b9", + "image": "vm2api:kubernetes-check", + "docker_reported_id": "sha256:db547ceaccc6803bbd1d14ec3dcd1fa5d3ca5dbe7779b65c0b67f93c01ea0db7", + "containerd_manifest_digest": "sha256:db547ceaccc6803bbd1d14ec3dcd1fa5d3ca5dbe7779b65c0b67f93c01ea0db7", + "manifest_config_digest": "sha256:9669bbe537df06ba2b76efdadbd085a5e6d37ece17cdec900e58da7dfbfb01de", + "observed_pod_image_id": "sha256:9669bbe537df06ba2b76efdadbd085a5e6d37ece17cdec900e58da7dfbfb01de", + "namespace": "vm2api-k8s-649-ef1854c411", + "namespace_deleted": true, + "exit_code": 1, + "failure": "SSE text deltas were buffered until completion", + "first_text_delta_ms": 1012.9, + "second_text_delta_ms": 1013.0, + "headers_ms": 1012.8, + "eof_ms": 1017.8, + "logs": [ + "smoke-runtime.log", + "smoke-runtime-second.log" + ], + "passed_before_failure": [ + "pod readiness", + "Service DNS and ClusterIP", + "healthz/readyz/console", + "panel cookie login", + "PostgreSQL metrics", + "managed API key create", + "nonstream managed API inference proof" + ], + "not_reached": [ + "duplicate controller rejection", + "Pod recreate restart", + "session/API key/PVC persistence" + ] +} diff --git a/docs/plans/evidence/kubernetes/verification.json b/docs/plans/evidence/kubernetes/verification.json new file mode 100644 index 000000000..c5728ad6d --- /dev/null +++ b/docs/plans/evidence/kubernetes/verification.json @@ -0,0 +1,59 @@ +{ + "source_commit": "6589eab3ca1179250a24c6ee5a5fee5e738e7f47", + "dependency_commit": "cf9462d5501c323efa95c254feb3821f7cf656f0", + "scope": "single active control plane; horizontal multi-active execution remains unimplemented", + "checks": { + "directed": { + "verified_commit": "1366e203", + "tests": 119, + "passed": 119, + "failed": 0, + "skipped": 0, + "cancelled": 0, + "log_sha256": "36e32464a9cb25fa99b50e8342172be7673e402a58cf6308f72bcd421b27c416" + }, + "database": { + "verified_commit": "6589eab3", + "tests": 44, + "passed": 44, + "failed": 0, + "skipped": 0, + "cancelled": 0, + "log_sha256": "f1b9c61f821a2b5c57f88863ae9f3b6a9691d3b23ff8d97eb2b173e5db9d0f3a" + }, + "unit": { + "verified_commit": "1366e203", + "tests": 3367, + "passed": 3350, + "failed": 7, + "skipped": 10, + "cancelled": 0, + "log_sha256": "35b1fc56c1c5f0d1349306c7436d6f3b1811438ce286cbdaa0c5bbc495e40df3" + }, + "e2e": { + "verified_commit": "6589eab3", + "tests": 119, + "passed": 101, + "failed": 17, + "skipped": 1, + "cancelled": 0, + "log_sha256": "8191fee5a620d736f7d084706823d8d118664ce787db6b75329457ccfc8a8a78" + }, + "deployment": { + "verified_commit": "6589eab3", + "tests": 33, + "passed": 33, + "failed": 0, + "skipped": 0, + "cancelled": 0, + "log_sha256": "4d21d1d3d6646a3e364d36c31ad851e4d0490ab7b6237703edebda446ced9a68" + } + }, + "go_api_kernel": { + "verified_commit": "11ee7dc1", + "command": "go test -race ./...", + "exit_code": 0, + "log_sha256": "570290cde41a737dd632b388f8ab6ab9b97344b1e0031d3d419016bbcfcfbd17" + }, + "runtime_evidence": "runtime-after.json" +} From 4f636cd802aa04262b511f392641d3685ef3789f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:42:19 +0800 Subject: [PATCH 051/473] test(settings): reproduce stale cross-replica settings snapshots --- test/unit/settings-state.test.mjs | 119 +++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/test/unit/settings-state.test.mjs b/test/unit/settings-state.test.mjs index f845911ca..a60e05443 100644 --- a/test/unit/settings-state.test.mjs +++ b/test/unit/settings-state.test.mjs @@ -1,7 +1,122 @@ import test from 'node:test' import assert from 'node:assert/strict' -import { getDb, withTransaction } from '../../src/lib/db/database.mjs' -import { RuntimeSettings } from '../../src/lib/db/settings-state.mjs' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { loadSettingsState, RuntimeSettings } from '../../src/lib/db/settings-state.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function otherConnection(t, db) { + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + await loadSettingsState(other) + return new RuntimeSettings(other) +} + +/** Hold one completed PG read so competing commits have deterministic order. */ +function holdSnapshot(t, db) { + const read = deferred() + const resume = deferred() + const query = db.query + let held = false + db.query = async function (sql, params) { + const result = await query.call(this, sql, params) + if (!held && sql === 'SELECT key, value, updated_at FROM settings ORDER BY key') { + held = true + read.resolve() + await resume.promise + } + return result + } + t.after(() => { + resume.resolve() + db.query = query + }) + return { read: read.promise, resume: resume.resolve } +} + +test('refresh makes another PostgreSQL connection set and remove visible to synchronous readers', async (t) => { + const db = getDb() + const settings = new RuntimeSettings(db) + const reader = new RuntimeSettings(db) + const other = await otherConnection(t, db) + await settings.set('runtime_shared', { version: 1 }) + await other.set('runtime_shared', { version: 2 }) + await other.set('runtime_added', true) + assert.deepEqual(reader.get('runtime_shared'), { version: 1 }) + await settings.refresh() + assert.deepEqual(reader.get('runtime_shared'), { version: 2 }) + assert.equal(reader.has('runtime_added'), true) + assert.deepEqual(reader.all().runtime_shared, { version: 2 }) + await other.remove('runtime_shared') + await settings.refresh() + assert.equal(reader.has('runtime_shared'), false) + assert.equal(reader.get('runtime_shared', 'missing'), 'missing') +}) + +test('an older settings snapshot cannot overwrite a newer completed refresh', async (t) => { + const db = getDb() + const settings = new RuntimeSettings(db) + const other = await otherConnection(t, db) + await settings.set('runtime_refresh_order', 1) + const held = holdSnapshot(t, db) + const oldRefresh = loadSettingsState(db) + try { + await held.read + await other.set('runtime_refresh_order', 2) + await loadSettingsState(db) + assert.equal(settings.get('runtime_refresh_order'), 2) + } finally { + held.resume() + await oldRefresh + } + assert.equal(settings.get('runtime_refresh_order'), 2) +}) + +test('an in-flight snapshot cannot overwrite a newer local committed set or remove', async (t) => { + const db = getDb() + const settings = new RuntimeSettings(db) + await settings.set('runtime_local_set', 1) + await settings.set('runtime_local_remove', true) + const held = holdSnapshot(t, db) + const oldRefresh = loadSettingsState(db) + try { + await held.read + // Direct repository callers must advance the same cache revision. + await withTransaction(db, async () => { + await new SettingsRepo(db).set('runtime_local_set', 2) + await new SettingsRepo(db).remove('runtime_local_remove') + }) + } finally { + held.resume() + await oldRefresh + } + assert.equal(settings.get('runtime_local_set'), 2) + assert.equal(settings.has('runtime_local_remove'), false) +}) + +test('a failed refresh preserves the previously published settings snapshot', async () => { + const db = getDb() + const settings = new RuntimeSettings(db) + await settings.set('runtime_refresh_failure', true) + const query = db.query + db.query = async function (sql, params) { + if (sql === 'SELECT key, value, updated_at FROM settings ORDER BY key') throw new Error('synthetic read failure') + return query.call(this, sql, params) + } + try { + await assert.rejects(settings.refresh(), /synthetic read failure/) + assert.equal(settings.get('runtime_refresh_failure'), true) + } finally { + db.query = query + } +}) test('settings cache changes only when its transaction commits', async () => { const db = getDb() From e43b3de72cea1cea2c1ab55cc1c5f8b85b66d438 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:44:10 +0800 Subject: [PATCH 052/473] test(k8s): reproduce multiplied gateway rate limits across controllers --- test/unit/shared-rate-limiter.test.mjs | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test/unit/shared-rate-limiter.test.mjs diff --git a/test/unit/shared-rate-limiter.test.mjs b/test/unit/shared-rate-limiter.test.mjs new file mode 100644 index 000000000..105b175f1 --- /dev/null +++ b/test/unit/shared-rate-limiter.test.mjs @@ -0,0 +1,44 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { createRateLimiter, hashKey } from '../../src/lib/core/security.mjs' + +test('independent control planes share one atomic gateway token bucket', async () => { + const db = getDb() + const other = await createDatabase({ schema: db.schema }) + try { + const first = createRateLimiter({ db, capacity: 3, refillPerSec: 0 }) + const second = createRateLimiter({ db: other, capacity: 3, refillPerSec: 0 }) + const admitted = await Promise.all(Array.from({ length: 20 }, (_, i) => (i % 2 ? first : second)('shared-secret'))) + assert.equal(admitted.filter(Boolean).length, 3) + assert.equal(await first('shared-secret'), false) + assert.equal(await second('different-secret'), true) + const rows = await db.prepare('SELECT bucket_hash FROM gateway_rate_buckets').all() + assert.deepEqual(new Set(rows.map((row) => row.bucket_hash)), new Set([hashKey('shared-secret'), hashKey('different-secret')])) + } finally { + await other.close() + } +}) + +test('bucket refill uses the database clock and survives a control-plane connection replacement', async () => { + const db = getDb() + const first = createRateLimiter({ db, capacity: 1, refillPerSec: 1 }) + assert.equal(await first('refill'), true) + await db.prepare('UPDATE gateway_rate_buckets SET updated_at = updated_at - 2000 WHERE bucket_hash = $1').run(hashKey('refill')) + const other = await createDatabase({ schema: db.schema }) + try { + const replacement = createRateLimiter({ db: other, capacity: 1, refillPerSec: 1 }) + assert.equal(await replacement('refill'), true) + assert.equal(await first('refill'), false) + } finally { + await other.close() + } +}) + +test('a shared limiter fails closed when PostgreSQL is unavailable', async () => { + const db = getDb() + const other = await createDatabase({ schema: db.schema }) + const limiter = createRateLimiter({ db: other, capacity: 1 }) + await other.close() + await assert.rejects(async () => await limiter('offline'), /closed/) +}) From 9c9e3bf4fa22861106ef2aa411cd364f15ba9d85 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:44:18 +0800 Subject: [PATCH 053/473] test(api): reproduce stale endpoint selection and shortened shared cooldowns --- test/unit/api-endpoints-shared.test.mjs | 119 ++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 test/unit/api-endpoints-shared.test.mjs diff --git a/test/unit/api-endpoints-shared.test.mjs b/test/unit/api-endpoints-shared.test.mjs new file mode 100644 index 000000000..4b21462b8 --- /dev/null +++ b/test/unit/api-endpoints-shared.test.mjs @@ -0,0 +1,119 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { ApiEndpointStore } from '../../src/lib/admin/api-endpoints.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { runApiInference } from '../../src/lib/pool/api-protocol.mjs' +import { ApiScheduler } from '../../src/lib/pool/api-scheduler.mjs' + +async function replicas() { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + return { writer: new ApiEndpointStore({ db }), reader: new ApiEndpointStore({ db: other }), other } +} + +async function endpoint(store) { + return store.create({ + name: 'shared-test', + base_url: 'https://synthetic.invalid', + api_key_entries: [{ api_key: 'synthetic-upstream-key' }], + models: [{ name: 'upstream-model', alias: 'public-model' }], + }) +} + +// Observe the real scheduler's selection without forwarding to an upstream. +async function selection(store, scheduler, model = 'public-model') { + let picked + await runApiInference({ + store, + scheduler: { + reload: (rows) => scheduler.reload(rows), + pick(requested) { + picked = scheduler.pick(requested) + return { ok: false, code: 'selection_observed', message: 'synthetic selection' } + }, + }, + convertedBody: { model, messages: [{ role: 'user', content: 'synthetic request' }] }, + }) + return picked +} + +test('API inference sees endpoint, key, model, and cooldown changes from another connection before selection', async () => { + const { writer, reader, other } = await replicas() + try { + const rec = await endpoint(writer) + const keyId = rec.api_key_entries[0].id + const scheduler = new ApiScheduler() + scheduler.reload(await reader.listRaw()) + + await writer.update(rec.id, { disabled: true }) + assert.equal((await selection(reader, scheduler)).code, 'model_not_found', 'disabled endpoint stays unavailable') + + await writer.update(rec.id, { disabled: false }) + await writer.updateKey(keyId, { disabled: true }) + assert.equal((await selection(reader, scheduler)).code, 'api_pool_exhausted', 'disabled key stays unavailable') + + await writer.updateKey(keyId, { disabled: false }) + await writer.setKeyCooldown(keyId, new Date(Date.now() + 60_000).toISOString()) + assert.equal((await selection(reader, scheduler)).code, 'api_pool_exhausted', 'another replica cooldown applies') + + await writer.updateKey(keyId, { cooldown_until: null }) + await writer.update(rec.id, { models: [{ name: 'replacement-upstream', alias: 'replacement-model' }] }) + assert.equal((await selection(reader, scheduler)).code, 'model_not_found', 'removed model alias stays unavailable') + assert.equal((await selection(reader, scheduler, 'replacement-model')).upstream_model, 'replacement-upstream') + + await writer.remove(rec.id) + assert.equal((await selection(reader, scheduler, 'replacement-model')).code, 'model_not_found') + } finally { + await other.close() + } +}) + +test('API inference rejects a shared endpoint read failure before selecting a cached endpoint', async () => { + let selected = false + const error = new Error('synthetic shared endpoint read failed') + await assert.rejects( + runApiInference({ + convertedBody: { model: 'public-model', messages: [] }, + store: { + listRaw: async () => { + throw error + }, + }, + scheduler: { + reload() {}, + pick() { + selected = true + return { ok: false, code: 'cached_selection', message: 'cached selection' } + }, + }, + }), + error, + ) + assert.equal(selected, false) +}) + +test('automatic cooldown updates only extend across replicas while explicit administrator reset still clears', async () => { + const { writer, reader, other } = await replicas() + try { + const rec = await endpoint(writer) + const keyId = rec.api_key_entries[0].id + const start = Date.now() + const longer = new Date(start + 300_000).toISOString() + await writer.setKeyCooldown(keyId, longer) + await reader.setKeyCooldown(keyId, new Date(start + 30_000).toISOString()) + assert.equal((await reader.getRaw(rec.id)).api_key_entries[0].cooldown_until, longer) + + const longest = new Date(start + 900_000).toISOString() + await Promise.all([ + writer.setKeyCooldown(keyId, longest), + reader.setKeyCooldown(keyId, new Date(start + 60_000).toISOString()), + writer.setKeyCooldown(keyId, new Date(start + 600_000).toISOString()), + ]) + assert.equal((await reader.getRaw(rec.id)).api_key_entries[0].cooldown_until, longest) + + await writer.updateKey(keyId, { cooldown_until: null }) + assert.equal((await reader.getRaw(rec.id)).api_key_entries[0].cooldown_until, null) + } finally { + await other.close() + } +}) From 31bc7d21ad27d002acf74ddb255c4ded014e7ae4 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:45:01 +0800 Subject: [PATCH 054/473] fix(api): refresh shared endpoints and extend concurrent cooldowns atomically --- src/lib/admin/api-endpoints.mjs | 1 + src/lib/db/repos/api-endpoints-repo.mjs | 7 ++++++- src/lib/pool/api-protocol.mjs | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/admin/api-endpoints.mjs b/src/lib/admin/api-endpoints.mjs index efc257f3b..95c6a8a03 100644 --- a/src/lib/admin/api-endpoints.mjs +++ b/src/lib/admin/api-endpoints.mjs @@ -228,6 +228,7 @@ export class ApiEndpointStore { } async setKeyCooldown(id, untilIso) { + // Automatic upstream responses may extend cooldown; updateKey handles explicit administrator resets. await this.repo.setKeyCooldown(id, untilIso) } diff --git a/src/lib/db/repos/api-endpoints-repo.mjs b/src/lib/db/repos/api-endpoints-repo.mjs index 7deea21c9..1f65b49fa 100644 --- a/src/lib/db/repos/api-endpoints-repo.mjs +++ b/src/lib/db/repos/api-endpoints-repo.mjs @@ -99,7 +99,12 @@ export class ApiEndpointsRepo { `INSERT INTO api_endpoint_models (${MODEL_COLS.join(',')}) VALUES (${MODEL_COLS.map((_, i) => '$' + (i + 1)).join(',')})`, ) this._delModelsEp = db.prepare('DELETE FROM api_endpoint_models WHERE endpoint_id = $1') - this._cool = db.prepare('UPDATE api_endpoint_keys SET cooldown_until = $1 WHERE id = $2') + this._cool = db.prepare(` + UPDATE api_endpoint_keys SET cooldown_until = CASE + WHEN cooldown_until IS NULL OR cooldown_until::timestamptz < $1::text::timestamptz THEN $1::text + ELSE cooldown_until + END WHERE id = $2 + `) this._updKey = db.prepare( 'UPDATE api_endpoint_keys SET api_key = $1, proxy_url = $2, proxy_id = $3, disabled = $4, cooldown_until = $5 WHERE id = $6', ) diff --git a/src/lib/pool/api-protocol.mjs b/src/lib/pool/api-protocol.mjs index 047a7235c..f4547d7c0 100644 --- a/src/lib/pool/api-protocol.mjs +++ b/src/lib/pool/api-protocol.mjs @@ -71,6 +71,7 @@ export async function runApiInference({ } = {}) { const canonical = officialMessagesBody(convertedBody) const model = canonical.model + scheduler.reload(await store.listRaw()) const picked = scheduler.pick(model) if (!picked.ok) { return { From bec9e39881a5849725e848a062ae68591b253abc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:45:04 +0800 Subject: [PATCH 055/473] fix(settings): refresh shared snapshots without reverting newer commits --- src/lib/db/repos/settings-repo.mjs | 30 +++++++++++++++++++--- src/lib/db/settings-state.mjs | 9 +++++-- test/unit/settings-state.test.mjs | 40 ++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/lib/db/repos/settings-repo.mjs b/src/lib/db/repos/settings-repo.mjs index 674a0776c..e5fba389b 100644 --- a/src/lib/db/repos/settings-repo.mjs +++ b/src/lib/db/repos/settings-repo.mjs @@ -5,6 +5,17 @@ import { getDb } from '../database.mjs' +const cacheRevisions = new WeakMap() + +function cacheRevision(db) { + let revision = cacheRevisions.get(db) + if (!revision) { + revision = { refresh: 0, published: 0, write: 0 } + cacheRevisions.set(db, revision) + } + return revision +} + export class SettingsRepo { constructor(db = getDb()) { this.db = db @@ -29,7 +40,10 @@ export class SettingsRepo { async set(key, value) { await this._set.run(key, JSON.stringify(value), new Date().toISOString()) - this.db.afterCommit(() => this.db.settingsState?.set(key, value)) + this.db.afterCommit(() => { + cacheRevision(this.db).write += 1 + this.db.settingsState?.set(key, value) + }) return value } @@ -39,10 +53,16 @@ export class SettingsRepo { async remove(key) { await this._del.run(key) - this.db.afterCommit(() => this.db.settingsState?.delete(key)) + this.db.afterCommit(() => { + cacheRevision(this.db).write += 1 + this.db.settingsState?.delete(key) + }) } async all() { + const revision = cacheRevision(this.db) + const refresh = ++revision.refresh + const write = revision.write const out = {} for (const row of await this._all.all()) { try { @@ -52,7 +72,11 @@ export class SettingsRepo { } } this.db.afterCommit(() => { - this.db.settingsState = new Map(Object.entries(out)) + // A delayed read must not replace a newer refresh or local commit. + if (revision.published < refresh && revision.write === write) { + revision.published = refresh + this.db.settingsState = new Map(Object.entries(out)) + } }) return out } diff --git a/src/lib/db/settings-state.mjs b/src/lib/db/settings-state.mjs index da4771201..3ed689868 100644 --- a/src/lib/db/settings-state.mjs +++ b/src/lib/db/settings-state.mjs @@ -3,8 +3,7 @@ import { getDb } from './database.mjs' import { SettingsRepo } from './repos/settings-repo.mjs' export async function loadSettingsState(db = getDb()) { - const settings = await new SettingsRepo(db).all() - db.settingsState = new Map(Object.entries(settings)) + await new SettingsRepo(db).all() return db.settingsState } @@ -14,6 +13,12 @@ export class RuntimeSettings { this.repo = new SettingsRepo(db) } + /** Refresh the shared snapshot before calling synchronous policy readers. */ + async refresh() { + await this.repo.all() + return this.all() + } + get(key, fallback = null) { if (!this.db.settingsState) throw new Error('settings not loaded; await loadSettingsState() before startup') return this.db.settingsState.has(key) ? structuredClone(this.db.settingsState.get(key)) : fallback diff --git a/test/unit/settings-state.test.mjs b/test/unit/settings-state.test.mjs index a60e05443..3a64adcd2 100644 --- a/test/unit/settings-state.test.mjs +++ b/test/unit/settings-state.test.mjs @@ -101,6 +101,29 @@ test('an in-flight snapshot cannot overwrite a newer local committed set or remo assert.equal(settings.has('runtime_local_remove'), false) }) +test('a newer failed refresh does not discard an older successful snapshot', async (t) => { + const db = getDb() + const settings = new RuntimeSettings(db) + const other = await otherConnection(t, db) + await settings.set('runtime_refresh_retry', 0) + await other.set('runtime_refresh_retry', 1) + const held = holdSnapshot(t, db) + const oldRefresh = loadSettingsState(db) + try { + await held.read + const query = db.query + db.query = async function (sql, params) { + if (sql === 'SELECT key, value, updated_at FROM settings ORDER BY key') throw new Error('synthetic read failure') + return query.call(this, sql, params) + } + await assert.rejects(settings.refresh(), /synthetic read failure/) + } finally { + held.resume() + await oldRefresh + } + assert.equal(settings.get('runtime_refresh_retry'), 1) +}) + test('a failed refresh preserves the previously published settings snapshot', async () => { const db = getDb() const settings = new RuntimeSettings(db) @@ -118,6 +141,23 @@ test('a failed refresh preserves the previously published settings snapshot', as } }) +test('refresh inside a transaction does not publish uncommitted settings or a rolled-back snapshot', async () => { + const settings = new RuntimeSettings(getDb()) + await settings.set('runtime_refresh_rollback', 1) + await assert.rejects( + withTransaction(getDb(), async () => { + await settings.set('runtime_refresh_rollback', 2) + await settings.refresh() + assert.equal(settings.get('runtime_refresh_rollback'), 1) + throw new Error('synthetic rollback') + }), + /synthetic rollback/, + ) + assert.equal(settings.get('runtime_refresh_rollback'), 1) + await settings.refresh() + assert.equal(settings.get('runtime_refresh_rollback'), 1) +}) + test('settings cache changes only when its transaction commits', async () => { const db = getDb() const settings = new RuntimeSettings(db) From 1509eab580fd603b0b015bf565dd85f4e2a15c7d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:46:44 +0800 Subject: [PATCH 056/473] test(k8s): require managed key reservations on Codex routes --- test/unit/protocol-admission.test.mjs | 87 +++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 test/unit/protocol-admission.test.mjs diff --git a/test/unit/protocol-admission.test.mjs b/test/unit/protocol-admission.test.mjs new file mode 100644 index 000000000..452e56e87 --- /dev/null +++ b/test/unit/protocol-admission.test.mjs @@ -0,0 +1,87 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Readable } from 'node:stream' +import { EventEmitter } from 'node:events' +import { createHandleProtocol } from '../../src/lib/protocol/handle-protocol.mjs' +import { readBody } from '../../src/lib/http/respond.mjs' + +async function call({ search = false, denied = true }) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-protocol-admission-')) + const events = [] + const key = { id: 'managed-key', category: 'oauth', group_type: 'all', allowed_vms: [] } + const handler = createHandleProtocol({ + cfg: { + limits: { max_body_bytes: 1 << 20 }, + rewrite: { enabled: false }, + intercept: { rules: [] }, + distill: { enabled: false }, + paths: { project: root, data: root }, + }, + json(res, status, body) { + res.statusCode = status + res.body = body + }, + readBody, + requireAuth: (req) => { + req.apiKeyRecord = key + req.apiKeyKind = 'managed' + return true + }, + requestLog: { start: () => ({ request_id: 'admission', t0: Date.now() }) }, + stats: { errors: 0, requests: 0, by_route: {} }, + groupsRepo: { rateMultiplier: () => 1 }, + apiKeyStore: { + async reserve() { + events.push('reserve') + return denied + ? { ok: false, status: 429, code: 'api_key_concurrency_limit', message: 'shared capacity exhausted' } + : { + ok: true, + release: async () => { + events.push('release') + }, + } + }, + }, + stickyRouter: { resolve: () => null }, + routingConfig: { + codex: { + clients: { openai_compatible: 'allow', unknown: 'allow' }, + protocols: { 'openai.responses': { enabled: true, mode: 'native' } }, + }, + }, + }) + const body = search + ? { model: 'gpt-5.4', commands: { search_query: [{ q: 'x' }] } } + : { model: 'gpt-5.4', input: 'x' } + const req = Readable.from([Buffer.from(JSON.stringify(body))]) + req.headers = { 'user-agent': 'openai-node/1.0' } + const res = new EventEmitter() + res.setHeader = () => {} + try { + if (search) await handler.handleSearch(req, res, '/v1/alpha/search') + else await handler.handleProtocol(req, res, 'openai.responses', '/v1/responses') + return { res, events } + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +} + +for (const search of [false, true]) { + const route = search ? 'Codex search' : 'Codex responses' + test(`${route} respects managed-key admission before entering the execution backend`, async () => { + const { res, events } = await call({ search }) + assert.equal(res.statusCode, 429) + assert.equal(res.body.error.code, 'api_key_concurrency_limit') + assert.deepEqual(events, ['reserve']) + }) + + test(`${route} awaits reservation release after a failed selection`, async () => { + const { res, events } = await call({ search, denied: false }) + assert.ok(res.statusCode >= 400) + assert.deepEqual(events, ['reserve', 'release']) + }) +} From 03777116a3e1ddaadbb966fcd04a505f16181ceb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:48:30 +0800 Subject: [PATCH 057/473] test: reproduce missing cross-replica API key admission --- test/unit/api-key-admission.test.mjs | 167 +++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 test/unit/api-key-admission.test.mjs diff --git a/test/unit/api-key-admission.test.mjs b/test/unit/api-key-admission.test.mjs new file mode 100644 index 000000000..445b8ac9a --- /dev/null +++ b/test/unit/api-key-admission.test.mjs @@ -0,0 +1,167 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' + +async function replicas() { + const db = getDb() + const other = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: db.schema }) + return [new ApiKeyStore({ db, sharedAdmission: true }), new ApiKeyStore({ db: other, sharedAdmission: true })] +} + +async function user(store, { role = 'user', concurrency = 1 } = {}) { + const id = 'user-' + crypto.randomUUID() + const now = new Date().toISOString() + return store.users.insert({ id, username: id, password_hash: 'test', role, concurrency, created_at: now, updated_at: now }) +} + +test('independent replicas atomically enforce key concurrency and release once', async () => { + const stores = await replicas() + const key = await stores[0].create({ max_concurrency: 1 }) + const gates = await Promise.all(Array.from({ length: 12 }, (_, i) => stores[i % 2].reserve(key))) + assert.equal(gates.filter((g) => g.ok).length, 1) + assert.ok(gates.filter((g) => !g.ok).every((g) => g.code === 'api_key_concurrency_limit')) + const held = gates.find((g) => g.ok) + assert.equal(held.shared, true) + assert.equal((await stores[1].list())[0].inflight, 1) + await Promise.all([held.release(), held.release()]) + const next = await stores[1].reserve(key) + assert.equal(next.ok, true) + await held.release() + assert.equal((await stores[0].reserve(key)).code, 'api_key_concurrency_limit') + await next.release() +}) + +test('different keys share their owner concurrency across replicas', async () => { + const stores = await replicas() + const owner = await user(stores[0]) + const keys = await Promise.all(stores.map((s) => s.create({ max_concurrency: 0, user_id: owner.id }))) + const gates = await Promise.all(stores.map((s, i) => s.reserve(keys[i]))) + assert.equal(gates.filter((g) => g.ok).length, 1) + assert.equal(gates.find((g) => !g.ok).code, 'user_concurrency_limit') + await gates.find((g) => g.ok).release() + const accepted = await stores[1].reserve(keys[1]) + assert.equal(accepted.ok, true) + await accepted.release() +}) + +test('released reservations remain in the global RPM window', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0, rpm: 1 }) + const gate = await first.reserve(key) + assert.equal(gate.ok, true) + await gate.release() + const denied = await second.reserve(key) + assert.equal(denied.code, 'api_key_rate_limit') + assert.ok(denied.retry_after_ms > 0 && denied.retry_after_ms <= 60_000) + await first.db.prepare("UPDATE api_key_admissions SET created_at = clock_timestamp() - interval '61 seconds'").run() + const next = await second.reserve(key) + assert.equal(next.ok, true) + await next.release() +}) + +test('request quota includes pending reservations and settles exactly once', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0, quota_requests: 1 }) + const gates = await Promise.all([first.reserve(key), second.reserve(key)]) + assert.equal(gates.filter((g) => g.ok).length, 1) + assert.equal(gates.find((g) => !g.ok).code, 'api_key_quota_exhausted') + const gate = gates.find((g) => g.ok) + await first.recordUsage(key, { input_tokens: 5 }, { reservation: gate }) + assert.equal(gate.settled, true) + await first.recordUsage(key, { input_tokens: 5 }, { reservation: gate }) + const saved = await first.getById(key.id) + assert.equal(saved.requests, 1) + assert.equal(saved.quota_requests_used, 1) + assert.equal(saved.tokens_in, 5) + assert.equal((await second.reserve(key)).code, 'api_key_quota_exhausted') + assert.equal((await first.list())[0].inflight, 1, 'settlement does not release active execution') + await gate.release() + assert.equal((await first.list())[0].inflight, 0) + assert.equal((await second.reserve(key)).code, 'api_key_quota_exhausted') +}) + +test('quota count and pending settlement commit together under the key lock', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0, quota_requests: 2 }) + const gate = await first.reserve(key) + let entered + const waiting = new Promise((resolve) => { entered = resolve }) + let resume + const proceed = new Promise((resolve) => { resume = resolve }) + const record = first.repo.recordUsage.bind(first.repo) + first.repo.recordUsage = async (...args) => { + const saved = await record(...args) + entered() + await proceed + return saved + } + const settlement = first.recordUsage(key, {}, { reservation: gate }) + await waiting + const observed = await second.getById(key.id) + assert.equal(observed.quota_requests_used, 0, 'an uncommitted completion is invisible') + const otherPending = await second.db.prepare('SELECT count(*) AS count FROM api_key_admissions WHERE settled_at IS NULL AND released_at IS NULL').get() + assert.equal(otherPending.count, 1, 'the pending reservation remains visible until completion commits') + const competing = second.reserve(key) + resume() + await settlement + const next = await competing + assert.equal(next.ok, true, 'one completion and one new pending request fit a quota of two') + assert.equal((await second.reserve(key)).code, 'api_key_quota_exhausted') + await gate.release() + await next.release() +}) + +test('unbilled failures release pending quota but cannot be settled later', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0, quota_requests: 1 }) + const gate = await first.reserve(key) + await gate.release() + await assert.rejects(first.recordUsage(key, {}, { reservation: gate }), { code: 'api_key_reservation_released' }) + assert.equal((await first.getById(key.id)).requests, 0) + const next = await second.reserve(key) + assert.equal(next.ok, true) + await next.release() +}) + +test('reservations re-read disabled, expired, rotated keys and missing or disabled owners', async () => { + const [first, second] = await replicas() + const owner = await user(first) + const key = await first.create({ user_id: owner.id }) + await first.update(key.id, { status: 'disabled' }) + assert.equal((await second.reserve(key)).code, 'api_key_disabled') + await first.update(key.id, { status: 'active', expires_at: new Date(Date.now() - 1000).toISOString() }) + assert.equal((await second.reserve(key)).code, 'api_key_expired') + await first.update(key.id, { expires_at: null }) + await first.rotate(key.id) + assert.equal((await second.reserve(key)).code, 'invalid_api_key') + const current = await first.getById(key.id) + await first.users.update({ ...owner, enabled: false }) + assert.equal((await second.reserve(current)).code, 'user_disabled') + await first.users.remove(owner.id) + assert.equal((await second.reserve(current)).code, 'user_disabled') +}) + +test('admin owner remains uncapped and current key configuration accompanies a reservation', async () => { + const [first, second] = await replicas() + const owner = await user(first, { role: 'admin' }) + const key = await first.create({ user_id: owner.id, max_concurrency: 0 }) + await first.update(key.id, { category: 'api' }) + const gates = await Promise.all([first.reserve(key), second.reserve(key)]) + assert.ok(gates.every((g) => g.ok)) + assert.ok(gates.every((g) => g.record.category === 'api')) + await Promise.all(gates.map((g) => g.release())) +}) + +test('old executing reservations do not expire or disappear when their connection closes', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 1 }) + const gate = await second.reserve(key) + await first.db.prepare("UPDATE api_key_admissions SET created_at = clock_timestamp() - interval '2 hours'").run() + await second.db.close() + assert.equal((await first.reserve(key)).code, 'api_key_concurrency_limit') + const row = await first.db.prepare('SELECT owner_id, released_at FROM api_key_admissions WHERE id = $1').get(gate.reservationId) + assert.equal(row.owner_id, gate.ownerId) + assert.equal(row.released_at, null) +}) From be5e890d41d73a56b60b3aa3bf85d1b808c784a6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:50:25 +0800 Subject: [PATCH 058/473] test(k8s): cover shared bucket clock and numeric boundaries --- test/unit/shared-rate-limiter.test.mjs | 44 ++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/test/unit/shared-rate-limiter.test.mjs b/test/unit/shared-rate-limiter.test.mjs index 105b175f1..e0c01067f 100644 --- a/test/unit/shared-rate-limiter.test.mjs +++ b/test/unit/shared-rate-limiter.test.mjs @@ -14,7 +14,10 @@ test('independent control planes share one atomic gateway token bucket', async ( assert.equal(await first('shared-secret'), false) assert.equal(await second('different-secret'), true) const rows = await db.prepare('SELECT bucket_hash FROM gateway_rate_buckets').all() - assert.deepEqual(new Set(rows.map((row) => row.bucket_hash)), new Set([hashKey('shared-secret'), hashKey('different-secret')])) + assert.deepEqual( + new Set(rows.map((row) => row.bucket_hash)), + new Set([hashKey('shared-secret'), hashKey('different-secret')]), + ) } finally { await other.close() } @@ -24,7 +27,9 @@ test('bucket refill uses the database clock and survives a control-plane connect const db = getDb() const first = createRateLimiter({ db, capacity: 1, refillPerSec: 1 }) assert.equal(await first('refill'), true) - await db.prepare('UPDATE gateway_rate_buckets SET updated_at = updated_at - 2000 WHERE bucket_hash = $1').run(hashKey('refill')) + await db + .prepare('UPDATE gateway_rate_buckets SET updated_at = updated_at - 2000 WHERE bucket_hash = $1') + .run(hashKey('refill')) const other = await createDatabase({ schema: db.schema }) try { const replacement = createRateLimiter({ db: other, capacity: 1, refillPerSec: 1 }) @@ -42,3 +47,38 @@ test('a shared limiter fails closed when PostgreSQL is unavailable', async () => await other.close() await assert.rejects(async () => await limiter('offline'), /closed/) }) + +test('a database clock adjustment cannot refill an already future-dated bucket twice', async () => { + const db = getDb() + const limiter = createRateLimiter({ db, capacity: 1, refillPerSec: 1 }) + assert.equal(await limiter('clock-adjustment'), true) + const { future } = await db + .prepare(` + UPDATE gateway_rate_buckets SET updated_at = updated_at + 60_000 + WHERE bucket_hash = $1 RETURNING updated_at AS future + `) + .get(hashKey('clock-adjustment')) + assert.equal(await limiter('clock-adjustment'), false) + assert.equal( + ( + await db + .prepare('SELECT updated_at FROM gateway_rate_buckets WHERE bucket_hash = $1') + .get(hashKey('clock-adjustment')) + ).updated_at, + future, + ) +}) + +test('shared buckets support zero capacity and reject invalid rate settings before taking a connection', async () => { + const db = getDb() + assert.equal(await createRateLimiter({ db, capacity: 0, refillPerSec: 0 })('zero'), false) + for (const options of [ + { capacity: Infinity }, + { capacity: NaN }, + { capacity: -1 }, + { refillPerSec: -1 }, + { refillPerSec: Infinity }, + ]) { + assert.throws(() => createRateLimiter({ db, ...options }), /finite non-negative/) + } +}) From c8c22d664be08adefa4f9b5a9f44a10ca944fbd6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:51:01 +0800 Subject: [PATCH 059/473] test: reject key policy changes after authentication --- test/unit/api-key-admission.test.mjs | 40 ++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/test/unit/api-key-admission.test.mjs b/test/unit/api-key-admission.test.mjs index 445b8ac9a..c4cb3d894 100644 --- a/test/unit/api-key-admission.test.mjs +++ b/test/unit/api-key-admission.test.mjs @@ -147,13 +147,49 @@ test('admin owner remains uncapped and current key configuration accompanies a r const [first, second] = await replicas() const owner = await user(first, { role: 'admin' }) const key = await first.create({ user_id: owner.id, max_concurrency: 0 }) - await first.update(key.id, { category: 'api' }) + await first.update(key.id, { rpm: 20 }) const gates = await Promise.all([first.reserve(key), second.reserve(key)]) assert.ok(gates.every((g) => g.ok)) - assert.ok(gates.every((g) => g.record.category === 'api')) + assert.ok(gates.every((g) => g.record.rpm === 20)) await Promise.all(gates.map((g) => g.release())) }) +test('authorization policy changes after authentication require a fresh request', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0 }) + for (const [column, value] of [ + ['category', 'api'], + ['group_type', 'platform'], + ['allowed_vms', '["vm-other"]'], + ['vm_pool_id', 'pool-other'], + ['user_id', 'owner-other'], + ['group_id', 2], + ]) { + const stale = await first.getById(key.id) + await first.db.prepare(`UPDATE api_keys SET ${column} = $1 WHERE id = $2`).run(value, key.id) + assert.equal((await second.reserve(stale)).code, 'api_key_changed', column) + } +}) + +test('a failed usage transaction retains pending quota and can be safely retried', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0, quota_requests: 1 }) + const gate = await first.reserve(key) + const record = first.repo.recordUsage.bind(first.repo) + first.repo.recordUsage = async (...args) => { + await record(...args) + throw new Error('settlement failed before commit') + } + await assert.rejects(first.recordUsage(key, {}, { reservation: gate }), /settlement failed before commit/) + assert.equal(gate.settled, false) + assert.equal((await first.getById(key.id)).quota_requests_used, 0) + assert.equal((await second.reserve(key)).code, 'api_key_quota_exhausted') + first.repo.recordUsage = record + await first.recordUsage(key, {}, { reservation: gate }) + await gate.release() + assert.equal((await first.getById(key.id)).quota_requests_used, 1) +}) + test('old executing reservations do not expire or disappear when their connection closes', async () => { const [first, second] = await replicas() const key = await first.create({ max_concurrency: 1 }) From dd00b1983ef00463ec5f42739783b73a970b45fb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:51:18 +0800 Subject: [PATCH 060/473] feat(k8s): coordinate gateway token buckets through PostgreSQL --- src/lib/core/security.mjs | 11 +++++- .../004_gateway_rate_buckets.sql | 5 +++ .../db/repos/gateway-rate-buckets-repo.mjs | 35 +++++++++++++++++++ src/server.mjs | 7 ++-- 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 src/lib/db/postgres-migrations/004_gateway_rate_buckets.sql create mode 100644 src/lib/db/repos/gateway-rate-buckets-repo.mjs diff --git a/src/lib/core/security.mjs b/src/lib/core/security.mjs index af188aab1..791574996 100644 --- a/src/lib/core/security.mjs +++ b/src/lib/core/security.mjs @@ -4,6 +4,7 @@ import path from 'node:path' import { atomicWriteJson } from '../vm/vm-file.mjs' import { isKubernetesMode } from './control-plane-mode.mjs' import { PanelSessionsRepo } from '../db/repos/panel-sessions-repo.mjs' +import { GatewayRateBucketsRepo } from '../db/repos/gateway-rate-buckets-repo.mjs' export function hashKey(key) { return crypto.createHash('sha256').update(String(key)).digest('hex') @@ -37,7 +38,15 @@ export function extractApiKey(req) { } /** Simple token bucket per key */ -export function createRateLimiter({ capacity = 60, refillPerSec = 1 } = {}) { +export function createRateLimiter({ capacity = 60, refillPerSec = 1, db = null } = {}) { + if (db || isKubernetesMode()) { + if (!Number.isFinite(capacity) || capacity < 0 || !Number.isFinite(refillPerSec) || refillPerSec < 0) { + throw new Error('Shared rate limits require finite non-negative capacity and refillPerSec') + } + return async function allow(key) { + return new GatewayRateBucketsRepo(db || undefined).allow(hashKey(key), { capacity, refillPerSec }) + } + } const buckets = new Map() return function allow(key) { const now = Date.now() diff --git a/src/lib/db/postgres-migrations/004_gateway_rate_buckets.sql b/src/lib/db/postgres-migrations/004_gateway_rate_buckets.sql new file mode 100644 index 000000000..8dc264038 --- /dev/null +++ b/src/lib/db/postgres-migrations/004_gateway_rate_buckets.sql @@ -0,0 +1,5 @@ +CREATE TABLE gateway_rate_buckets ( + bucket_hash text COLLATE "C" PRIMARY KEY, + tokens double precision NOT NULL CHECK (tokens >= 0), + updated_at bigint NOT NULL +); diff --git a/src/lib/db/repos/gateway-rate-buckets-repo.mjs b/src/lib/db/repos/gateway-rate-buckets-repo.mjs new file mode 100644 index 000000000..b8eafdf20 --- /dev/null +++ b/src/lib/db/repos/gateway-rate-buckets-repo.mjs @@ -0,0 +1,35 @@ +import { getDb, withTransaction } from '../database.mjs' + +export class GatewayRateBucketsRepo { + constructor(db = getDb()) { + this.db = db + } + + async allow(bucketHash, { capacity, refillPerSec }) { + return withTransaction(this.db, async () => { + await this.db + .prepare(` + INSERT INTO gateway_rate_buckets (bucket_hash, tokens, updated_at) + VALUES ($1, $2, (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint) + ON CONFLICT (bucket_hash) DO NOTHING + `) + .run(bucketHash, capacity) + const bucket = await this.db + .prepare(` + SELECT tokens, updated_at FROM gateway_rate_buckets WHERE bucket_hash = $1 FOR UPDATE + `) + .get(bucketHash) + // Read the shared clock after taking the row lock, including time spent waiting. + const { now: databaseNow } = await this.db + .prepare('SELECT (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint AS now') + .get() + const now = Math.max(databaseNow, bucket.updated_at) + const tokens = Math.min(capacity, bucket.tokens + (Math.max(0, now - bucket.updated_at) / 1000) * refillPerSec) + const allowed = tokens >= 1 + await this.db + .prepare('UPDATE gateway_rate_buckets SET tokens = $2, updated_at = $3 WHERE bucket_hash = $1') + .run(bucketHash, allowed ? tokens - 1 : tokens, now) + return allowed + }) + } +} diff --git a/src/server.mjs b/src/server.mjs index 492577815..be3aa18ea 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -815,7 +815,7 @@ async function requireAuth(req, res) { // Panel session const session = await verifyPanelSession(token) if (session) { - if (!allowRate('panel:' + session.user)) { + if (!(await allowRate('panel:' + session.user))) { return rejectAuth( req, res, @@ -836,7 +836,7 @@ async function requireAuth(req, res) { } // Master env API key — unlimited admin if (timingSafeEqualStr(token, cfg.api_key)) { - if (!allowRate(token)) { + if (!(await allowRate(token))) { return rejectAuth( req, res, @@ -888,7 +888,7 @@ async function requireAuth(req, res) { }), ) } - if (!allowRate(token)) { + if (!(await allowRate(token))) { return rejectAuth( req, res, @@ -1264,6 +1264,7 @@ const server = http.createServer(async (req, res) => { if (req.method === 'GET' && p === '/v1/models') { if (!(await requireAuth(req, res))) return if (resolveInferenceBackend(req) === 'api') { + apiScheduler.reload(await apiEndpointStore.listRaw()) return json(res, 200, apiScheduler.catalog()) } const result = await fetchWorkerModels({ groupType: (await keyScopeFromRequest(req)).group_type }) From d30aeba9fe4baf54272a19d947ff47ced6399422 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:52:17 +0800 Subject: [PATCH 061/473] test(websocket): verify per-request admission and quota settlement --- test/unit/responses-ws.test.mjs | 132 +++++++++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 9 deletions(-) diff --git a/test/unit/responses-ws.test.mjs b/test/unit/responses-ws.test.mjs index f43e806c5..7f6bf2e13 100644 --- a/test/unit/responses-ws.test.mjs +++ b/test/unit/responses-ws.test.mjs @@ -166,7 +166,7 @@ async function listenWithHandleProtocol( { codexOps, requestLog, - apiKeyStore, + apiKeyStore = { reserve: async (record) => ({ ok: true, record, release: async () => {} }) }, routing = { websocket: { enabled: true }, pool: { fallback_wait_timeout_ms: 3000 }, @@ -802,10 +802,10 @@ test('real handleProtocol: one in-flight interrupt yields a single cancelled fin }, }, apiKeyStore: { + reserve: async (record) => ({ ok: true, record, release: async () => {} }), recordUsage(_key, usage) { usageRecords.push(usage) }, - release() {}, }, codexOps: { streamCodexKernel: async ({ onEvent, signal }) => { @@ -827,8 +827,8 @@ test('real handleProtocol: one in-flight interrupt yields a single cancelled fin }, }, }) + const opened = await openWs(live.url, { authorization: 'Bearer sk-test' }) try { - const opened = await openWs(live.url, { authorization: 'Bearer sk-test' }) opened.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'bill' })) await nextFrame(opened.ws, (frame) => frame.type === 'response.created') opened.ws.send(JSON.stringify({ type: 'response.interrupt', previous_response_id: 'resp_hp1' })) @@ -837,8 +837,8 @@ test('real handleProtocol: one in-flight interrupt yields a single cancelled fin assert.equal(finishCalls.length, 1) assert.equal(finishCalls[0].error_code, 'client_cancelled') assert.equal(usageRecords.length, 0) - opened.ws.close() } finally { + opened.ws?.terminate() await live.close() fs.rmSync(root, { recursive: true, force: true }) } @@ -850,6 +850,8 @@ test('real handleProtocol: interrupt after hop finish bills once without duplica const finishCalls = [] const usageRecords = [] const usage = { input_tokens: 12, output_tokens: 4, total_tokens: 16 } + const leaseEvents = [] + let reservation const live = await listenWithHandleProtocol(root, { requestLog: { start: () => ({ request_id: 'ws-complete-interrupt', t0: Date.now() }), @@ -858,10 +860,15 @@ test('real handleProtocol: interrupt after hop finish bills once without duplica }, }, apiKeyStore: { - recordUsage(_key, usageRow) { + async reserve(record) { + reservation = { ok: true, record, release: async () => leaseEvents.push('release') } + return reservation + }, + recordUsage(_key, usageRow, options) { + assert.equal(options.reservation, reservation) + leaseEvents.push('usage') usageRecords.push(usageRow) }, - release() {}, }, codexOps: { streamCodexKernel: async ({ onEvent }) => { @@ -875,8 +882,8 @@ test('real handleProtocol: interrupt after hop finish bills once without duplica }, }, }) + const opened = await openWs(live.url, { authorization: 'Bearer sk-test' }) try { - const opened = await openWs(live.url, { authorization: 'Bearer sk-test' }) const frames = [] opened.ws.on('message', (data) => frames.push(JSON.parse(String(data)))) opened.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'first' })) @@ -897,9 +904,116 @@ test('real handleProtocol: interrupt after hop finish bills once without duplica assert.equal(finishCalls[0].error_code, null) assert.equal(finishCalls[0].input_tokens, 12) assert.equal(finishCalls[0].output_tokens, 4) - assert.equal(usageRecords.length, 0) - opened.ws.close() + assert.equal(usageRecords.length, 1, 'a verified Codex stream consumes quota once despite a later interrupt') + assert.equal(usageRecords[0].input_tokens, 12) + assert.equal(usageRecords[0].output_tokens, 4) + assert.deepEqual(leaseEvents, ['usage', 'release'], 'quota is settled before releasing the request reservation') + } finally { + opened.ws?.terminate() + await live.close() + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('real handleProtocol: shared key admission spans WebSockets and cancellation releases each request lease', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-responses-ws-shared-key-')) + writeGptVm(root, 'vm-gpt-a') + let reserveCalls = 0 + let inflight = 0 + let releases = 0 + let upstreamHops = 0 + const usageRecords = [] + const live = await listenWithHandleProtocol(root, { + requestLog: { start: () => ({ request_id: 'ws-shared-key', t0: Date.now() }), finish() {} }, + apiKeyStore: { + async reserve(record) { + reserveCalls++ + if (inflight) { + return { + ok: false, + status: 429, + code: 'api_key_concurrency_limit', + message: 'API key concurrency limit (1)', + } + } + inflight++ + let released = false + return { + ok: true, + record, + async release() { + if (released) return + released = true + inflight-- + releases++ + }, + } + }, + recordUsage(_key, usage) { + usageRecords.push(usage) + }, + }, + codexOps: { + async streamCodexKernel({ onEvent, signal }) { + const id = `resp_shared_${++upstreamHops}` + await onEvent(`data: ${JSON.stringify({ type: 'response.created', response: { id, output: [] } })}\n\n`) + await new Promise((resolve) => { + if (signal.aborted) resolve() + else signal.addEventListener('abort', resolve, { once: true }) + }) + return { ok: false, status: 200, committed: false, body: { error: { code: 'client_cancelled' } } } + }, + }, + }) + const sockets = [] + const waitForRelease = async (count) => { + const deadline = Date.now() + 2000 + while (releases < count) { + assert.ok(Date.now() < deadline, 'cancelled request lease release timeout') + await new Promise((resolve) => setTimeout(resolve, 10)) + } + } + try { + const first = await openWs(live.url, { authorization: 'Bearer sk-test' }) + sockets.push(first.ws) + const second = await openWs(live.url, { authorization: 'Bearer sk-test' }) + sockets.push(second.ws) + assert.equal(first.ok, true) + assert.equal(second.ok, true) + assert.equal(reserveCalls, 0, 'WebSocket handshakes do not reserve inference capacity') + + const created = nextFrame(first.ws, (frame) => frame.type === 'response.created') + first.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'pending' })) + assert.equal((await created).response.id, 'resp_shared_1') + assert.equal(inflight, 1) + + // Each socket serializes its own queue. A second socket exercises overlapping requests with the same key. + const denied = nextFrame(second.ws, (frame) => frame.type === 'error') + second.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'overlap' })) + const error = await denied + assert.equal(error.status, 429) + assert.equal(error.error.code, 'api_key_concurrency_limit') + assert.equal(upstreamHops, 1) + + const accepted = nextFrame(first.ws, (frame) => frame.type === 'response.interrupt.accepted') + first.ws.send(JSON.stringify({ type: 'response.interrupt', previous_response_id: 'resp_shared_1' })) + await accepted + await waitForRelease(1) + assert.equal(inflight, 0) + + const retried = nextFrame(second.ws, (frame) => frame.type === 'response.created') + second.ws.send(JSON.stringify({ type: 'response.create', model: 'gpt-5.4', input: 'retry after cancellation' })) + assert.equal((await retried).response.id, 'resp_shared_2') + const cancelled = nextFrame(second.ws, (frame) => frame.type === 'response.interrupt.accepted') + second.ws.send(JSON.stringify({ type: 'response.interrupt', previous_response_id: 'resp_shared_2' })) + await cancelled + await waitForRelease(2) + assert.equal(inflight, 0) + assert.equal(reserveCalls, 3) + assert.equal(usageRecords.length, 0, 'cancelled requests do not consume completed-request quota') } finally { + for (const ws of sockets) ws?.terminate() + await live.handler.drain() await live.close() fs.rmSync(root, { recursive: true, force: true }) } From 126c4d73a743dd9daa89492265dcc48a61ccb1f6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:52:26 +0800 Subject: [PATCH 062/473] test: retain user concurrency for deleted executing keys --- test/unit/api-key-admission.test.mjs | 63 +++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/test/unit/api-key-admission.test.mjs b/test/unit/api-key-admission.test.mjs index c4cb3d894..3509c7f1a 100644 --- a/test/unit/api-key-admission.test.mjs +++ b/test/unit/api-key-admission.test.mjs @@ -13,7 +13,15 @@ async function replicas() { async function user(store, { role = 'user', concurrency = 1 } = {}) { const id = 'user-' + crypto.randomUUID() const now = new Date().toISOString() - return store.users.insert({ id, username: id, password_hash: 'test', role, concurrency, created_at: now, updated_at: now }) + return store.users.insert({ + id, + username: id, + password_hash: 'test', + role, + concurrency, + created_at: now, + updated_at: now, + }) } test('independent replicas atomically enforce key concurrency and release once', async () => { @@ -87,9 +95,13 @@ test('quota count and pending settlement commit together under the key lock', as const key = await first.create({ max_concurrency: 0, quota_requests: 2 }) const gate = await first.reserve(key) let entered - const waiting = new Promise((resolve) => { entered = resolve }) + const waiting = new Promise((resolve) => { + entered = resolve + }) let resume - const proceed = new Promise((resolve) => { resume = resolve }) + const proceed = new Promise((resolve) => { + resume = resolve + }) const record = first.repo.recordUsage.bind(first.repo) first.repo.recordUsage = async (...args) => { const saved = await record(...args) @@ -101,7 +113,9 @@ test('quota count and pending settlement commit together under the key lock', as await waiting const observed = await second.getById(key.id) assert.equal(observed.quota_requests_used, 0, 'an uncommitted completion is invisible') - const otherPending = await second.db.prepare('SELECT count(*) AS count FROM api_key_admissions WHERE settled_at IS NULL AND released_at IS NULL').get() + const otherPending = await second.db + .prepare('SELECT count(*) AS count FROM api_key_admissions WHERE settled_at IS NULL AND released_at IS NULL') + .get() assert.equal(otherPending.count, 1, 'the pending reservation remains visible until completion commits') const competing = second.reserve(key) resume() @@ -197,7 +211,46 @@ test('old executing reservations do not expire or disappear when their connectio await first.db.prepare("UPDATE api_key_admissions SET created_at = clock_timestamp() - interval '2 hours'").run() await second.db.close() assert.equal((await first.reserve(key)).code, 'api_key_concurrency_limit') - const row = await first.db.prepare('SELECT owner_id, released_at FROM api_key_admissions WHERE id = $1').get(gate.reservationId) + const row = await first.db + .prepare('SELECT owner_id, released_at FROM api_key_admissions WHERE id = $1') + .get(gate.reservationId) assert.equal(row.owner_id, gate.ownerId) assert.equal(row.released_at, null) }) + +test('deleting an executing key retains its owner concurrency until release', async () => { + const [first, second] = await replicas() + const owner = await user(first) + const key = await first.create({ user_id: owner.id, max_concurrency: 0 }) + const otherKey = await first.create({ user_id: owner.id, max_concurrency: 0 }) + const gate = await first.reserve(key) + assert.equal(await first.remove(key.id), true) + assert.equal((await second.reserve(otherKey)).code, 'user_concurrency_limit') + await gate.release() + const next = await second.reserve(otherKey) + assert.equal(next.ok, true) + await next.release() +}) + +test('pending quota arithmetic remains exact above the safe integer boundary', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0 }) + await first.db.prepare('UPDATE api_keys SET quota_requests = $1, quota_requests_used = $2 WHERE id = $3') + .run('9007199254740994', '9007199254740993', key.id) + const gate = await first.reserve(key) + assert.equal(gate.ok, true) + assert.equal((await second.reserve(key)).code, 'api_key_quota_exhausted') + await first.recordUsage(key, {}, { reservation: gate }) + await gate.release() + assert.equal((await first.getById(key.id)).quota_requests_used, '9007199254740994') + assert.equal((await second.reserve(key)).code, 'api_key_quota_exhausted') +}) + +test('an unavailable shared database cannot fall back to process-local admission', async () => { + const [first, second] = await replicas() + const key = await first.create({ max_concurrency: 0 }) + await second.db.close() + await assert.rejects(second.reserve(key), /database is closed/) + assert.equal(second.inflight.size, 0) + assert.equal((await first.list())[0].inflight, 0) +}) From c736e81d823953c9d88536be1448c659e42b0b55 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:52:44 +0800 Subject: [PATCH 063/473] feat: atomically reserve API key limits in PostgreSQL --- src/lib/admin/api-keys.mjs | 120 ++++++++++++++---- .../003_api_key_admissions.sql | 17 +++ src/lib/db/repos/api-key-admissions-repo.mjs | 106 ++++++++++++++++ test/unit/api-key-admission.test.mjs | 3 +- 4 files changed, 222 insertions(+), 24 deletions(-) create mode 100644 src/lib/db/postgres-migrations/003_api_key_admissions.sql create mode 100644 src/lib/db/repos/api-key-admissions-repo.mjs diff --git a/src/lib/admin/api-keys.mjs b/src/lib/admin/api-keys.mjs index a2c35fa5c..5acb50b47 100644 --- a/src/lib/admin/api-keys.mjs +++ b/src/lib/admin/api-keys.mjs @@ -7,8 +7,8 @@ * * Master env key KIN_API_KEY is separate and unlimited. * Managed keys live in the `api_keys` table (see lib/db/). - * Transient state (inflight concurrency, RPM buckets) stays in memory, - * mirroring sub2api's Redis-transient split. + * Standalone state (inflight concurrency, RPM buckets) stays in memory. + * Kubernetes admissions use PostgreSQL for cross-replica limits and quotas. * * Auth always goes through `key_hash` (HMAC). `key_secret` is a separate * at-rest copy of the plaintext — encrypted with KIN_DB_SECRET when set — so @@ -24,6 +24,8 @@ import { ApiKeysRepo } from '../db/repos/api-keys-repo.mjs' import { maybeEncrypt, maybeDecrypt } from '../db/secure.mjs' import { calculateCost, normalizeUsage } from './pricing.mjs' import { UsersRepo } from '../db/repos/users-repo.mjs' +import { ApiKeyAdmissionsRepo } from '../db/repos/api-key-admissions-repo.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' const KEY_PREFIX = 'sk-vm-' const HASH_MARKER = 'hmac:' @@ -121,7 +123,7 @@ function timingSafeEqualStr(a, b) { } export class ApiKeyStore { - constructor({ dataDir, db, hashSecret } = {}) { + constructor({ dataDir, db, hashSecret, sharedAdmission = isKubernetesMode() } = {}) { this.db = resolveStoreDb({ db, dataDir }) this.repo = new ApiKeysRepo(this.db) this.hashSecret = String( @@ -137,6 +139,9 @@ export class ApiKeyStore { this.inflight = new Map() // id → count this.rpmBuckets = new Map() // id → number[] timestamps ms this.users = new UsersRepo(this.db) + this.sharedAdmission = sharedAdmission + this.admissionOwnerId = 'controller_' + crypto.randomUUID() + this.admissions = new ApiKeyAdmissionsRepo(this.db) } /** sub2api: an api key of a disabled/deleted user must not authenticate. */ @@ -159,10 +164,12 @@ export class ApiKeyStore { this.db = db this.repo = new ApiKeysRepo(db) this.users = new UsersRepo(db) + this.admissions = new ApiKeyAdmissionsRepo(db) } async list({ reveal = false } = {}) { const names = new Map() + const inflight = this.sharedAdmission ? await this.admissions.inflight() : this.inflight try { for (const pool of await new VmPoolsRepo(this.db).list()) names.set(pool.id, pool.name) } catch { @@ -170,7 +177,7 @@ export class ApiKeyStore { } return (await this.repo.list()).map((k) => { const v = publicKeyView(k, { reveal }) - v.inflight = this.inflight.get(k.id) || 0 + v.inflight = inflight.get(k.id) || 0 if (v.vm_pool_id) v.vm_pool_name = names.get(v.vm_pool_id) || null return v }) @@ -374,9 +381,9 @@ export class ApiKeyStore { * Pre-flight gate for a managed key. Does not acquire concurrency. * @returns {{ ok: true } | { ok: false, code, message, status }} */ - async canAccept(rec, now = Date.now(), { reserve = false } = {}) { - const owner = rec?.user_id ? await this.users.getById(rec.user_id) : null - const ownedKeys = rec?.user_id ? await this.repo.list() : [] + async canAccept(rec, now = Date.now(), { reserve = false, state = null } = {}) { + const owner = state ? state.owner : rec?.user_id ? await this.users.getById(rec.user_id) : null + const ownedKeys = !state && rec?.user_id ? await this.repo.list() : [] if (!rec) return { ok: false, code: 'invalid_api_key', message: 'Invalid credentials', status: 401 } if (rec.status !== 'active') { if (rec.status === 'quota_exhausted') { @@ -387,7 +394,13 @@ export class ApiKeyStore { if (this._expired(rec, now)) { return { ok: false, code: 'api_key_expired', message: 'API key has expired', status: 403 } } - if (rec.quota_requests > 0 && BigInt(rec.quota_requests_used ?? 0) >= BigInt(rec.quota_requests)) { + if (rec.user_id && (!owner || owner.status !== 'active')) { + return { ok: false, code: 'user_disabled', message: 'API key owner is disabled', status: 403 } + } + if ( + rec.quota_requests > 0 && + BigInt(rec.quota_requests_used ?? 0) + BigInt(state?.pendingRequests ?? 0) >= BigInt(rec.quota_requests) + ) { return { ok: false, code: 'api_key_quota_exhausted', @@ -426,7 +439,7 @@ export class ApiKeyStore { } } if (rec.rpm > 0) { - const n = this._rpmCount(rec.id, now) + const n = state ? state.rpmCount : this._rpmCount(rec.id, now) if (n >= rec.rpm) { return { ok: false, @@ -434,10 +447,11 @@ export class ApiKeyStore { message: `API key rate limit exceeded (${rec.rpm}/min)`, status: 429, detail: { rpm: rec.rpm, current: n }, + ...(state?.retryAfterMs ? { retry_after_ms: state.retryAfterMs } : {}), } } } - const inflight = this.inflight.get(rec.id) || 0 + const inflight = state ? state.inflight : this.inflight.get(rec.id) || 0 if (rec.max_concurrency > 0 && inflight >= rec.max_concurrency) { return { ok: false, @@ -452,10 +466,11 @@ export class ApiKeyStore { if (rec.user_id) { const cap = owner?.role === 'admin' ? 0 : Number(owner?.concurrency) || 0 if (cap > 0) { - let total = 0 - for (const k of ownedKeys) { - if (k.user_id === rec.user_id) total += this.inflight.get(k.id) || 0 - } + let total = state ? state.userInflight : 0 + if (!state) + for (const k of ownedKeys) { + if (k.user_id === rec.user_id) total += this.inflight.get(k.id) || 0 + } if (total >= cap) { return { ok: false, @@ -480,6 +495,59 @@ export class ApiKeyStore { return this.canAccept(rec, now, { reserve: true }) } + /** + * Reserve before starting work, settle billable usage with { reservation }, + * then await release in the request's outer finally. A failed settlement + * must retain its reservation until a fenced recovery can reconcile it. + */ + async reserve(rec) { + if (!this.sharedAdmission) { + const gate = await this.acquire(rec) + if (!gate.ok) return gate + let released = false + return { + ok: true, + shared: false, + record: rec, + release: async () => { + if (released) return + released = true + this.release(rec) + }, + } + } + const admissions = this.admissions + const gate = await admissions.reserve(rec?.id, this.admissionOwnerId, (current, state) => { + if (current && rec.key_hash !== current.key_hash) { + return { ok: false, code: 'invalid_api_key', message: 'Invalid credentials', status: 401 } + } + // Scope checks happen before this gate. Require authentication again + // if an operator changed the policy while this request was prepared. + if ( + current && + ['category', 'group_type', 'allowed_vms', 'vm_pool_id', 'user_id', 'group_id'].some( + (field) => (rec[field] ?? null) !== (current[field] ?? null), + ) + ) { + return { ok: false, code: 'api_key_changed', message: 'API key policy changed; retry the request', status: 403 } + } + return this.canAccept(current, state.now, { state }) + }) + if (!gate.ok) return gate + let settled = false + return { + ...gate, + shared: true, + get settled() { + return settled + }, + _markSettled() { + settled = true + }, + release: () => admissions.release(gate), + } + } + release(recOrId) { const id = typeof recOrId === 'string' ? recOrId : recOrId?.id if (!id) return @@ -488,7 +556,7 @@ export class ApiKeyStore { else this.inflight.set(id, n) } - async recordUsage(recOrId, usage = {}, { model = null, rateMultiplier = 1, officialCost } = {}) { + async recordUsage(recOrId, usage = {}, { model = null, rateMultiplier = 1, officialCost, reservation } = {}) { const id = typeof recOrId === 'string' ? recOrId : recOrId?.id if (!(await this.repo.getById(id))) return null const normalized = normalizeUsage(usage) @@ -504,14 +572,20 @@ export class ApiKeyStore { // key quota/windows track actual (group-rated) spend, sub2api-style const rate = Number.isFinite(Number(rateMultiplier)) ? Number(rateMultiplier) : 1 const cost = billed * rate - const rec = await this.repo.recordUsage(id, { - tokens_in: integerValue(normalized.input_tokens ?? normalized.tokens_in), - tokens_out: integerValue(normalized.output_tokens ?? normalized.tokens_out), - cache_read_tokens: integerValue(normalized.cache_read_input_tokens ?? normalized.cache_read_tokens), - cache_creation_tokens: integerValue(normalized.cache_creation_input_tokens ?? normalized.cache_creation_tokens), - cost, - }) - return rec + const write = () => + this.repo.recordUsage(id, { + tokens_in: integerValue(normalized.input_tokens ?? normalized.tokens_in), + tokens_out: integerValue(normalized.output_tokens ?? normalized.tokens_out), + cache_read_tokens: integerValue(normalized.cache_read_input_tokens ?? normalized.cache_read_tokens), + cache_creation_tokens: integerValue(normalized.cache_creation_input_tokens ?? normalized.cache_creation_tokens), + cost, + }) + if (reservation?.shared) { + const rec = await this.admissions.settle(id, reservation, write) + if (rec) reservation._markSettled?.() + return rec + } + return write() } async snapshot() { diff --git a/src/lib/db/postgres-migrations/003_api_key_admissions.sql b/src/lib/db/postgres-migrations/003_api_key_admissions.sql new file mode 100644 index 000000000..7400cbfcb --- /dev/null +++ b/src/lib/db/postgres-migrations/003_api_key_admissions.sql @@ -0,0 +1,17 @@ +CREATE TABLE api_key_admissions ( + id text COLLATE "C" PRIMARY KEY, + key_id text COLLATE "C" NOT NULL, + user_id text COLLATE "C", + owner_id text COLLATE "C" NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + settled_at timestamptz, + released_at timestamptz +); + +CREATE INDEX idx_api_key_admissions_key_window ON api_key_admissions (key_id, created_at); +CREATE INDEX idx_api_key_admissions_key_active ON api_key_admissions (key_id) WHERE released_at IS NULL; +CREATE INDEX idx_api_key_admissions_user_active ON api_key_admissions (user_id) WHERE released_at IS NULL; + +-- A reservation has no expiry: an old request may still execute upstream. +-- owner_id identifies the controller for a future fenced recovery protocol. +-- Key deletion must retain an executing request's user concurrency reservation. diff --git a/src/lib/db/repos/api-key-admissions-repo.mjs b/src/lib/db/repos/api-key-admissions-repo.mjs new file mode 100644 index 000000000..58b7982fd --- /dev/null +++ b/src/lib/db/repos/api-key-admissions-repo.mjs @@ -0,0 +1,106 @@ +import crypto from 'node:crypto' +import { withTransaction } from '../database.mjs' +import { ApiKeysRepo } from './api-keys-repo.mjs' +import { UsersRepo } from './users-repo.mjs' + +/** Shared concurrency, request-quota reservations and the rolling RPM history. */ +export class ApiKeyAdmissionsRepo { + constructor(db) { + this.db = db + this.keys = new ApiKeysRepo(db) + this.users = new UsersRepo(db) + } + + async reserve(keyId, ownerId, validate) { + return withTransaction(this.db, async () => { + // Every admission locks key then owner. The owner lock serializes + // requests made with different keys belonging to the same user. + const record = await this.keys.getById(keyId, { forUpdate: true }) + const owner = record?.user_id ? await this.users.getById(record.user_id, { forUpdate: true }) : null + const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() + const nowMs = now.getTime() + if (!record) return validate(record, { owner, now: nowMs }) + // Released rows outlive the full RPM window. Active rows never expire, + // including those whose original controller has disconnected or died. + await this.db + .prepare(`DELETE FROM api_key_admissions + WHERE key_id = $1 AND released_at IS NOT NULL AND created_at <= $2::timestamptz - interval '60 seconds'`) + .run(keyId, now) + const state = await this.db + .prepare(`SELECT + count(*) FILTER (WHERE released_at IS NULL) AS inflight, + count(*) FILTER (WHERE released_at IS NULL AND settled_at IS NULL) AS pending_requests, + count(*) FILTER (WHERE created_at > $2::timestamptz - interval '60 seconds') AS rpm_count, + min(created_at) FILTER (WHERE created_at > $2::timestamptz - interval '60 seconds') AS rpm_oldest + FROM api_key_admissions WHERE key_id = $1`) + .get(keyId, now) + const userInflight = record.user_id + ? ( + await this.db + .prepare('SELECT count(*) AS count FROM api_key_admissions WHERE user_id = $1 AND released_at IS NULL') + .get(record.user_id) + ).count + : 0 + const gate = await validate(record, { + owner, + now: nowMs, + inflight: state.inflight, + pendingRequests: state.pending_requests, + userInflight, + rpmCount: state.rpm_count, + retryAfterMs: state.rpm_oldest ? Math.max(1, state.rpm_oldest.getTime() + 60_000 - nowMs) : undefined, + }) + if (!gate.ok) return gate + const reservationId = crypto.randomUUID() + await this.db + .prepare(`INSERT INTO api_key_admissions (id, key_id, user_id, owner_id, created_at) + VALUES ($1, $2, $3, $4, $5)`) + .run(reservationId, keyId, record.user_id, ownerId, now) + return { ok: true, record, reservationId, ownerId } + }) + } + + async release({ reservationId, ownerId }) { + // Retain the history: cancellation releases concurrency/quota but a + // request admitted in the last minute still consumes RPM. + await this.db + .prepare(`UPDATE api_key_admissions SET released_at = clock_timestamp() + WHERE id = $1 AND owner_id = $2 AND released_at IS NULL`) + .run(reservationId, ownerId) + } + + async settle(keyId, { reservationId, ownerId }, recordUsage) { + return withTransaction(this.db, async () => { + // Match reserve's key lock. The counter increment and pending-quota + // removal become visible in one commit, so neither overlaps nor gaps. + const record = await this.keys.getById(keyId, { forUpdate: true }) + if (!record) return null + const row = await this.db.prepare('SELECT * FROM api_key_admissions WHERE id = $1 FOR UPDATE').get(reservationId) + if (!row || row.key_id !== keyId || row.owner_id !== ownerId) { + throw Object.assign(new Error('Invalid API key reservation'), { code: 'invalid_api_key_reservation' }) + } + if (row.settled_at) return record + if (row.released_at) { + throw Object.assign(new Error('API key reservation has been released'), { + code: 'api_key_reservation_released', + }) + } + const saved = await recordUsage() + await this.db + .prepare('UPDATE api_key_admissions SET settled_at = clock_timestamp() WHERE id = $1') + .run(reservationId) + return saved + }) + } + + async inflight() { + return new Map( + ( + await this.db + .prepare(`SELECT key_id, count(*) AS count FROM api_key_admissions + WHERE released_at IS NULL GROUP BY key_id`) + .all() + ).map((row) => [row.key_id, row.count]), + ) + } +} diff --git a/test/unit/api-key-admission.test.mjs b/test/unit/api-key-admission.test.mjs index 3509c7f1a..3c8d24e39 100644 --- a/test/unit/api-key-admission.test.mjs +++ b/test/unit/api-key-admission.test.mjs @@ -235,7 +235,8 @@ test('deleting an executing key retains its owner concurrency until release', as test('pending quota arithmetic remains exact above the safe integer boundary', async () => { const [first, second] = await replicas() const key = await first.create({ max_concurrency: 0 }) - await first.db.prepare('UPDATE api_keys SET quota_requests = $1, quota_requests_used = $2 WHERE id = $3') + await first.db + .prepare('UPDATE api_keys SET quota_requests = $1, quota_requests_used = $2 WHERE id = $3') .run('9007199254740994', '9007199254740993', key.id) const gate = await first.reserve(key) assert.equal(gate.ok, true) From 2605bd6b2c255bdbf3e00c4c34f7ef359ccb0c06 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:53:36 +0800 Subject: [PATCH 064/473] test(policy): cover cross-replica read-only cache refresh --- test/unit/shared-policy-cache.test.mjs | 96 ++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 test/unit/shared-policy-cache.test.mjs diff --git a/test/unit/shared-policy-cache.test.mjs b/test/unit/shared-policy-cache.test.mjs new file mode 100644 index 000000000..c1dffcd9c --- /dev/null +++ b/test/unit/shared-policy-cache.test.mjs @@ -0,0 +1,96 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { RuntimeSettings } from '../../src/lib/db/settings-state.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' +import * as claudePolicy from '../../src/lib/protocol/model-policy.mjs' +import * as gptPolicy from '../../src/lib/protocol/gpt-model-policy.mjs' +import { clearModelsCache, gatewayModelCatalog } from '../../src/lib/protocol/models.mjs' +import { invalidatePriceCatalog, resolvePrice } from '../../src/lib/admin/price-catalog.mjs' + +const claude = 'claude-sonnet-5-5' +const gpt = 'gpt-5.6' +const price = (input) => ({ input, output: 11, cache_read: 0.9, cache_5m: 11.25, cache_1h: 18 }) + +function refreshPolicies(snapshot) { + const nextClaude = claudePolicy.refreshModelPolicy(snapshot.model_policy) + const nextGpt = gptPolicy.refreshGptModelPolicy(snapshot.gpt_model_policy) + invalidatePriceCatalog() + assert.equal(nextClaude, claudePolicy.getModelPolicy()) + assert.equal(nextGpt, gptPolicy.getGptModelPolicy()) + assert.equal(nextClaude.then, undefined) + assert.equal(nextGpt.then, undefined) +} + +test('a refreshed PG settings snapshot updates model switches, catalog and prices without writing settings', async (t) => { + const db = getDb() + const settings = new RuntimeSettings(db) + const repo = new SettingsRepo(db) + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(async () => { + await other.close() + clearModelsCache() + invalidatePriceCatalog() + }) + await repo.set('model_policy', claudePolicy.seedDefaultPolicy()) + await repo.set('gpt_model_policy', gptPolicy.seedGptPolicy()) + await repo.set('model_price_overrides', { [`anthropic/${claude}`]: price(1) }) + await claudePolicy.loadModelPolicy({ force: true }) + await gptPolicy.loadGptModelPolicy({ force: true }) + clearModelsCache() + invalidatePriceCatalog() + assert.equal(claudePolicy.isModelEnabled(claude), true) + assert.equal(gptPolicy.isGptModelEnabled(gpt), true) + assert.equal(resolvePrice(claude).price.input, 1) + + const remote = new SettingsRepo(other) + await withTransaction(other, async () => { + await remote.set('model_policy', { + catalog_mode: 'policy_only', + models: { [claude]: { enabled: false }, 'claude-sonnet-9-review': { enabled: true } }, + }) + await remote.set('gpt_model_policy', { + source: 'codex', + models: { [gpt]: { enabled: false }, 'gpt-9-review': { enabled: true } }, + }) + await remote.set('model_price_overrides', { [`anthropic/${claude}`]: price(9) }) + }) + const before = await db.prepare('SELECT key, value, updated_at FROM settings ORDER BY key').all() + // A write from another connection does not change this process's loaded policies. + assert.equal(claudePolicy.isModelEnabled(claude), true) + assert.equal(gptPolicy.isGptModelEnabled(gpt), true) + assert.equal(resolvePrice(claude).price.input, 1) + + const snapshot = await settings.refresh() + const originalSnapshot = structuredClone(snapshot) + refreshPolicies(snapshot) + assert.deepEqual(snapshot, originalSnapshot) + assert.equal(claudePolicy.isModelEnabled(claude), false) + assert.equal(gptPolicy.isGptModelEnabled(gpt), false) + const ids = gatewayModelCatalog().data.map((row) => row.id) + assert.equal(ids.includes(claude), false) + assert.equal(ids.includes(gpt), false) + assert.equal(ids.includes('claude-sonnet-9-review'), true) + assert.equal(ids.includes('gpt-9-review'), true) + // A live GPT catalog must not add seed-only ids on every refresh. + assert.equal(ids.includes('gpt-6-sol'), false) + assert.equal(resolvePrice(claude).price.input, 9) + // Loaded state prevents a later ordinary load from persisting a seed or migration. + await claudePolicy.loadModelPolicy() + await gptPolicy.loadGptModelPolicy() + assert.deepEqual(await db.prepare('SELECT key, value, updated_at FROM settings ORDER BY key').all(), before) +}) + +test('missing policy values use runtime defaults without seeding PostgreSQL', async () => { + const db = getDb() + const settings = new RuntimeSettings(db) + const before = await db.prepare('SELECT key, value, updated_at FROM settings ORDER BY key').all() + refreshPolicies(await settings.refresh()) + assert.equal(claudePolicy.isModelEnabled(claude), true) + assert.equal(gptPolicy.isGptModelEnabled(gpt), true) + await claudePolicy.loadModelPolicy() + await gptPolicy.loadGptModelPolicy() + assert.deepEqual(await db.prepare('SELECT key, value, updated_at FROM settings ORDER BY key').all(), before) + assert.equal(await new SettingsRepo(db).has('model_policy'), false) + assert.equal(await new SettingsRepo(db).has('gpt_model_policy'), false) +}) From e1646c23228f3203dc00dddac6252c7e79b3754b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:54:48 +0800 Subject: [PATCH 065/473] feat(policy): apply refreshed settings without persistence --- src/lib/protocol/gpt-model-policy.mjs | 30 +++++++++++++++++++-------- src/lib/protocol/model-policy.mjs | 7 +++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/lib/protocol/gpt-model-policy.mjs b/src/lib/protocol/gpt-model-policy.mjs index ac25de0c1..97024cb05 100644 --- a/src/lib/protocol/gpt-model-policy.mjs +++ b/src/lib/protocol/gpt-model-policy.mjs @@ -64,18 +64,23 @@ async function persist() { } } +function normalizeGptPolicy(stored) { + if (!stored || typeof stored !== 'object' || !stored.models || typeof stored.models !== 'object') return null + const live = stored.source === 'codex' && Object.keys(stored.models).length + const seed = seedGptPolicy() + const normalized = { + ...(live ? stored : { ...seed, ...stored, models: { ...seed.models, ...stored.models } }), + platform: 'openai', + } + normalized.source = normalized.source || 'settings' + return normalized +} + export async function loadGptModelPolicy({ force = false } = {}) { if (loaded && !force) return policy try { - const stored = new SettingsRepo().get(SETTINGS_KEY, null) - if (stored && typeof stored === 'object' && stored.models && typeof stored.models === 'object') { - const live = stored.source === 'codex' && Object.keys(stored.models).length - policy = { - ...(live ? stored : { ...seedGptPolicy(), ...stored, models: { ...seedGptPolicy().models, ...stored.models } }), - platform: 'openai', - } - policy.source = policy.source || 'settings' - } else { + policy = normalizeGptPolicy(new SettingsRepo().get(SETTINGS_KEY, null)) + if (!policy) { policy = seedGptPolicy() await persist() } @@ -86,6 +91,13 @@ export async function loadGptModelPolicy({ force = false } = {}) { return policy } +/** Apply a refreshed settings value without persisting missing-policy defaults. */ +export function refreshGptModelPolicy(stored) { + policy = normalizeGptPolicy(stored) || seedGptPolicy() + loaded = true + return policy +} + export function getGptModelPolicy() { return policy } diff --git a/src/lib/protocol/model-policy.mjs b/src/lib/protocol/model-policy.mjs index 72f8c5cb4..eb5561427 100644 --- a/src/lib/protocol/model-policy.mjs +++ b/src/lib/protocol/model-policy.mjs @@ -624,6 +624,13 @@ async function persistHaiku55Model(repo, stored) { return next } +/** Apply a refreshed settings value without running startup migrations or writes. */ +export function refreshModelPolicy(stored) { + policy = normalizePolicy(stored) + loaded = true + return policy +} + export function getModelPolicy() { return policy } From 1b327b32871cee70fc80fc2cfc5a3691dff33371 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:58:49 +0800 Subject: [PATCH 066/473] test: retain request reservations after execution failures --- test/fixtures/protocol-settlement-failure.mjs | 81 +++++++++++++++++++ .../unit/protocol-settlement-failure.test.mjs | 27 +++++++ 2 files changed, 108 insertions(+) create mode 100644 test/fixtures/protocol-settlement-failure.mjs create mode 100644 test/unit/protocol-settlement-failure.test.mjs diff --git a/test/fixtures/protocol-settlement-failure.mjs b/test/fixtures/protocol-settlement-failure.mjs new file mode 100644 index 000000000..b33ee4386 --- /dev/null +++ b/test/fixtures/protocol-settlement-failure.mjs @@ -0,0 +1,81 @@ +/** Isolate module mocks so ordinary tests need no experimental Node flag. */ +import { test, mock } from 'node:test' +import assert from 'node:assert/strict' +import { Readable } from 'node:stream' +import { EventEmitter } from 'node:events' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { readBody } from '../../src/lib/http/respond.mjs' + +const [route, failure] = process.env.VM2API_SETTLEMENT_FAILURE_CASE.split(':') +async function completedUpstream({ logBag, json, res }) { + logBag.final_state = 'verified' + logBag.usage = { input_tokens: 1, output_tokens: 1 } + const result = json(res, 200, { upstream_completed: true }) + if (failure === 'backend') throw new Error('simulated execution-stage backend failure') + return result +} +mock.module(new URL('../../src/lib/protocol/handle-codex.mjs', import.meta.url), { + namedExports: { + handleCodexProtocol: completedUpstream, + handleCodexSearch: completedUpstream, + usageFromSseLine: () => null, + }, +}) +const { createHandleProtocol } = await import('../../src/lib/protocol/handle-protocol.mjs') + +test(`${route}:${failure}`, async () => { + const db = getDb() + const store = new ApiKeyStore({ db, sharedAdmission: true }) + const key = await store.create({ max_concurrency: 0, quota_requests: 1 }) + const handler = createHandleProtocol({ + cfg: { + limits: { max_body_bytes: 1 << 20 }, + rewrite: { enabled: false }, + intercept: { rules: [] }, + distill: { enabled: false }, + paths: { project: '/tmp', data: '/tmp' }, + }, + json(res, status, body) { + res.statusCode = status + res.body = body + }, + readBody, + requireAuth(req) { + req.apiKeyRecord = key + req.apiKeyKind = 'managed' + return true + }, + requestLog: { start: () => ({ request_id: 'settlement-failure', t0: Date.now() }) }, + stats: { errors: 0, requests: 0, by_route: {} }, + groupsRepo: { + rateMultiplier() { + if (failure === 'multiplier') throw new Error('simulated execution-stage multiplier failure') + return 1 + }, + }, + apiKeyStore: store, + stickyRouter: { resolve: () => null }, + routingConfig: { + codex: { + clients: { openai_compatible: 'allow', unknown: 'allow' }, + protocols: { 'openai.responses': { enabled: true, mode: 'native' } }, + }, + }, + }) + const req = Readable.from([Buffer.from(JSON.stringify({ model: 'gpt-5.4', input: 'x' }))]) + req.headers = { 'user-agent': 'openai-node/1.0' } + const res = new EventEmitter() + res.setHeader = () => {} + const request = route === 'search' + ? handler.handleSearch(req, res, '/v1/alpha/search') + : handler.handleProtocol(req, res, 'openai.responses', '/v1/responses') + await assert.rejects(request, new RegExp(`simulated execution-stage ${failure} failure`)) + assert.equal(res.body.upstream_completed, true) + assert.equal((await store.getById(key.id)).quota_requests_used, 0) + const reservation = await db.prepare('SELECT released_at, settled_at FROM api_key_admissions WHERE key_id = $1').get(key.id) + assert.equal(reservation.settled_at, null) + assert.equal(reservation.released_at, null, 'retain the uncertain completed request until recovery can reconcile usage') + const next = await store.reserve(key) + assert.equal(next.code, 'api_key_quota_exhausted', 'pending quota must prevent another execution') +}) diff --git a/test/unit/protocol-settlement-failure.test.mjs b/test/unit/protocol-settlement-failure.test.mjs new file mode 100644 index 000000000..00e603695 --- /dev/null +++ b/test/unit/protocol-settlement-failure.test.mjs @@ -0,0 +1,27 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const fixture = fileURLToPath(new URL('../fixtures/protocol-settlement-failure.mjs', import.meta.url)) +const setup = fileURLToPath(new URL('../pg-setup.mjs', import.meta.url)) + +for (const route of ['responses', 'search']) { + for (const failure of ['multiplier', 'backend']) { + test(`${route} preserves pending request quota after an execution-stage ${failure} failure`, () => { + const env = { ...process.env, VM2API_SETTLEMENT_FAILURE_CASE: `${route}:${failure}` } + delete env.NODE_TEST_CONTEXT + const result = spawnSync( + process.execPath, + ['--experimental-test-module-mocks', '--import', setup, '--test', fixture], + { + env, + encoding: 'utf8', + timeout: 15_000, + }, + ) + assert.equal(result.status, 0, result.stdout + result.stderr) + assert.match(result.stdout, /# tests 1\b/, 'the isolated fixture must execute its assertion') + }) + } +} From bf9f508dc0f7bb7232a6ed0fd1a0bcdf6e85a5f1 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:04:34 +0800 Subject: [PATCH 067/473] feat(k8s): enforce shared admission across inference protocols --- src/lib/protocol/handle-protocol.mjs | 161 +++++++++++++-------- src/server.mjs | 12 +- test/e2e/kubernetes-admission.e2e.test.mjs | 70 +++++++++ test/unit/backup-service.test.mjs | 16 +- test/unit/database.test.mjs | 25 ++-- 5 files changed, 204 insertions(+), 80 deletions(-) create mode 100644 test/e2e/kubernetes-admission.e2e.test.mjs diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 04978346e..9c7404e79 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -199,7 +199,10 @@ export function createHandleProtocol(deps) { return disjointOpenaiUsage(next) } - async function attachPricing(logBag, { managedKey = null, rateMultiplier = 1, bill = false } = {}) { + async function attachPricing( + logBag, + { managedKey = null, rateMultiplier = 1, bill = false, reservation = null } = {}, + ) { const model = logBag.upstream_model || logBag.model const priced = costColumnsFromUsage(logBag.usage || {}, model) Object.assign(logBag, priced) @@ -209,8 +212,16 @@ export function createHandleProtocol(deps) { model, rateMultiplier, officialCost: priced.total_cost, + reservation, }) - } catch {} + } catch (error) { + if (reservation?.shared) { + // The request executed but its quota settlement is uncertain. Preserve the + // durable reservation until recovery can establish the result. + reservation.billingFailed = true + throw error + } + } } function finishClientCancel(res, result, logBag) { @@ -466,6 +477,42 @@ export function createHandleProtocol(deps) { } async function handleProtocol(req, res, protocol, pathName) { + const admission = { lease: null } + try { + return await handleProtocolRequest(req, res, protocol, pathName, admission) + } catch (error) { + if (admission.executionStarted && admission.lease?.shared && !admission.lease.settled) { + admission.lease.billingFailed = true + } + throw error + } finally { + if (!admission.lease?.billingFailed) await admission.lease?.release?.() + } + } + + async function reserveManagedKey(req, res) { + if (!req.apiKeyRecord) return { ok: true } + const gate = await apiKeyStore.reserve(req.apiKeyRecord) + if (!gate.ok) { + stats.errors++ + json( + res, + gate.status, + makeError({ + type: gate.status === 429 ? ErrorType.RATE_LIMIT : ErrorType.PERMISSION, + code: gate.code, + message: gate.message, + status: gate.status, + details: gate.detail || undefined, + }).body, + ) + } else if (gate.record) { + req.apiKeyRecord = gate.record + } + return gate + } + + async function handleProtocolRequest(req, res, protocol, pathName, admission) { const logCtx = requestLog.start(req, { protocol, pathName }) res._kinRequestId = logCtx.request_id // Registers its own 'finish' listener first so status/headers are final when finish() runs. @@ -738,6 +785,20 @@ export function createHandleProtocol(deps) { return } if (platform.platform === 'openai') { + const gate = await reserveManagedKey(req, res) + if (!gate.ok) return + admission.lease = gate + const runCodex = async (options) => { + admission.executionStarted = true + const output = await handleCodexProtocol(options) + await attachPricing(logBag, { + managedKey: req.apiKeyRecord, + reservation: gate, + rateMultiplier: await deps.groupsRepo.rateMultiplier(req.apiKeyRecord?.group_id ?? 1), + bill: !!req.apiKeyRecord && logBag.final_state === 'verified', + }) + return output + } const routing = getRouting() || {} const codexSticky = { requestStartedAt: logCtx.t0, @@ -760,7 +821,7 @@ export function createHandleProtocol(deps) { }) if (protocol === 'anthropic.messages') { const codex = normalizeCodexRouting(routing.codex) - return await handleCodexProtocol({ + return await runCodex({ req, res, protocol, @@ -788,7 +849,7 @@ export function createHandleProtocol(deps) { ...codexSticky, }) } - return await handleCodexProtocol({ + return await runCodex({ req, res, protocol, @@ -1028,24 +1089,10 @@ export function createHandleProtocol(deps) { }) } if (inferenceBackend === 'api') { + const gate = await reserveManagedKey(req, res) + if (!gate.ok) return + admission.lease = gate const managedKey = req.apiKeyRecord || null - if (managedKey) { - const gate = await apiKeyStore.acquire(managedKey) - if (!gate.ok) { - stats.errors++ - return json( - res, - gate.status, - makeError({ - type: gate.status === 429 ? ErrorType.RATE_LIMIT : ErrorType.PERMISSION, - code: gate.code, - message: gate.message, - status: gate.status, - details: gate.detail || undefined, - }).body, - ) - } - } const clientAbort = bindClientAbort(req, res) const clientStream = isClientStream(inbound, req.headers) const requestedDelivery = String( @@ -1057,6 +1104,7 @@ export function createHandleProtocol(deps) { logBag.via = 'api-kernel' let result try { + admission.executionStarted = true result = await runApiInference({ req, res, @@ -1088,11 +1136,6 @@ export function createHandleProtocol(deps) { }) } finally { clientAbort.settle() - if (managedKey) { - try { - apiKeyStore.release(managedKey) - } catch {} - } } // The api backend forwards the converted body untouched: its session is the outbound one. logBag.outbound_session_id = sessionIdForLog(sessionIdFromOutboundBody(ctx.body) || logBag.session_id) @@ -1108,6 +1151,7 @@ export function createHandleProtocol(deps) { logBag.final_state = result?.terminalState || null await attachPricing(logBag, { managedKey, + reservation: admission.lease, rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), bill: !!(result?.ok && managedKey), }) @@ -1145,24 +1189,10 @@ export function createHandleProtocol(deps) { // anything still waiting past this writes SSE so Cloudflare does not 524. const queueKeepaliveDelay = queueKeepaliveDelayMs(getRouting(), cfg.limits?.queue_keepalive_delay_ms) const streamIdleTimeoutMs = resolveStreamIdleTimeoutMs(getRouting(), cfg) + const gate = await reserveManagedKey(req, res) + if (!gate.ok) return + admission.lease = gate const managedKey = req.apiKeyRecord || null - if (managedKey) { - const gate = await apiKeyStore.acquire(managedKey) - if (!gate.ok) { - stats.errors++ - return json( - res, - gate.status, - makeError({ - type: gate.status === 429 ? ErrorType.RATE_LIMIT : ErrorType.PERMISSION, - code: gate.code, - message: gate.message, - status: gate.status, - details: gate.detail || undefined, - }).body, - ) - } - } const clientAbort = bindClientAbort(req, res) @@ -1216,6 +1246,7 @@ export function createHandleProtocol(deps) { } let result try { + admission.executionStarted = true result = await getFailoverRunner().run({ requestId: logCtx.request_id, canonicalBody, @@ -1569,11 +1600,6 @@ export function createHandleProtocol(deps) { } finally { anthropicKeepalive?.stop() clientAbort.settle() - if (managedKey) { - try { - apiKeyStore.release(managedKey) - } catch {} - } } if (result?.responseSent) return @@ -1649,6 +1675,7 @@ export function createHandleProtocol(deps) { } catch {} await attachPricing(logBag, { managedKey, + reservation: admission.lease, rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), bill: !!managedKey, }) @@ -1792,17 +1819,33 @@ export function createHandleProtocol(deps) { } logBag.model = typeof body?.model === 'string' ? body.model : null logBag.inbound_summary = summarizeBody(body) - return await handleCodexSearch({ - req, - res, - body, - logBag, - stats, - json, - routing: getRouting() || {}, - projectRoot: cfg.paths.project, - stickyRouter, - }) + const gate = await reserveManagedKey(req, res) + if (!gate.ok) return + try { + const output = await handleCodexSearch({ + req, + res, + body, + logBag, + stats, + json, + routing: getRouting() || {}, + projectRoot: cfg.paths.project, + stickyRouter, + }) + await attachPricing(logBag, { + managedKey: req.apiKeyRecord, + reservation: gate, + rateMultiplier: await deps.groupsRepo.rateMultiplier(req.apiKeyRecord?.group_id ?? 1), + bill: !!req.apiKeyRecord && logBag.final_state === 'verified', + }) + return output + } catch (error) { + if (gate.shared && !gate.settled) gate.billingFailed = true + throw error + } finally { + if (!gate.billingFailed) await gate.release?.() + } } return { diff --git a/src/server.mjs b/src/server.mjs index be3aa18ea..23850e17d 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -20,7 +20,7 @@ import { attachPanelAuth, getPanelAdmin, } from './lib/core/security.mjs' -import { loadModelPolicy } from './lib/protocol/model-policy.mjs' +import { loadModelPolicy, refreshModelPolicy } from './lib/protocol/model-policy.mjs' import { gatewayModelCatalog } from './lib/protocol/models.mjs' import { createCredentialRefreshMonitor, @@ -34,7 +34,7 @@ import { ensureTelemetrySidecar } from './lib/vm/slot-process-status.mjs' import { createSlotBridgeHealer, powerOnSlot, reviveDownSlots } from './lib/vm/slot-recovery.mjs' import { createUsageProbeMonitor, normalizeUsageProbeConfig } from './lib/oauth/usage-probe-monitor.mjs' import { createCatalogMonitor } from './lib/admin/catalog-monitor.mjs' -import { pausePriceCatalog, resumePriceCatalog } from './lib/admin/price-catalog.mjs' +import { pausePriceCatalog, resumePriceCatalog, invalidatePriceCatalog } from './lib/admin/price-catalog.mjs' import { normalizeOfficialCcConfig } from './lib/oauth/official-cc-bootstrap.mjs' import { invalidateLiveCredentialCache } from './lib/admin/panel-live-credentials.mjs' import { normalizeHealthProbeConfig, createHealthProbeMonitor, HEALTH_REAL_HEADER } from './lib/admin/health-probe.mjs' @@ -75,7 +75,7 @@ import { openDatabase, closeDatabase } from './lib/db/database.mjs' import { runLegacyImport } from './lib/db/legacy-import.mjs' import { initVmDbSync, stopVmWatch, flushVmDbSync, pauseVmDbSync, resumeVmDbSync } from './lib/vm/vm-db-sync.mjs' import { flushPendingWrites } from './lib/db/pending-writes.mjs' -import { loadGptModelPolicy } from './lib/protocol/gpt-model-policy.mjs' +import { loadGptModelPolicy, refreshGptModelPolicy } from './lib/protocol/gpt-model-policy.mjs' import { BackupService } from './lib/admin/backup-service.mjs' import { ClusterNodesRepo } from './lib/db/repos/cluster-nodes-repo.mjs' import { ClusterManager } from './lib/cluster/cluster-manager.mjs' @@ -812,6 +812,12 @@ async function requireAuth(req, res) { }), ) } + if (kubernetesMode) { + const snapshot = await new SettingsRepo().refresh() + refreshModelPolicy(snapshot.model_policy) + refreshGptModelPolicy(snapshot.gpt_model_policy) + invalidatePriceCatalog() + } // Panel session const session = await verifyPanelSession(token) if (session) { diff --git a/test/e2e/kubernetes-admission.e2e.test.mjs b/test/e2e/kubernetes-admission.e2e.test.mjs new file mode 100644 index 000000000..b92f35b37 --- /dev/null +++ b/test/e2e/kubernetes-admission.e2e.test.mjs @@ -0,0 +1,70 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { startGateway, api } from '../harness.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' + +const env = { VM2API_CONTROL_PLANE_MODE: 'kubernetes', KIN_BACKUP_DISABLED: '1' } +const model = 'claude-haiku-4-5-20251001' + +test('Kubernetes HTTP inference atomically reserves the last request quota and releases after settlement', async () => { + const gw = await startGateway({ env, mockText: 'one-global-request' }) + try { + const created = await api(gw, 'POST', '/api/panel/api-keys', { + body: { name: 'shared-quota', max_concurrency: 0, quota_requests: 1, rpm: 0 }, + }) + assert.equal(created.status, 201, created.text) + const key = created.json.item + const results = await Promise.all( + Array.from({ length: 12 }, () => + api(gw, 'POST', '/v1/messages', { + headers: { authorization: `Bearer ${key.key}` }, + body: { model, max_tokens: 8, stream: true, messages: [{ role: 'user', content: 'one request' }] }, + }), + ), + ) + assert.equal(results.filter((r) => r.status === 200).length, 1, results.map((r) => r.text).join('\n')) + assert.ok(results.filter((r) => r.status !== 200).every((r) => r.status === 429)) + const saved = await gw.db.prepare('SELECT requests, quota_requests_used FROM api_keys WHERE id = $1').get(key.id) + assert.deepEqual(saved, { requests: 1, quota_requests_used: 1 }) + let remaining + const deadline = Date.now() + 2000 + do { + remaining = ( + await gw.db + .prepare('SELECT count(*) AS n FROM api_key_admissions WHERE key_id = $1 AND released_at IS NULL') + .get(key.id) + ).n + if (!remaining) break + await new Promise((resolve) => setTimeout(resolve, 10)) + } while (Date.now() < deadline) + assert.equal(remaining, 0) + const rows = await gw.db + .prepare('SELECT settled_at, released_at FROM api_key_admissions WHERE key_id = $1') + .all(key.id) + assert.equal(rows.length, 1) + assert.ok(rows[0].settled_at <= rows[0].released_at) + } finally { + await gw.stop() + } +}) + +test('Kubernetes request authentication reloads a model policy changed by another process connection', async () => { + const gw = await startGateway({ env }) + try { + const settings = new SettingsRepo(gw.db) + const policy = await settings.get('model_policy') + assert.ok(policy.models[model]) + policy.models[model].enabled = false + await settings.set('model_policy', policy) + const before = await gw.db.prepare("SELECT value, updated_at FROM settings WHERE key = 'model_policy'").get() + const response = await api(gw, 'GET', '/api/panel/model-policy') + assert.equal(response.status, 200, response.text) + assert.equal(response.json.data.policy.models[model].enabled, false) + assert.deepEqual( + await gw.db.prepare("SELECT value, updated_at FROM settings WHERE key = 'model_policy'").get(), + before, + ) + } finally { + await gw.stop() + } +}) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 5b5b8132c..7ed1de750 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -76,9 +76,11 @@ test('restoring the application schema preserves later writes in a neighboring s } }) -test('restoring a PG 001 snapshot removes later tables before reapplying migration 002', async () => { +test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() - await db.exec("DROP TABLE panel_sessions; DELETE FROM schema_migrations WHERE version='002'") + await db.exec( + "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets; DELETE FROM schema_migrations WHERE version <> '001'", + ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() await applyMigrations(db) @@ -88,13 +90,17 @@ test('restoring a PG 001 snapshot removes later tables before reapplying migrati await new SettingsRepo(db).set('older-snapshot', 'after') assert.equal((await svc.restoreBackup(rec.id)).ok, true) assert.equal(await new SettingsRepo(db).get('older-snapshot'), 'before') - assert.equal((await db.prepare('SELECT count(*) AS n FROM panel_sessions').get()).n, 0) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 2) + for (const name of ['panel_sessions', 'api_key_admissions', 'gateway_rate_buckets']) { + assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) + } + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 4) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() - await db.exec("DROP TABLE panel_sessions; DELETE FROM schema_migrations WHERE version='002'") + await db.exec( + "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets; DELETE FROM schema_migrations WHERE version <> '001'", + ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() const archive = await svc._extractArchive(rec) diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index af74078be..ac062cb8c 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -1,6 +1,7 @@ import { test, after } from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' +import path from 'node:path' import crypto from 'node:crypto' import { createDatabase, @@ -34,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 31) + assert.equal(rows.length, 33) for (const name of [ 'settings', 'users', @@ -45,6 +46,8 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'accounts_legacy', 'panel_users_legacy', 'panel_sessions', + 'api_key_admissions', + 'gateway_rate_buckets', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -105,27 +108,23 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 2) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 4) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) test('failed migrations rollback DDL and migration records', async () => { const dir = fs.mkdtempSync('/tmp/vm2api-pg-migrations-') - fs.writeFileSync(dir + '/003_bad.sql', 'CREATE TABLE partial_write(id text); INVALID SQL;') + fs.writeFileSync(dir + '/999_bad.sql', 'CREATE TABLE partial_write(id text); INVALID SQL;') const db = await database() await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /unknown PostgreSQL migration/) - fs.copyFileSync( - new URL('../../src/lib/db/postgres-migrations/001_baseline.sql', import.meta.url), - dir + '/001_baseline.sql', - ) - fs.copyFileSync( - new URL('../../src/lib/db/postgres-migrations/002_control_plane_sessions.sql', import.meta.url), - dir + '/002_control_plane_sessions.sql', - ) - await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /003_bad.sql failed/) + const migrations = new URL('../../src/lib/db/postgres-migrations/', import.meta.url) + for (const file of fs.readdirSync(migrations).filter((name) => name.endsWith('.sql'))) { + fs.copyFileSync(new URL(file, migrations), path.join(dir, file)) + } + await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 2) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 4) fs.rmSync(dir, { recursive: true, force: true }) }) From f4ff7004530791b877cc7ac27971596628e42d13 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:12:18 +0800 Subject: [PATCH 068/473] test(protocol): adapt admission leases and shared endpoint mocks --- test/unit/api-backend.test.mjs | 36 ++++++++++++++++++++--------- test/unit/auto-mode-api.test.mjs | 3 ++- test/unit/claude-code-only.test.mjs | 5 ++-- test/unit/distill-detect.test.mjs | 2 +- test/unit/thinking-owner.test.mjs | 2 +- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/test/unit/api-backend.test.mjs b/test/unit/api-backend.test.mjs index bec7682b0..52cc0c3d4 100644 --- a/test/unit/api-backend.test.mjs +++ b/test/unit/api-backend.test.mjs @@ -98,8 +98,9 @@ test('API backend applies the global official_full persona setting', async () => requestLog: { start: () => ({ request_id: 'api-persona-test' }), finish() {} }, stickyRouter: {}, accountQuota: {}, - apiKeyStore: { acquire: () => ({ ok: true }), release() {} }, + apiKeyStore: { reserve: async (record) => ({ ok: true, record, release: async () => {} }) }, apiScheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'ep-test', kind: 'claude', protocol: 'anthropic', base_url: 'https://upstream.invalid' }, @@ -107,7 +108,7 @@ test('API backend applies the global official_full persona setting', async () => key: { id: 'key-test', api_key: 'secret' }, }), }, - apiEndpointStore: {}, + apiEndpointStore: { listRaw: async () => [] }, stats, routingConfigPath: routingFile, routingConfig: { @@ -487,8 +488,7 @@ async function runManagedKeyHop(root, result) { stickyRouter: { extractPoolKey: () => null, collectPoolKeys: () => [] }, accountQuota: { ingestHeaders() {} }, apiKeyStore: { - acquire: () => ({ ok: true }), - release() {}, + reserve: async (record) => ({ ok: true, record, release: async () => {} }), recordUsage: (key, usage, opts) => charges.push({ key, usage, opts }), }, apiScheduler: {}, @@ -684,12 +684,15 @@ function protocolHarness({ body, stickyRouter, failoverRunner, apiKeyRecord = nu stickyRouter, accountQuota: {}, apiKeyStore: { - acquire: () => { + reserve: async (record) => { apiKeyCalls.acquire += 1 - return { ok: true } - }, - release: () => { - apiKeyCalls.release += 1 + return { + ok: true, + record, + release: async () => { + apiKeyCalls.release += 1 + }, + } }, }, apiScheduler: {}, @@ -1121,6 +1124,7 @@ test('Responses API backend keeps parallel done-only tools in one stream state', } try { const result = await runApiInference({ + store: { listRaw: async () => [] }, cfg: { paths: { data: root } }, res, protocol: 'openai.chat', @@ -1134,6 +1138,7 @@ test('Responses API backend keeps parallel done-only tools in one stream state', }, }, scheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'ep', kind: 'openai', protocol: 'openai', base_url: 'https://fixture.invalid' }, @@ -1178,6 +1183,7 @@ test('API Anthropic non-stream rejects message_start without message_stop', asyn await new Promise((resolve) => kernel.listen(socket, resolve)) try { const result = await runApiInference({ + store: { listRaw: async () => [] }, cfg: { paths: { data: root } }, res: { headersSent: false, write: () => {} }, protocol: 'anthropic.messages', @@ -1187,6 +1193,7 @@ test('API Anthropic non-stream rejects message_start without message_stop', asyn inbound: {}, converters: { createClaudeMessageAssembler, applyClaudeSSELineToMessage }, scheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'ep', kind: 'claude', protocol: 'anthropic', base_url: 'https://fixture.invalid' }, @@ -1234,6 +1241,7 @@ test('API Anthropic non-stream rejects stop_reason and text without message_stop await new Promise((resolve) => kernel.listen(socket, resolve)) try { const result = await runApiInference({ + store: { listRaw: async () => [] }, cfg: { paths: { data: root } }, res: { headersSent: false, write: () => {} }, protocol: 'anthropic.messages', @@ -1243,6 +1251,7 @@ test('API Anthropic non-stream rejects stop_reason and text without message_stop inbound: {}, converters: { createClaudeMessageAssembler, applyClaudeSSELineToMessage }, scheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'ep', kind: 'claude', protocol: 'anthropic', base_url: 'https://fixture.invalid' }, @@ -1276,6 +1285,7 @@ test('API Anthropic non-stream accepts message_stop as terminal', async () => { await new Promise((resolve) => kernel.listen(socket, resolve)) try { const result = await runApiInference({ + store: { listRaw: async () => [] }, cfg: { paths: { data: root } }, res: { headersSent: false, write: () => {} }, protocol: 'anthropic.messages', @@ -1285,6 +1295,7 @@ test('API Anthropic non-stream accepts message_stop as terminal', async () => { inbound: {}, converters: { createClaudeMessageAssembler, applyClaudeSSELineToMessage }, scheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'ep', kind: 'claude', protocol: 'anthropic', base_url: 'https://fixture.invalid' }, @@ -1343,8 +1354,9 @@ test('API backend non-stream maps incomplete assistant hops to incomplete_respon requestLog: { start: () => ({ request_id: 'api-incomplete-test' }), finish() {} }, stickyRouter: {}, accountQuota: {}, - apiKeyStore: { acquire: () => ({ ok: true }), release() {} }, + apiKeyStore: { reserve: async (record) => ({ ok: true, record, release: async () => {} }) }, apiScheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'ep-test', kind: 'claude', protocol: 'anthropic', base_url: 'https://upstream.invalid' }, @@ -1352,7 +1364,7 @@ test('API backend non-stream maps incomplete assistant hops to incomplete_respon key: { id: 'key-test', api_key: 'secret' }, }), }, - apiEndpointStore: {}, + apiEndpointStore: { listRaw: async () => [] }, stats: { errors: 0, requests: 0, by_route: {}, passthrough: 0, rewrite: 0, convert: 0 }, routingConfigPath: routingFile, routingConfig: { compatibility: { persona_preset: 'zero' }, failover: {} }, @@ -1401,6 +1413,7 @@ test('Responses API backend returns 502 for conflicting tool arguments when the await new Promise((resolve) => kernel.listen(socket, resolve)) try { const result = await runApiInference({ + store: { listRaw: async () => [] }, cfg: { paths: { data: root } }, res: { headersSent: false, write: () => {} }, protocol: 'anthropic.messages', @@ -1410,6 +1423,7 @@ test('Responses API backend returns 502 for conflicting tool arguments when the inbound: {}, converters: { writeSSEHeaders: () => {} }, scheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'ep', kind: 'openai', protocol: 'openai', base_url: 'https://fixture.invalid' }, diff --git a/test/unit/auto-mode-api.test.mjs b/test/unit/auto-mode-api.test.mjs index df52c64d2..eaeb4f25d 100644 --- a/test/unit/auto-mode-api.test.mjs +++ b/test/unit/auto-mode-api.test.mjs @@ -79,6 +79,7 @@ test('API classifier backend preserves contract, rejects model substitution and runApiInference({ cfg: { paths: { data: temp } }, scheduler: { + reload() {}, pick: () => ({ ok: true, endpoint: { id: 'fixture', kind: 'claude', protocol: 'anthropic', base_url: 'https://fixture.invalid' }, @@ -86,7 +87,7 @@ test('API classifier backend preserves contract, rejects model substitution and key: { id: 'fixture', api_key: 'fixture-key' }, }), }, - store: {}, + store: { listRaw: async () => [] }, inbound: input, convertedBody: input, clientStream: false, diff --git a/test/unit/claude-code-only.test.mjs b/test/unit/claude-code-only.test.mjs index 5f7acabe3..8ac8766a3 100644 --- a/test/unit/claude-code-only.test.mjs +++ b/test/unit/claude-code-only.test.mjs @@ -190,8 +190,9 @@ function protocolHarness({ routing, body, healthMonitor }) { requestLog: { start: () => ({ request_id: 'cc-only-test' }), finish() {} }, stickyRouter: {}, accountQuota: {}, - apiKeyStore: { acquire: () => ({ ok: true }), release() {} }, + apiKeyStore: { reserve: async (record) => ({ ok: true, record, release: async () => {} }) }, apiScheduler: { + reload() {}, pick: () => { throw REACHED }, @@ -201,7 +202,7 @@ function protocolHarness({ routing, body, healthMonitor }) { throw REACHED }, }, - apiEndpointStore: {}, + apiEndpointStore: { listRaw: async () => [] }, stats, getRoutingConfig: () => routing, groupsRepo: { rateMultiplier: () => 1 }, diff --git a/test/unit/distill-detect.test.mjs b/test/unit/distill-detect.test.mjs index ee873a0c7..7f5d23066 100644 --- a/test/unit/distill-detect.test.mjs +++ b/test/unit/distill-detect.test.mjs @@ -608,7 +608,7 @@ test('normalize keeps default fingerprints when omitted', () => { test('handleProtocol intercepts before credential hop', () => { const src = fs.readFileSync(path.join(root, 'src/lib/protocol/handle-protocol.mjs'), 'utf8') const early = src.indexOf('await applyProtocolIntercept(') - const codex = src.indexOf('handleCodexProtocol({') + const codex = src.indexOf('await handleCodexProtocol(options)') assert.ok(early > 0 && early < codex) const before = src.indexOf("applyIntercept(cfg.intercept.rules, 'before_upstream'") const late = src.indexOf('await applyProtocolIntercept(', before) diff --git a/test/unit/thinking-owner.test.mjs b/test/unit/thinking-owner.test.mjs index 3384c2966..b915aaa79 100644 --- a/test/unit/thinking-owner.test.mjs +++ b/test/unit/thinking-owner.test.mjs @@ -127,7 +127,7 @@ test('a handler records the served signature and the next account replays it as requestLog: { start: () => ({ request_id: 'thinking-owner', mode: 'normal' }), finish() {} }, stickyRouter: { extractPoolKey: () => null, collectPoolKeys: () => [] }, accountQuota: { ingestHeaders() {} }, - apiKeyStore: { acquire: () => ({ ok: true }), release() {} }, + apiKeyStore: { reserve: async (record) => ({ ok: true, record, release: async () => {} }) }, apiScheduler: {}, apiEndpointStore: {}, stats: { errors: 0, requests: 0, by_route: {}, passthrough: 0, rewrite: 0, convert: 0 }, From 5abd21d95da3eb18f0b9a2b06b179ff96983f744 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:20:35 +0800 Subject: [PATCH 069/473] test(k8s): isolate shell readiness from host containers --- test/unit/slot-shell-rc.test.mjs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/unit/slot-shell-rc.test.mjs b/test/unit/slot-shell-rc.test.mjs index ece0584fa..a2dc42ff4 100644 --- a/test/unit/slot-shell-rc.test.mjs +++ b/test/unit/slot-shell-rc.test.mjs @@ -13,6 +13,7 @@ import { syncShellLoginCredentials, } from '../../src/lib/vm/slot-shell.mjs' import { FULL_OAUTH_SCOPE } from '../../src/lib/oauth/oauth-contract.mjs' +import { assertAuthLoginReady } from '../../src/lib/oauth/claude-auth-login.mjs' import { OFFICIAL_CLI_VERSION } from '../../src/lib/identity/vm-identity.mjs' import { ensureOfficialCredentialLink, @@ -449,7 +450,18 @@ test('official login tickets check a cluster slot for its container cli-node', a fs.mkdirSync(path.join(root, 'vms'), { recursive: true }) fs.writeFileSync(path.join(root, 'vms', 'vm-01.json'), JSON.stringify({ id: 'vm-01', node_id: 'node-a' })) fs.writeFileSync(path.join(root, 'vms', 'vm-02.json'), JSON.stringify({ id: 'vm-02' })) - const shell = createSlotShell({ projectRoot: root, logger: { warn() {} } }) + const inspected = [] + const shell = createSlotShell({ + projectRoot: root, + logger: { warn() {} }, + authLoginReady: (projectRoot, vm) => + assertAuthLoginReady(projectRoot, vm, { + inspect: (name) => { + inspected.push(name) + return null + }, + }), + }) try { const remote = await shell.issueTicket('vm-01', { login: true }) assert.equal(remote.ok, false) @@ -457,6 +469,7 @@ test('official login tickets check a cluster slot for its container cli-node', a const missing = await shell.issueTicket('vm-02', { login: true }) assert.equal(missing.ok, false) assert.equal(missing.error.code, 'container_not_ready') + assert.ok(inspected.includes('kin-02')) const ops = await shell.issueTicket('vm-02') assert.equal(ops.ok, true) } finally { From 919c4e5ede42d1812791ce0e27f4a7b54f043db8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:21:53 +0800 Subject: [PATCH 070/473] docs(k8s): record shared-state progress and verification limits --- .../2026-10-21-kubernetes-control-plane.md | 23 ++++ ...6-10-21-kubernetes-shared-state-results.md | 42 ++++++ .../kubernetes/shared-state/baseline-e2e.json | 126 ++++++++++++++++++ .../shared-state/baseline-unit.json | 56 ++++++++ .../kubernetes/shared-state/verification.json | 55 ++++++++ 5 files changed, 302 insertions(+) create mode 100644 docs/plans/2026-10-21-kubernetes-shared-state-results.md create mode 100644 docs/plans/evidence/kubernetes/shared-state/baseline-e2e.json create mode 100644 docs/plans/evidence/kubernetes/shared-state/baseline-unit.json create mode 100644 docs/plans/evidence/kubernetes/shared-state/verification.json diff --git a/docs/plans/2026-10-21-kubernetes-control-plane.md b/docs/plans/2026-10-21-kubernetes-control-plane.md index f13abec2e..1d998261e 100644 --- a/docs/plans/2026-10-21-kubernetes-control-plane.md +++ b/docs/plans/2026-10-21-kubernetes-control-plane.md @@ -34,3 +34,26 @@ ## 后续多副本验收 后续必须使用至少两个不同节点上的活动副本，验证全局准入/限流/座位、配置同步、认证和一次性票据、原子计数、任务唯一执行、全局维护，以及故障后的 fencing 和安全接管。不同副本须同时处理重叠推理请求；不能以将全部推理转发给 leader 代替水平扩容。 + +## 多副本实施增量（进行中） + +#648 的完成条件仍包含两个不同 Kubernetes 节点上的活动副本同时处理请求，以及扩缩容、滚动更新和异常退出。单活动部署仅是已完成的第一阶段。本轮继续实现跨副本共享状态，没有将目标缩减为只支持 API endpoint。 + +在 `bf9f508d` 上已实现的增量： + +- `api_key_admissions` 通过 PG 事务、密钥锁和用户锁原子协调密钥/用户并发、密钥 RPM 和请求数额度。请求计数与 pending 额度结算同事务；释放幂等且保留 RPM 历史。删除执行中的密钥仍保留用户并发占位。 +- HTTP、SSE、Codex Responses/Search 与 WebSocket 每个推理请求均使用同一预留接口。WebSocket 握手不预留推理容量。计费完成前保留占位；执行后异常或计费结果不明时保留持久占位。 +- 网关 token bucket 存 PG，使用取锁后的数据库时间。存储键为摘要；PG 故障不会回退到各进程分别限流。 +- 每次 API endpoint 调度及模型目录读取前刷新 PG 数据。自动 cooldown 只延长截止时间；管理员显式清除继续可用。 +- 设置快照显式刷新，防止迟到查询覆盖较新的本地提交或已发布快照。Kubernetes 认证前刷新设置，并同步更新 Claude/GPT 模型及价格缓存；读取路径不执行策略迁移或默认值回写。 + +当前没有开放多活动控制面，独占锁与 `replicas: 1` 保留。新增的共享预留没有 TTL，不会把仍可能执行中的容量自动重新分配。进程死亡后的自动回收尚需执行端 fencing 和可确认终止的协议。 + +接下来的必要工作： + +1. 将账号/VM 并发、账号 RPM、设备座位、同会话顺序、Sticky 映射及运行态变更纳入共享原子协调；控制整行快照写入冲突。 +2. 明确远端执行请求和控制面 owner/epoch 的绑定，覆盖 TLS gate、SSH fallback、UDS、Codex/Search 与 VM 生命周期命令。确认旧执行终止后才允许回收占位。 +3. 协调后台任务、全局维护、备份恢复和凭据刷新；完成 routing/VM/凭据文件的共享存储与更新协议。减少每请求全量设置读取的成本。 +4. 全部条件具备后再允许两个活动副本，补充 HPA、PodDisruptionBudget 和多节点部署，并验证并发、滚动更新、缩容与故障接管。 + +本轮运行证据及未验证项将在增量验证报告中记录。第一阶段的镜像 digest 与 k3s 结果只覆盖当时源码，不能用于证明本轮新代码已部署。 diff --git a/docs/plans/2026-10-21-kubernetes-shared-state-results.md b/docs/plans/2026-10-21-kubernetes-shared-state-results.md new file mode 100644 index 000000000..fc0c550a4 --- /dev/null +++ b/docs/plans/2026-10-21-kubernetes-shared-state-results.md @@ -0,0 +1,42 @@ +# Kubernetes 共享状态增量验证（#648，进行中） + +日期：2026-10-21。实现提交：`bf9f508dc0f7bb7232a6ed0fd1a0bcdf6e85a5f1`。后续 `f4ff7004530791b877cc7ac27971596628e42d13` 只适配既有测试替身。分支：`codex/kubernetes-control-plane`。关联 Draft PR #651，依赖 #649。 + +本轮继续实现 #648 的共享准入和一致性要求。不同节点上的多个活动控制面仍未开放，Issue 不满足完成条件。`replicas: 1`、Recreate 和独占锁继续保留。第一阶段的镜像及 k3s 证据不覆盖本轮新增源码，本轮没有部署现有服务。 + +## 实现与回归 + +- PG 原子协调 managed API key/user 并发、key RPM、请求数额度和 pending reservation。计费计数与 pending 结算同事务；重复结算和释放幂等。删除执行中 key 不删除其 user 并发占位。授权字段或凭据在认证后变化时拒绝旧请求。 +- HTTP/SSE、Claude API/OAuth、Codex Responses/Search 和 WebSocket 的每个推理请求都使用 reservation。握手不占推理容量，同 WebSocket 保持原有串行队列。跨两条 WebSocket 的同 key 并发通过同一准入接口检查。 +- 执行后计费倍率读取、结算或后处理异常时保留持久占位。正常响应处理先结算再释放。该行为不等于已实现执行端取消确认；真实 kernel 的异常退出和取消仍需后续 fencing。 +- PG 网关 token bucket 使用取锁后的数据库时间，并保持更新时间单调。存储键为摘要，数据库失败不回退内存限流。此时 HTTP 当前表现为服务器错误，不能表述为 429 或 503。 +- API endpoint 每次调度及 `/v1/models` 读取前刷新 PG。自动 cooldown 原子保留较晚截止时间；管理员仍可显式清除。 +- settings 刷新防止本地乱序查询覆盖新提交。Kubernetes 认证前从共享设置刷新模型和价格缓存，策略缓存刷新不写入默认值、不触发启动迁移。 + +失败测试先独立提交。关键复现包括：两个 limiter 原先合计放行 6 次，配置容量为 3；Codex 原先绕过 reserve，实际进入后端并返回 503，预期应在准入处返回 429；执行后倍率读取失败曾释放未计费占位，现改为保留 pending，下一请求继续被额度限制。 + +## 环境与结果 + +使用专用 PostgreSQL 17.11 测试容器、独立随机 schema、Node.js 22.23.2 和合成上游/请求。未访问现有业务数据库或真实供应商。PG 备份客户端版本为 17.11。 + +- `bf9f508d`：迁移/备份 31/31 通过。旧 001 快照恢复后补建 002–004，新增会话、准入和 token bucket 表无残留；跨 schema 保护继续通过。 +- `bf9f508d`：实际 Node 控制面 HTTP/SSE、managed key、会话/探针/终止及新增共享额度/设置 E2E 共 14/14 通过。12 个并发 SSE 请求争用最后一次额度时只有 1 个成功，结束后 request/quota 计数各为 1，无活动 reservation。 +- `bf9f508d`：全量 E2E 28 文件，103 pass、17 fail、1 skip。17 个失败断言逐摘要与原始 `eee8f6e` 相同。 +- 首轮全量 unit 在 `bf9f508d` 上为 3367 pass、31 fail、10 skip。24 个新增失败来自旧测试替身缺少 reserve/listRaw/reload 或静态检查仍定位旧调用字符串；`f4ff7004` 已适配，93/93 定向检查通过。未修改行为预期或原始基线断言。 +- `f4ff7004`：完整 unit 复验为 291 文件、3390 pass、8 fail、10 skip。7 项与原始基线逐摘要相同。另 1 项终端测试实际返回 `cli_node_missing`、预期 `container_not_ready`；该旧测试读取了宿主机运行中的 `kin-02`。`5abd21d9` 使用已有 inspect 注入接口提供合成状态，保持原断言，19/19 定向测试通过。此后没有再次运行全量 unit，不能声称全量只剩 7 项失败。 +- 本轮修改的 31 个 MJS 文件通过 Node 语法检查，Biome 检查通过，`git diff --check` 通过。 + +脱敏结果见 [验证元数据](evidence/kubernetes/shared-state/verification.json)、[unit 基线断言](evidence/kubernetes/shared-state/baseline-unit.json) 和 [E2E 基线断言](evidence/kubernetes/shared-state/baseline-e2e.json)。 + +基线断言的具体差异仍然存在。例如 unit 的实际 TTL 为 `1h`、预期 `undefined`；既有 Responses E2E 实际为 400 `protocol_not_allowed`、预期 200。不能将全量回归标记为全部通过。断言比较按原 runner 的逐文件 PG 预加载规则执行，移除空行和调用栈后计算摘要。 + +## 尚未实现或验证 + +- 账号/VM 并发及账号 RPM、设备座位、Sticky generation、同会话跨副本顺序。 +- AccountQuota、ProxyPool、SchedulingRuntime 的字段级原子写入及旧快照冲突控制。 +- routing/VM/凭据文件归属和传播、后台任务唯一执行、全局备份/恢复维护。 +- 执行端 owner/epoch、匹配任务的取消/终止证明、占位安全接管。当前 reservation 没有 TTL；控制面死亡后占位保留，不能据此声明自动恢复。 +- settings 的轻量修订与增量刷新。当前每次认证读取完整设置，可能传输并复制 models.dev 快照；本轮未做性能测量。 +- 两节点活动副本同时推理、HPA、PodDisruptionBudget、滚动更新、缩容和故障接管。 + +下一步入口与完成条件见 [实施计划](2026-10-21-kubernetes-control-plane.md)。本轮改动在本地提交；推送及 PR 更新仍须遵守用户要求的 push 前确认。 diff --git a/docs/plans/evidence/kubernetes/shared-state/baseline-e2e.json b/docs/plans/evidence/kubernetes/shared-state/baseline-e2e.json new file mode 100644 index 000000000..0cc183e92 --- /dev/null +++ b/docs/plans/evidence/kubernetes/shared-state/baseline-e2e.json @@ -0,0 +1,126 @@ +{ + "verified_commit": "bf9f508dc0f7bb7232a6ed0fd1a0bcdf6e85a5f1", + "baseline_commit": "eee8f6ecb6ca04455e6edef0c8e498fb3be21f21", + "assertions": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "baseline_assertion_sha256": "b94b33d5b07ef7a98da1199dde49d3bb8be79daed51ca7d5c01ce3002d3e4811", + "matches_baseline": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "baseline_assertion_sha256": "e736744c0afda8b9160312af46c31dfde85514afaf09a5d1a97575b09101da15", + "matches_baseline": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "matches_baseline": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "baseline_assertion_sha256": "5e6c5a7d2769e0e83c04a4a727c75808b0e1592375b77a815a02b9677de5c477", + "matches_baseline": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "baseline_assertion_sha256": "5bff2f11fcb7ad2b0c71a78cf620dc3070ecddd3a78d954956a32c006ff887f4", + "matches_baseline": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "baseline_assertion_sha256": "6a63caf64d29521ed813c597ef288b59270dd6cd3789d4c2a11fe430f2ef95f3", + "matches_baseline": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "baseline_assertion_sha256": "858fac7a2bd27316ba43e74a56d0ae6e0bcf45a553d8ecc3da006bfc0a768afb", + "matches_baseline": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "baseline_assertion_sha256": "fc9f564f4db729f7ec444eca13379843ea4c861646e4d21c6c7fdac8b629e390", + "matches_baseline": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "baseline_assertion_sha256": "1c1ddb4f06cf94c9408f62e32d5a13a572bceccc1b0acdb5eb9f04acb2a8538b", + "matches_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "matches_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "baseline_assertion_sha256": "d435dea7913cfa24178f66420d521003431251d6b764517887968b26f123a027", + "matches_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "baseline_assertion_sha256": "9a16b07bbe941aca1d5d3d2cac804b9f432fea6f74a28b38cceaadabc1af53a4", + "matches_baseline": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "baseline_assertion_sha256": "0af5a9fd20a0a5f8cdfa29b648678f9284a376f484b82d04a1a0230bdf6aa41b", + "matches_baseline": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "baseline_assertion_sha256": "5af4ac27ac1e335b2217102a235b4901fbcf18d33188d54a12497261e6e87049", + "matches_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "matches_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "baseline_assertion_sha256": "017b08bb23fcae3cb2a5bb334bf68a032eedf15572fce0c3f585154b3b940833", + "matches_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "baseline_assertion_sha256": "152e5bc121c8a068b2003c5d9e4ce943560281dba6f96f1b238eb1b3e89a2eb7", + "matches_baseline": true + } + ], + "all_failures_match_baseline": true +} diff --git a/docs/plans/evidence/kubernetes/shared-state/baseline-unit.json b/docs/plans/evidence/kubernetes/shared-state/baseline-unit.json new file mode 100644 index 000000000..9f76e6816 --- /dev/null +++ b/docs/plans/evidence/kubernetes/shared-state/baseline-unit.json @@ -0,0 +1,56 @@ +{ + "verified_commit": "bf9f508dc0f7bb7232a6ed0fd1a0bcdf6e85a5f1", + "baseline_commit": "eee8f6ecb6ca04455e6edef0c8e498fb3be21f21", + "assertions": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181", + "baseline_assertion_sha256": "4e6c294bbe40bd85115a154e355bc81e5b21e3e781d2e0dcd1feafcd85a57181", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11", + "baseline_assertion_sha256": "e37b9c82cfabec8b2d18adaf338ef1368f30e11e38cd9e46c5f5d2d212ec2d11", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d", + "baseline_assertion_sha256": "f68c71379df4d2ba7748acb814268e5f1dcc6590c86098766cf66043bb9fdf9d", + "matches_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca", + "baseline_assertion_sha256": "5044751390e94cf06d620d10ac9c06e61d7a6a9d27b67bf37eb9711fa14b24ca", + "matches_baseline": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "baseline_assertion_sha256": "a81c34c8c33c94dae4bef4af01c4db186fa7431766e106dedb0c93fac4a3f55b", + "matches_baseline": true + } + ], + "all_failures_match_baseline": true +} diff --git a/docs/plans/evidence/kubernetes/shared-state/verification.json b/docs/plans/evidence/kubernetes/shared-state/verification.json new file mode 100644 index 000000000..518a1adea --- /dev/null +++ b/docs/plans/evidence/kubernetes/shared-state/verification.json @@ -0,0 +1,55 @@ +{ + "source_commit": "bf9f508dc0f7bb7232a6ed0fd1a0bcdf6e85a5f1", + "scope": "shared key/user admission, token buckets and selected configuration caches; active-active control plane remains unimplemented", + "node": "22.23.2", + "postgresql": "17.11", + "checks": { + "unit": { + "verified_commit": "f4ff7004530791b877cc7ac27971596628e42d13", + "files": 291, + "exit_code": 1, + "log_sha256": "088eae935b5c46a2c994b1345b4cbfebbba56132cf7522926c9644d6a30ae0b9", + "totals": { + "tests": 3408, + "pass": 3390, + "fail": 8, + "cancelled": 0, + "skipped": 10 + }, + "non_baseline_failure": { + "file": "test/unit/slot-shell-rc.test.mjs", + "test": "official login tickets check a cluster slot for its container cli-node", + "actual": "cli_node_missing", + "expected": "container_not_ready", + "classification": "test consulted running host container rather than synthetic stopped-container state", + "test_fix_commit": "5abd21d95da3eb18f0b9a2b06b179ff96983f744", + "directed_after_fix": { + "tests": 19, + "pass": 19, + "fail": 0 + }, + "full_unit_after_test_fix_rerun": false + } + }, + "e2e": { + "verified_commit": "bf9f508dc0f7bb7232a6ed0fd1a0bcdf6e85a5f1", + "files": 28, + "exit_code": 1, + "log_sha256": "3d7dc174461c1b76667706bf054311ffc3b3fc69beb6d491a8b081570f0d97e6", + "totals": { + "tests": 121, + "pass": 103, + "fail": 17, + "cancelled": 0, + "skipped": 1 + } + } + }, + "runtime_deployment_performed": false, + "limitations": [ + "account/VM/seat/session global admission remains unimplemented", + "executor fencing and safe orphan recovery remain unimplemented", + "background tasks, routing and full-snapshot write conflicts remain uncoordinated", + "no multi-node active replicas, HPA or rollout validation" + ] +} From 62740718b36f4d81c6daef3d6c5f9625e95cfdf2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:26:05 +0800 Subject: [PATCH 071/473] test(pool): reproduce reservation leaks before attempt execution --- test/unit/failover-release.test.mjs | 176 ++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 test/unit/failover-release.test.mjs diff --git a/test/unit/failover-release.test.mjs b/test/unit/failover-release.test.mjs new file mode 100644 index 000000000..40d0ec0f3 --- /dev/null +++ b/test/unit/failover-release.test.mjs @@ -0,0 +1,176 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setImmediate } from 'node:timers/promises' +import { getDb } from '../../src/lib/db/database.mjs' +import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +const success = () => ({ + ok: true, + status: 200, + terminalState: 'verified', + body: { + type: 'message', + id: 'msg_release', + role: 'assistant', + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + }, +}) + +function schedulerWithRelease(release, extra = {}) { + return { + async selectAndReserve() { + return { ok: true, accountId: 'account-release', vmId: 'vm-release', release, ...extra } + }, + markSuccess() {}, + } +} + +function request(requestId, overrides = {}) { + return { + requestId, + model: 'claude-sonnet-4-6', + canonicalBody: { model: 'claude-sonnet-4-6' }, + callAttempt: success, + ...overrides, + } +} + +test('prebinding failure releases the selected reservation before propagating', async () => { + let releases = 0 + const runner = new FailoverRunner({ + scheduler: schedulerWithRelease(async () => { + await getDb().prepare('SELECT 1').get() + releases++ + }), + stickyRouter: { + bind() { + throw new Error('prebinding failed') + }, + }, + }) + await assert.rejects( + runner.run( + request('bind-release', { + stickyKey: 'bind-session', + callAttempt: () => assert.fail('prebinding failure must not execute upstream'), + }), + ), + /prebinding failed/, + ) + assert.equal(releases, 1) +}) + +test('family prebinding failure also releases a successful reservation', async () => { + let releases = 0 + const runner = new FailoverRunner({ + scheduler: schedulerWithRelease(() => releases++, { familyGated: true }), + stickyRouter: { + resolve() { + throw new Error('family binding unavailable') + }, + }, + }) + await assert.rejects( + runner.run( + request('family-release', { + familyKey: 'family-session', + familyVmId: 'vm-family', + callAttempt: () => assert.fail('family binding failure must not execute upstream'), + }), + ), + /family binding unavailable/, + ) + assert.equal(releases, 1) +}) + +test('a PostgreSQL attempt begin failure releases capacity without completing or executing the attempt', async () => { + const attempts = new RequestAttemptsRepo(getDb()) + await attempts.begin({ requestId: 'duplicate-attempt', attemptNo: 1 }) + let releases = 0 + const runner = new FailoverRunner({ + scheduler: schedulerWithRelease(async () => { + await getDb().prepare('SELECT 1').get() + releases++ + }), + attemptsRepo: attempts, + }) + await assert.rejects( + runner.run( + request('duplicate-attempt', { + callAttempt: () => assert.fail('attempt begin failure must not execute upstream'), + }), + ), + { code: '23505' }, + ) + assert.equal(releases, 1) + const rows = await attempts.list('duplicate-attempt') + assert.equal(rows.length, 1) + assert.equal(rows[0].terminal_state, 'started') + assert.equal(rows[0].completed_at, null) +}) + +test('same-session requests wait until asynchronous capacity release completes', async () => { + const releasing = deferred() + const finishRelease = deferred() + let reservations = 0 + let releases = 0 + let calls = 0 + let holderSettled = false + const runner = new FailoverRunner({ + scheduler: { + async selectAndReserve() { + const turn = ++reservations + return { + ok: true, + accountId: 'account-release', + vmId: 'vm-release', + async release() { + if (turn === 1) { + releasing.resolve() + await finishRelease.promise + } + await getDb().prepare('SELECT 1').get() + releases++ + }, + } + }, + markSuccess() {}, + }, + }) + const args = (id) => + request(id, { + stickyKey: 'shared-release-session', + callAttempt: () => { + calls++ + return success() + }, + }) + const holder = runner.run(args('release-holder')) + holder.then(() => { + holderSettled = true + }) + await releasing.promise + const follower = runner.run(args('release-follower')) + try { + await setImmediate() + assert.equal(holderSettled, false) + assert.equal(reservations, 1) + assert.equal(calls, 1) + } finally { + finishRelease.resolve() + await Promise.all([holder, follower]) + } + assert.equal(reservations, 2) + assert.equal(releases, 2) + assert.equal(calls, 2) +}) From 86eb7eb53cd6acdb2a5da2e922e91b84500fa98a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:26:28 +0800 Subject: [PATCH 072/473] test(pool): reproduce missing shared admission and asynchronous seat grant races --- test/unit/pool-scheduler-shared.test.mjs | 152 +++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 test/unit/pool-scheduler-shared.test.mjs diff --git a/test/unit/pool-scheduler-shared.test.mjs b/test/unit/pool-scheduler-shared.test.mjs new file mode 100644 index 000000000..a0caa27d9 --- /dev/null +++ b/test/unit/pool-scheduler-shared.test.mjs @@ -0,0 +1,152 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' +import { SeatPlanner } from '../../src/lib/pool/seat-planner.mjs' + +function gate() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +function sharedPool(admissions) { + const pool = new PoolScheduler({ + projectRoot: '/tmp', + executionAdmissions: admissions, + accountQuota: { + canAccept: () => ({ ok: true }), + limitFor: () => 2, + policyFor: () => ({}), + rpmLimitFor: () => 0, + }, + config: { seat_grace_ms: 0 }, + }) + pool.eligibleCandidates = async () => [ + { + ok: true, + accountId: 'shared-account', + vmId: 'shared-vm', + model: 'claude-test', + vm: { id: 'shared-vm', policy: { maxConcurrency: 2 } }, + account: { account_id: 'shared-account' }, + maxConcurrency: 2, + inflight: 0, + loadRatio: 0, + priority: 0, + seatCap: 2, + headroom: null, + }, + ] + return pool +} + +test('Kubernetes selection awaits the shared admission decision instead of reserving process-local capacity', async () => { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + let calls = 0 + const pool = sharedPool({ + async tryReserve() { + calls++ + return { ok: false, code: 'concurrency_limit' } + }, + }) + let selected + try { + selected = await pool.selectAndReserve({ model: 'claude-test', allowWait: false }) + assert.equal(calls, 1, 'the PostgreSQL admission service decides each attempted reservation') + assert.equal(selected.ok, false) + assert.equal(pool.inflight.size, 0) + } finally { + await selected?.release?.() + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + } +}) + +test('Kubernetes selection ignores stale local inflight and exposes the shared execution lease', async () => { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + let released = 0 + const lease = { ok: true, admissionId: 'admission-test', release: async () => released++ } + const pool = sharedPool({ tryReserve: async () => lease }) + pool.inflight.set('shared-account', 99) + let selected + try { + selected = await pool.selectAndReserve({ model: 'claude-test', allowWait: false }) + assert.equal(selected.ok, true) + assert.equal(selected.admissionId, lease.admissionId) + await selected.release() + assert.equal(released, 1) + } finally { + await selected?.release?.() + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + } +}) + +test('an asynchronous seat refusal stays queued until its admission decision resolves', async () => { + const decision = gate() + const planner = new SeatPlanner({ asyncGrants: true }) + const ticket = { + seatKey: 'device-test', + vmId: 'vm-test', + kind: 'conc', + enqueuedAt: Date.now(), + tryGrant: () => decision.promise, + } + planner.enqueue(ticket) + planner.pump('vm-test') + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(ticket.queued, true, 'a pending Promise must not be treated as a successful grant') + decision.resolve(false) + await planner.flush() + assert.equal(ticket.queued, true) + } finally { + decision.resolve(false) + planner.remove(ticket) + } +}) + +test('a cancelled queued request awaits release of a late asynchronous seat grant', async () => { + const decision = gate() + const released = gate() + let releaseCalls = 0 + const planner = new SeatPlanner({ asyncGrants: true }) + const ticket = { + seatKey: 'device-late', + vmId: 'vm-late', + kind: 'conc', + enqueuedAt: Date.now(), + async tryGrant() { + await decision.promise + ticket.granted = { + reservation: { + async release() { + releaseCalls++ + await released.promise + }, + }, + } + return true + }, + } + planner.enqueue(ticket) + planner.pump('vm-late') + planner.remove(ticket) + decision.resolve() + try { + let flushed = false + const pending = planner.flush().then(() => (flushed = true)) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(releaseCalls, 1) + assert.equal(flushed, false, 'shutdown flush includes the late grant release') + released.resolve() + await pending + assert.equal(ticket.granted, null) + } finally { + released.resolve() + } +}) From 7412df2bc08e3a26293bc212932e2f3de81b190f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:27:02 +0800 Subject: [PATCH 073/473] test: require shared execution concurrency and RPM admission --- test/unit/execution-admissions.test.mjs | 198 ++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 test/unit/execution-admissions.test.mjs diff --git a/test/unit/execution-admissions.test.mjs b/test/unit/execution-admissions.test.mjs new file mode 100644 index 000000000..7afd99ff9 --- /dev/null +++ b/test/unit/execution-admissions.test.mjs @@ -0,0 +1,198 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' + +async function replicas() { + const { ExecutionAdmissionsRepo } = await import('../../src/lib/db/repos/execution-admissions-repo.mjs') + const db = getDb() + const other = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: db.schema }) + return [ + new ExecutionAdmissionsRepo(db, { ownerId: 'controller-a' }), + new ExecutionAdmissionsRepo(other, { ownerId: 'controller-b' }), + ] +} + +function candidate(patch = {}) { + return { accountId: 'account-1', vmId: 'vm-1', accountLimit: 8, vmLimit: 8, fableLimit: 0, rpmLimit: 0, ...patch } +} + +test('replicas share account concurrency across different VMs', async () => { + const repos = await replicas() + const gates = await Promise.all( + Array.from({ length: 12 }, (_, i) => repos[i % 2].tryReserve(candidate({ accountLimit: 1, vmId: `vm-${i}` }))), + ) + assert.equal(gates.filter((gate) => gate.ok).length, 1) + assert.ok(gates.filter((gate) => !gate.ok).every((gate) => gate.code === 'concurrency_limit')) + assert.equal((await repos[1].getCounts(candidate())).accountInflight, 1) + await gates.find((gate) => gate.ok).release() +}) + +test('replicas share VM concurrency across different accounts', async () => { + const repos = await replicas() + const gates = await Promise.all( + Array.from({ length: 12 }, (_, i) => repos[i % 2].tryReserve(candidate({ accountId: `account-${i}`, vmLimit: 2 }))), + ) + assert.equal(gates.filter((gate) => gate.ok).length, 2) + assert.ok(gates.filter((gate) => !gate.ok).every((gate) => gate.code === 'vm_concurrency_limit')) + assert.equal((await repos[0].getCounts(candidate())).vmInflight, 2) + await Promise.all(gates.filter((gate) => gate.ok).map((gate) => gate.release())) +}) + +test('Fable concurrency is an account sublimit and leaves room for ordinary models', async () => { + const [first, second] = await replicas() + const fable = candidate({ accountLimit: 3, fableLimit: 1, family: 'fable' }) + const gate = await first.tryReserve(fable) + assert.equal(gate.ok, true) + assert.equal((await second.tryReserve({ ...fable, vmId: 'vm-other' })).code, 'fable_concurrency') + const regular = await second.tryReserve({ ...fable, vmId: 'vm-other', family: null }) + assert.equal(regular.ok, true) + const counts = await second.getCounts(candidate()) + assert.equal(counts.accountInflight, 2) + assert.equal(counts.fableInflight, 1) + await gate.release() + await regular.release() +}) + +test('account RPM includes released admissions on other VMs until the window ends', async () => { + const [first, second] = await replicas() + const input = candidate({ rpmLimit: 1 }) + const gate = await first.tryReserve(input) + await gate.release() + const denied = await second.tryReserve({ ...input, vmId: 'vm-other' }) + assert.equal(denied.code, 'rpm_limit') + assert.ok(denied.retry_after_ms > 0 && denied.retry_after_ms <= 60_000) + const counts = await second.getCounts(input) + assert.equal(counts.accountInflight, 0) + assert.equal(counts.rpmCount, 1) + assert.ok(counts.rpmResetAt > Date.now()) + await first.db.prepare("UPDATE execution_admissions SET created_at = clock_timestamp() - interval '61 seconds'").run() + const next = await second.tryReserve(input) + assert.equal(next.ok, true) + assert.equal((await second.db.prepare('SELECT count(*) AS count FROM execution_admissions').get()).count, 1) + await next.release() +}) + +test('zero concurrency rejects while zero RPM and Fable sublimits are unlimited', async () => { + const [first] = await replicas() + assert.equal((await first.tryReserve(candidate({ accountLimit: 0 }))).code, 'concurrency_limit') + assert.equal((await first.tryReserve(candidate({ vmLimit: 0 }))).code, 'vm_concurrency_limit') + const gates = [] + for (let i = 0; i < 4; i++) { + const gate = await first.tryReserve(candidate({ family: 'fable', fableLimit: 0, rpmLimit: 0 })) + assert.equal(gate.ok, true) + gates.push(gate) + } + await Promise.all(gates.map((gate) => gate.release())) +}) + +test('missing and invalid limits reject without creating an admission or RPM event', async () => { + const [first] = await replicas() + for (const field of ['accountLimit', 'vmLimit', 'fableLimit', 'rpmLimit']) { + for (const value of [undefined, NaN, Infinity, -1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + await assert.rejects(first.tryReserve(candidate({ [field]: value })), { code: 'invalid_execution_admission' }) + } + } + assert.equal((await first.db.prepare('SELECT count(*) AS count FROM execution_admissions').get()).count, 0) +}) + +test('a VM refusal does not consume the account RPM budget', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate({ accountId: 'account-held', vmLimit: 1 })) + assert.equal((await second.tryReserve(candidate({ rpmLimit: 1, vmLimit: 1 }))).code, 'vm_concurrency_limit') + assert.equal((await second.getCounts(candidate())).rpmCount, 0) + const accepted = await second.tryReserve(candidate({ vmId: 'vm-free', rpmLimit: 1, vmLimit: 1 })) + assert.equal(accepted.ok, true) + await held.release() + await accepted.release() +}) + +test('release is idempotent and cannot release a later request or another owner', async () => { + const [first, second] = await replicas() + const input = candidate({ accountLimit: 1 }) + const held = await first.tryReserve(input) + await second.release({ admissionId: held.admissionId, ownerId: 'controller-b' }) + assert.equal((await second.tryReserve(input)).code, 'concurrency_limit') + await Promise.all([held.release(), held.release()]) + const next = await second.tryReserve(input) + assert.equal(next.ok, true) + await held.release() + assert.equal((await first.tryReserve(input)).code, 'concurrency_limit') + await next.release() +}) + +test('uncertain executions remain active after release attempts, aging and connection loss', async () => { + const [first, second] = await replicas() + const input = candidate({ accountLimit: 1, family: 'fable', fableLimit: 1 }) + const held = await second.tryReserve(input) + await held.markUncertain() + await held.release() + await first.db.prepare("UPDATE execution_admissions SET created_at = clock_timestamp() - interval '2 hours'").run() + await second.db.close() + assert.equal((await first.tryReserve(input)).code, 'concurrency_limit') + const counts = await first.getCounts(input) + assert.equal(counts.accountInflight, 1) + assert.equal(counts.vmInflight, 1) + assert.equal(counts.fableInflight, 1) + assert.equal(counts.rpmCount, 0) + const row = await first.db + .prepare('SELECT owner_id, uncertain_at, released_at FROM execution_admissions WHERE id = $1') + .get(held.admissionId) + assert.equal(row.owner_id, 'controller-b') + assert.ok(row.uncertain_at) + assert.equal(row.released_at, null) +}) + +test('release with an uncertain outcome persists the hold before any later release', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate({ accountLimit: 1 })) + await held.release({ uncertain: true }) + await held.release() + assert.equal((await second.tryReserve(candidate({ accountLimit: 1 }))).code, 'concurrency_limit') +}) + +test('admission and RPM creation roll back with an enclosing seat transaction', async () => { + const [first, second] = await replicas() + await assert.rejects( + withTransaction(first.db, async () => { + const held = await first.tryReserve(candidate({ accountLimit: 1, rpmLimit: 1 })) + assert.equal(held.ok, true) + throw new Error('seat unavailable') + }), + /seat unavailable/, + ) + assert.deepEqual(await second.getCounts(candidate()), { + accountInflight: 0, + vmInflight: 0, + fableInflight: 0, + rpmCount: 0, + rpmResetAt: null, + }) + const next = await second.tryReserve(candidate({ accountLimit: 1, rpmLimit: 1 })) + assert.equal(next.ok, true) + await next.release() +}) + +test('accepted requests persist their correlation and optional seat binding', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate({ requestId: 'request-http-1' })) + assert.equal(held.ownerId, 'controller-a') + assert.equal(held.requestId, 'request-http-1') + await first.db + .prepare('UPDATE execution_admissions SET seat_key = $1, seat_generation = $2, slot_index = $3 WHERE id = $4') + .run('seat-device-1', 'seat-generation-1', 2, held.admissionId) + const row = await second.db.prepare('SELECT * FROM execution_admissions WHERE id = $1').get(held.admissionId) + assert.equal(row.request_id, held.requestId) + assert.equal(row.account_id, 'account-1') + assert.equal(row.vm_id, 'vm-1') + assert.equal(row.seat_key, 'seat-device-1') + assert.equal(row.seat_generation, 'seat-generation-1') + assert.equal(row.slot_index, 2) + await held.release() +}) + +test('a shared database failure never creates an in-memory fallback admission', async () => { + const [first, second] = await replicas() + await second.db.close() + await assert.rejects(second.tryReserve(candidate()), /database is closed/) + assert.equal((await first.getCounts(candidate())).accountInflight, 0) +}) From 389d2dee4624e10228ebf2166bbf93ad9e7f4e57 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:27:20 +0800 Subject: [PATCH 074/473] fix(pool): release reservations across setup failures and await cleanup --- src/lib/pool/failover-runner.mjs | 626 ++++++++++++++++--------------- 1 file changed, 314 insertions(+), 312 deletions(-) diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index df2068a86..f6b1dc7d4 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -757,344 +757,346 @@ export class FailoverRunner { } throw error } - if (selected?.familyGated && familyKey && familyVmId) { - // The family home left the pool (quota, credential): the family moves - // with the next bind. An exclusion this request made itself is not proof. - const home = this.stickyRouter?.resolve?.(familyKey) - const ownExclusion = - !!home && (budget.tried.has(home.accountId) || budget.tried.has(home.vmId) || spilled.has(home.accountId)) - if (!ownExclusion) this.stickyRouter?.unbind?.(familyKey) - familyVmId = null - } - if (!selected?.ok && selected?.reason === 'fable_requires_max') { - return { - ...fableRequiresMaxError({ - wait_ms: selected?.waitMs ?? selected?.wait_ms ?? 0, - eligible: selected?.eligible ?? 0, - available: selected?.available ?? 0, - attempt_count: budget.hops, - }), - ...attribution(), - } - } - if (!selected?.ok) { - const exhausted = selectionFailure(selected, { excluded, lastPolicy, lastResult, hops: budget.hops }) - return preferLastResult(lastResult, lastPolicy, exhausted, attribution()) - } - repairPending = false - snapshotPins() - const attemptStickyBound = pinBaseline.get(stickyKey) || null - lastSelected = selected - pinnedSlot = selected.slotIndex ?? null - if (selected.seatMoved) seatMovedTo = selected.vmId - bindAll({ accountId: selected.accountId, vmId: selected.vmId, slotIndex: pinnedSlot }, { countHit: false }) - const attemptStarted = Date.now() - await this.attemptsRepo?.begin?.({ - requestId, - attemptNo, - vmId: selected.vmId, - accountId: selected.accountId, - model, - selectionReason: selected.selectionReason, - waitMs: selected.waitMs, - }) - let result - let policy - let committed = false - let hopNoted = false - const noteHop = (value) => { - if (hopNoted) return - hopNoted = true - const executions = executionsOf(value) - if (executions > 0) budget.noteHop(selected.accountId, selected.vmId, executions) - } try { - const repairKey = searchRepairKey(stickyKey, model, selected) - const hint = this.searchRepairs.get(repairKey) - if (hint && hint.expiresAt <= Date.now()) this.searchRepairs.delete(repairKey) - const knownSearchRepair = hint && hint.expiresAt > Date.now() && hint.fingerprint === fingerprint() - const attemptBody = knownSearchRepair ? flattenSearchResultHistory(requestBody) : requestBody - const prepared = - typeof applyAttempt === 'function' - ? await applyAttempt(clone(attemptBody), selected, { - attemptNo, - repaired, - attemptStartedAt: attemptStarted, - freshSlot, - stickyBound: attemptStickyBound, - }) - : clone(attemptBody) - const wrappedAttempt = - prepared && - typeof prepared === 'object' && - Object.prototype.hasOwnProperty.call(prepared, 'body') && - Object.prototype.hasOwnProperty.call(prepared, 'meta') - const body = wrappedAttempt ? prepared.body : prepared - const attemptMeta = wrappedAttempt ? prepared.meta : null - if (attemptMeta?.sessionId) { - outboundSessionId = String(attemptMeta.sessionId) - outboundSessionAccountId = selected.accountId - bindAll( - { accountId: selected.accountId, vmId: selected.vmId, sessionId: outboundSessionId }, - { - countHit: false, - // Prebinding moved the VM already; replace the previous VM's session. - replaceSession: !!attemptStickyBound?.vmId && attemptStickyBound.vmId !== selected.vmId, - }, - ) - } - result = await callAttempt({ - candidate: selected, - body, - attemptMeta, - attemptNo, - stream, - deliveryMode: deliveryMode || this.config.delivery_mode, - signal, - onCommit: () => { - committed = true - }, - }) - noteHop(result) - if (result) result.committed = result.committed || committed - if (signal?.aborted || isClientCancelledResult(result)) { - await this.attemptsRepo?.complete?.(requestId, attemptNo, { - upstreamStatus: result?.status ?? null, - errorScope: 'request', - action: 'stop', - downstreamCommitted: !!(result?.committed || committed), - terminalState: 'cancelled', - latencyMs: Date.now() - attemptStarted, - }) - return { ...clientCancelledResult(result), via: result?.via || 'pool-failover', ...attribution() } + if (selected?.familyGated && familyKey && familyVmId) { + // The family home left the pool (quota, credential): the family moves + // with the next bind. An exclusion this request made itself is not proof. + const home = this.stickyRouter?.resolve?.(familyKey) + const ownExclusion = + !!home && (budget.tried.has(home.accountId) || budget.tried.has(home.vmId) || spilled.has(home.accountId)) + if (!ownExclusion) this.stickyRouter?.unbind?.(familyKey) + familyVmId = null } - policy = classifyAttempt( - result, - selected, - { - model, - repaired, - oauth401CooldownMs: this.config.oauth_401_cooldown_ms, - signatureRepair: signatureRepairEnabled(this.config), - }, - this.scheduler?.accountQuota, - ) - if (knownSearchRepair && policy.reason === 'search_repairable') { - // Repeating the identical flatten cannot fix a second rejection. - this.searchRepairs.delete(repairKey) - policy = { ...policy, action: 'stop' } - } - // One writer for rate_limit_reset_at / overload_until (sub2api HandleUpstreamError). - const hardBlock = - !result?.committed && !pinVmId - ? this.rateLimitService?.handleUpstreamError?.({ - accountId: selected.accountId, - vmId: selected.vmId, - result, - policy, - }) || null - : null - if (hardBlock?.until && policy.action === 'continue-and-cooldown' && policy.scope === 'account') { - policy = { ...policy, cooldownUntil: hardBlock.until } + if (!selected?.ok && selected?.reason === 'fable_requires_max') { + return { + ...fableRequiresMaxError({ + wait_ms: selected?.waitMs ?? selected?.wait_ms ?? 0, + eligible: selected?.eligible ?? 0, + available: selected?.available ?? 0, + attempt_count: budget.hops, + }), + ...attribution(), + } } - - lastResult = result - lastPolicy = policy - this.noteUnitHealth(selected, policy, result) - notifyProxyFailure(this.onProxyFailure, selected, policy) - if (policy.reason === 'fable_plan_denied' && typeof this.onFablePlanDenied === 'function') { - try { - this.onFablePlanDenied({ selected, policy }) - } catch {} + if (!selected?.ok) { + const exhausted = selectionFailure(selected, { excluded, lastPolicy, lastResult, hops: budget.hops }) + return preferLastResult(lastResult, lastPolicy, exhausted, attribution()) } - const terminalState = result?.terminalState || (result?.ok ? 'unknown' : 'error') - await this.attemptsRepo?.complete?.(requestId, attemptNo, { - upstreamStatus: result?.status ?? null, - errorScope: policy.scope, - action: policy.action, - cooldownUntil: policy.cooldownUntil, - downstreamCommitted: result?.committed || committed, - terminalState, - usage: usageOf(result), - ttftMs: result?.ttftMs ?? null, - latencyMs: Date.now() - attemptStarted, + repairPending = false + snapshotPins() + const attemptStickyBound = pinBaseline.get(stickyKey) || null + lastSelected = selected + pinnedSlot = selected.slotIndex ?? null + if (selected.seatMoved) seatMovedTo = selected.vmId + bindAll({ accountId: selected.accountId, vmId: selected.vmId, slotIndex: pinnedSlot }, { countHit: false }) + const attemptStarted = Date.now() + await this.attemptsRepo?.begin?.({ + requestId, + attemptNo, + vmId: selected.vmId, + accountId: selected.accountId, + model, + selectionReason: selected.selectionReason, + waitMs: selected.waitMs, }) - if (typeof onAttempt === 'function') { - await onAttempt({ attemptNo, selected, result, policy }) + let result + let policy + let committed = false + let hopNoted = false + const noteHop = (value) => { + if (hopNoted) return + hopNoted = true + const executions = executionsOf(value) + if (executions > 0) budget.noteHop(selected.accountId, selected.vmId, executions) } - if (policy.reason === 'content_filter_refusal' || policy.reason === 'usage_policy_refusal') { - this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, countUsage }) - bindAll({ accountId: selected.accountId, vmId: selected.vmId }) - return { - ...result, - ...attribution(), - finalState: policy.reason === 'usage_policy_refusal' ? 'rejected' : 'content_filter', - policy, + try { + const repairKey = searchRepairKey(stickyKey, model, selected) + const hint = this.searchRepairs.get(repairKey) + if (hint && hint.expiresAt <= Date.now()) this.searchRepairs.delete(repairKey) + const knownSearchRepair = hint && hint.expiresAt > Date.now() && hint.fingerprint === fingerprint() + const attemptBody = knownSearchRepair ? flattenSearchResultHistory(requestBody) : requestBody + const prepared = + typeof applyAttempt === 'function' + ? await applyAttempt(clone(attemptBody), selected, { + attemptNo, + repaired, + attemptStartedAt: attemptStarted, + freshSlot, + stickyBound: attemptStickyBound, + }) + : clone(attemptBody) + const wrappedAttempt = + prepared && + typeof prepared === 'object' && + Object.prototype.hasOwnProperty.call(prepared, 'body') && + Object.prototype.hasOwnProperty.call(prepared, 'meta') + const body = wrappedAttempt ? prepared.body : prepared + const attemptMeta = wrappedAttempt ? prepared.meta : null + if (attemptMeta?.sessionId) { + outboundSessionId = String(attemptMeta.sessionId) + outboundSessionAccountId = selected.accountId + bindAll( + { accountId: selected.accountId, vmId: selected.vmId, sessionId: outboundSessionId }, + { + countHit: false, + // Prebinding moved the VM already; replace the previous VM's session. + replaceSession: !!attemptStickyBound?.vmId && attemptStickyBound.vmId !== selected.vmId, + }, + ) } - } - if (verifiedSuccess(result)) { - if (repairKey && (knownSearchRepair || searchRepair?.key === repairKey)) { - this.searchRepairs.delete(repairKey) - while (this.searchRepairs.size >= SEARCH_REPAIR_LIMIT) { - this.searchRepairs.delete(this.searchRepairs.keys().next().value) - } - this.searchRepairs.set(repairKey, { - fingerprint: fingerprint(), - expiresAt: Date.now() + SEARCH_REPAIR_IDLE_MS, + result = await callAttempt({ + candidate: selected, + body, + attemptMeta, + attemptNo, + stream, + deliveryMode: deliveryMode || this.config.delivery_mode, + signal, + onCommit: () => { + committed = true + }, + }) + noteHop(result) + if (result) result.committed = result.committed || committed + if (signal?.aborted || isClientCancelledResult(result)) { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { + upstreamStatus: result?.status ?? null, + errorScope: 'request', + action: 'stop', + downstreamCommitted: !!(result?.committed || committed), + terminalState: 'cancelled', + latencyMs: Date.now() - attemptStarted, }) + return { ...clientCancelledResult(result), via: result?.via || 'pool-failover', ...attribution() } } - this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, countUsage }) - if (isFableModel(model) && typeof this.onFableSuccess === 'function') { - try { - this.onFableSuccess({ selected, model }) - } catch {} + policy = classifyAttempt( + result, + selected, + { + model, + repaired, + oauth401CooldownMs: this.config.oauth_401_cooldown_ms, + signatureRepair: signatureRepairEnabled(this.config), + }, + this.scheduler?.accountQuota, + ) + if (knownSearchRepair && policy.reason === 'search_repairable') { + // Repeating the identical flatten cannot fix a second rejection. + this.searchRepairs.delete(repairKey) + policy = { ...policy, action: 'stop' } } - bindAll({ accountId: selected.accountId, vmId: selected.vmId }) - return { ...result, ...attribution(), finalState: 'verified' } - } - if (policy.action === 'repair-and-retry' && !repaired && !result?.committed) { - repaired = true - repairPending = true - if (policy.reason === 'search_repairable' && repairKey && fingerprint()) { - searchRepair = { key: repairKey } + // One writer for rate_limit_reset_at / overload_until (sub2api HandleUpstreamError). + const hardBlock = + !result?.committed && !pinVmId + ? this.rateLimitService?.handleUpstreamError?.({ + accountId: selected.accountId, + vmId: selected.vmId, + result, + policy, + }) || null + : null + if (hardBlock?.until && policy.action === 'continue-and-cooldown' && policy.scope === 'account') { + policy = { ...policy, cooldownUntil: hardBlock.until } } - requestBody = repairAnthropicRequest(requestBody, policy) - continue - } - applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId }) - if (!pinVmId && isCredentialDeath(policy)) { - this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys }) - if (familyKey) { - return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } + + lastResult = result + lastPolicy = policy + this.noteUnitHealth(selected, policy, result) + notifyProxyFailure(this.onProxyFailure, selected, policy) + if (policy.reason === 'fable_plan_denied' && typeof this.onFablePlanDenied === 'function') { + try { + this.onFablePlanDenied({ selected, policy }) + } catch {} } - } - if (!shouldContinue(policy)) { - return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } - } - policy = await this.recoverCredential(selected, policy) - const hopMs = Date.now() - attemptStarted - if (isRetryableEmptyHop(policy)) { - if (!emptyHopReleased(result) || pinVmId) return incompleteHopResult(result, policy, attribution()) - this.retireOutboundSession(selected, bindKeys) - freshSlot = true - } - if (budget.allowSameUnit(selected.accountId, policy, hopMs, this.config.same_account_retry_max_hop_ms)) { - budget.noteSameUnit(selected.accountId) - avoid = new Set([selected.accountId, selected.vmId]) - continue - } - budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) - } catch (error) { - const invalidCache = error?.status === 400 && error.body?.error?.code === 'invalid_cache_control' - noteHop(invalidCache ? { upstreamExecutions: 0 } : null) - if (error?.code === 'response_sent') { + const terminalState = result?.terminalState || (result?.ok ? 'unknown' : 'error') await this.attemptsRepo?.complete?.(requestId, attemptNo, { - upstreamStatus: 403, - errorScope: 'request', - action: 'stop', - downstreamCommitted: true, - terminalState: 'policy_block', + upstreamStatus: result?.status ?? null, + errorScope: policy.scope, + action: policy.action, + cooldownUntil: policy.cooldownUntil, + downstreamCommitted: result?.committed || committed, + terminalState, + usage: usageOf(result), + ttftMs: result?.ttftMs ?? null, latencyMs: Date.now() - attemptStarted, }) - return { - ok: false, - responseSent: true, - status: 403, - via: 'account-scope', - vmId: selected.vmId, - accountId: selected.accountId, - attemptCount: attemptNo, - finalState: 'policy_block', + if (typeof onAttempt === 'function') { + await onAttempt({ attemptNo, selected, result, policy }) } - } - if (invalidCache) { - await this.attemptsRepo?.complete?.(requestId, attemptNo, { - upstreamStatus: 0, - errorScope: 'request', - action: 'stop', - downstreamCommitted: false, - terminalState: 'invalid_request', - latencyMs: Date.now() - attemptStarted, - }) - return { + if (policy.reason === 'content_filter_refusal' || policy.reason === 'usage_policy_refusal') { + this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, countUsage }) + bindAll({ accountId: selected.accountId, vmId: selected.vmId }) + return { + ...result, + ...attribution(), + finalState: policy.reason === 'usage_policy_refusal' ? 'rejected' : 'content_filter', + policy, + } + } + if (verifiedSuccess(result)) { + if (repairKey && (knownSearchRepair || searchRepair?.key === repairKey)) { + this.searchRepairs.delete(repairKey) + while (this.searchRepairs.size >= SEARCH_REPAIR_LIMIT) { + this.searchRepairs.delete(this.searchRepairs.keys().next().value) + } + this.searchRepairs.set(repairKey, { + fingerprint: fingerprint(), + expiresAt: Date.now() + SEARCH_REPAIR_IDLE_MS, + }) + } + this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, countUsage }) + if (isFableModel(model) && typeof this.onFableSuccess === 'function') { + try { + this.onFableSuccess({ selected, model }) + } catch {} + } + bindAll({ accountId: selected.accountId, vmId: selected.vmId }) + return { ...result, ...attribution(), finalState: 'verified' } + } + if (policy.action === 'repair-and-retry' && !repaired && !result?.committed) { + repaired = true + repairPending = true + if (policy.reason === 'search_repairable' && repairKey && fingerprint()) { + searchRepair = { key: repairKey } + } + requestBody = repairAnthropicRequest(requestBody, policy) + continue + } + applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId }) + if (!pinVmId && isCredentialDeath(policy)) { + this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys }) + if (familyKey) { + return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } + } + } + if (!shouldContinue(policy)) { + return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } + } + policy = await this.recoverCredential(selected, policy) + const hopMs = Date.now() - attemptStarted + if (isRetryableEmptyHop(policy)) { + if (!emptyHopReleased(result) || pinVmId) return incompleteHopResult(result, policy, attribution()) + this.retireOutboundSession(selected, bindKeys) + freshSlot = true + } + if (budget.allowSameUnit(selected.accountId, policy, hopMs, this.config.same_account_retry_max_hop_ms)) { + budget.noteSameUnit(selected.accountId) + avoid = new Set([selected.accountId, selected.vmId]) + continue + } + budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) + } catch (error) { + const invalidCache = error?.status === 400 && error.body?.error?.code === 'invalid_cache_control' + noteHop(invalidCache ? { upstreamExecutions: 0 } : null) + if (error?.code === 'response_sent') { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { + upstreamStatus: 403, + errorScope: 'request', + action: 'stop', + downstreamCommitted: true, + terminalState: 'policy_block', + latencyMs: Date.now() - attemptStarted, + }) + return { + ok: false, + responseSent: true, + status: 403, + via: 'account-scope', + vmId: selected.vmId, + accountId: selected.accountId, + attemptCount: attemptNo, + finalState: 'policy_block', + } + } + if (invalidCache) { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { + upstreamStatus: 0, + errorScope: 'request', + action: 'stop', + downstreamCommitted: false, + terminalState: 'invalid_request', + latencyMs: Date.now() - attemptStarted, + }) + return { + ok: false, + status: 400, + body: error.body, + terminalState: 'invalid_request', + finalState: 'invalid_request', + ...attribution(), + } + } + if (signal?.aborted || error?.code === 'selection_cancelled' || error?.code === 'request_cancelled') { + await this.attemptsRepo?.complete?.(requestId, attemptNo, { + upstreamStatus: 0, + errorScope: 'request', + action: 'stop', + downstreamCommitted: !!committed, + terminalState: 'cancelled', + latencyMs: Date.now() - attemptStarted, + }) + return { ...clientCancelledResult({ committed }), via: 'pool-failover', ...attribution() } + } + result = { ok: false, - status: 400, - body: error.body, - terminalState: 'invalid_request', - finalState: 'invalid_request', - ...attribution(), + status: 0, + transportError: true, + committed, + terminalState: committed ? 'incomplete' : 'transport_error', + body: { + type: 'error', + error: { + type: 'worker_error', + code: error.code || 'attempt_failed', + message: String(error.message || error).slice(0, 300), + }, + }, } - } - if (signal?.aborted || error?.code === 'selection_cancelled' || error?.code === 'request_cancelled') { + policy = classifyAttempt( + result, + selected, + { + model, + repaired, + oauth401CooldownMs: this.config.oauth_401_cooldown_ms, + signatureRepair: signatureRepairEnabled(this.config), + }, + this.scheduler?.accountQuota, + ) + lastResult = result + lastPolicy = policy + this.noteUnitHealth(selected, policy, result) + notifyProxyFailure(this.onProxyFailure, selected, policy) await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, - errorScope: 'request', - action: 'stop', - downstreamCommitted: !!committed, - terminalState: 'cancelled', + errorScope: policy.scope, + action: policy.action, + cooldownUntil: policy.cooldownUntil, + downstreamCommitted: committed, + terminalState: result.terminalState, latencyMs: Date.now() - attemptStarted, }) - return { ...clientCancelledResult({ committed }), via: 'pool-failover', ...attribution() } - } - result = { - ok: false, - status: 0, - transportError: true, - committed, - terminalState: committed ? 'incomplete' : 'transport_error', - body: { - type: 'error', - error: { - type: 'worker_error', - code: error.code || 'attempt_failed', - message: String(error.message || error).slice(0, 300), - }, - }, - } - policy = classifyAttempt( - result, - selected, - { - model, - repaired, - oauth401CooldownMs: this.config.oauth_401_cooldown_ms, - signatureRepair: signatureRepairEnabled(this.config), - }, - this.scheduler?.accountQuota, - ) - lastResult = result - lastPolicy = policy - this.noteUnitHealth(selected, policy, result) - notifyProxyFailure(this.onProxyFailure, selected, policy) - await this.attemptsRepo?.complete?.(requestId, attemptNo, { - upstreamStatus: 0, - errorScope: policy.scope, - action: policy.action, - cooldownUntil: policy.cooldownUntil, - downstreamCommitted: committed, - terminalState: result.terminalState, - latencyMs: Date.now() - attemptStarted, - }) - if (committed || !shouldContinue(policy)) { - return { ...result, ...attribution(), finalState: result.terminalState, policy } - } - applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId }) - if (!pinVmId && isCredentialDeath(policy)) { - this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys }) - if (familyKey) { + if (committed || !shouldContinue(policy)) { return { ...result, ...attribution(), finalState: result.terminalState, policy } } + applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId }) + if (!pinVmId && isCredentialDeath(policy)) { + this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys }) + if (familyKey) { + return { ...result, ...attribution(), finalState: result.terminalState, policy } + } + } + policy = await this.recoverCredential(selected, policy) + const hopMs = Date.now() - attemptStarted + if (budget.allowSameUnit(selected.accountId, policy, hopMs, this.config.same_account_retry_max_hop_ms)) { + budget.noteSameUnit(selected.accountId) + avoid = new Set([selected.accountId, selected.vmId]) + continue + } + budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) } - policy = await this.recoverCredential(selected, policy) - const hopMs = Date.now() - attemptStarted - if (budget.allowSameUnit(selected.accountId, policy, hopMs, this.config.same_account_retry_max_hop_ms)) { - budget.noteSameUnit(selected.accountId) - avoid = new Set([selected.accountId, selected.vmId]) - continue - } - budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) } finally { - selected.release?.() + await selected?.release?.() } } } From 70c0b794105994374eda1505f2c389b9de95eedd Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:29:55 +0800 Subject: [PATCH 075/473] feat: persist shared execution concurrency and RPM admission --- .../005_execution_admissions.sql | 22 +++ .../db/repos/execution-admissions-repo.mjs | 145 ++++++++++++++++++ test/unit/execution-admissions.test.mjs | 69 ++++++++- 3 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 src/lib/db/postgres-migrations/005_execution_admissions.sql create mode 100644 src/lib/db/repos/execution-admissions-repo.mjs diff --git a/src/lib/db/postgres-migrations/005_execution_admissions.sql b/src/lib/db/postgres-migrations/005_execution_admissions.sql new file mode 100644 index 000000000..05cb4077f --- /dev/null +++ b/src/lib/db/postgres-migrations/005_execution_admissions.sql @@ -0,0 +1,22 @@ +CREATE TABLE execution_admissions ( + id text COLLATE "C" PRIMARY KEY, + owner_id text COLLATE "C" NOT NULL, + request_id text COLLATE "C" NOT NULL, + account_id text COLLATE "C" NOT NULL, + vm_id text COLLATE "C" NOT NULL, + model_family text COLLATE "C", + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + uncertain_at timestamptz, + released_at timestamptz, + seat_key text COLLATE "C", + seat_generation text COLLATE "C", + slot_index integer +); + +CREATE INDEX idx_execution_admissions_account_window ON execution_admissions (account_id, created_at); +CREATE INDEX idx_execution_admissions_account_active ON execution_admissions (account_id) WHERE released_at IS NULL; +CREATE INDEX idx_execution_admissions_vm_active ON execution_admissions (vm_id) WHERE released_at IS NULL; +CREATE INDEX idx_execution_admissions_seat_active ON execution_admissions (seat_key, seat_generation) WHERE released_at IS NULL; + +-- No TTL or cascading resource deletion can prove that execution ended. +-- Released rows remain RPM history until their complete minute has elapsed. diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs new file mode 100644 index 000000000..d6a86d80b --- /dev/null +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -0,0 +1,145 @@ +import crypto from 'node:crypto' +import { withTransaction } from '../database.mjs' + +function invalid(field) { + return Object.assign(new Error(`Invalid execution admission ${field}`), { code: 'invalid_execution_admission' }) +} + +function identifier(value, field) { + if (typeof value !== 'string' || !value.trim()) throw invalid(field) + return value +} + +function limit(value, field) { + if (!Number.isSafeInteger(value) || value < 0) throw invalid(field) + return value +} + +function denied(code, inflight, max, extra = {}) { + return { ok: false, code, reason: code, detail: { inflight, max }, ...extra } +} + +/** Short PostgreSQL transactions for global account/VM execution limits. */ +export class ExecutionAdmissionsRepo { + constructor(db, { ownerId = 'controller_' + crypto.randomUUID() } = {}) { + this.db = db + this.ownerId = identifier(ownerId, 'ownerId') + } + + /** Shared order: device (caller), account, then every involved VM sorted. */ + async lockResources({ accountId, vmIds = [] } = {}) { + if (!this.db.isTransaction) { + throw Object.assign(new Error('Execution resource locks require a transaction'), { + code: 'execution_transaction_required', + }) + } + if (!Array.isArray(vmIds)) throw invalid('vmIds') + const resources = vmIds.map((id) => 'vm:' + identifier(id, 'vmId')) + if (accountId != null) resources.push('account:' + identifier(accountId, 'accountId')) + for (const resource of [...new Set(resources)].sort()) { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ + `vm2api:execution:${this.db.schema}`, + resource, + ]) + } + } + + async lockVm(vmId) { + return this.lockResources({ vmIds: [vmId] }) + } + + async _counts(accountId, vmId, now) { + const row = await this.db + .prepare(`SELECT + count(*) FILTER (WHERE account_id = $1 AND released_at IS NULL) AS account_inflight, + count(*) FILTER (WHERE vm_id = $2 AND released_at IS NULL) AS vm_inflight, + count(*) FILTER (WHERE account_id = $1 AND released_at IS NULL AND model_family = 'fable') AS fable_inflight, + count(*) FILTER (WHERE account_id = $1 AND created_at > $3::timestamptz - interval '60 seconds') AS rpm_count, + min(created_at) FILTER (WHERE account_id = $1 AND created_at > $3::timestamptz - interval '60 seconds') AS rpm_oldest + FROM execution_admissions WHERE account_id = $1 OR vm_id = $2`) + .get(accountId, vmId, now) + return { + accountInflight: row.account_inflight, + vmInflight: row.vm_inflight, + fableInflight: row.fable_inflight, + rpmCount: row.rpm_count, + rpmResetAt: row.rpm_oldest ? row.rpm_oldest.getTime() + 60_000 : null, + } + } + + /** An eligibility snapshot only; tryReserve is the atomic admission gate. */ + async getCounts({ accountId, vmId } = {}) { + if (accountId != null) identifier(accountId, 'accountId') + if (vmId != null) identifier(vmId, 'vmId') + if (accountId == null && vmId == null) throw invalid('resource') + const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() + return this._counts(accountId ?? null, vmId ?? null, now) + } + + async tryReserve(input = {}) { + const accountId = identifier(input.accountId, 'accountId') + const vmId = identifier(input.vmId, 'vmId') + const ownerId = identifier(input.ownerId === undefined ? this.ownerId : input.ownerId, 'ownerId') + const requestId = identifier(input.requestId === undefined ? crypto.randomUUID() : input.requestId, 'requestId') + const accountLimit = limit(input.accountLimit, 'accountLimit') + const vmLimit = limit(input.vmLimit, 'vmLimit') + const fableLimit = limit(input.fableLimit, 'fableLimit') + const rpmLimit = limit(input.rpmLimit, 'rpmLimit') + const family = input.family == null ? null : identifier(input.family, 'family') + return withTransaction(this.db, async () => { + await this.lockResources({ accountId, vmIds: [vmId] }) + const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() + await this.db + .prepare(`DELETE FROM execution_admissions WHERE account_id = $1 + AND released_at IS NOT NULL AND created_at <= $2::timestamptz - interval '60 seconds'`) + .run(accountId, now) + const counts = await this._counts(accountId, vmId, now) + // Concurrency zero disables execution; RPM/Fable zero adds no sublimit. + if (counts.accountInflight >= accountLimit) + return denied('concurrency_limit', counts.accountInflight, accountLimit) + if (counts.vmInflight >= vmLimit) return denied('vm_concurrency_limit', counts.vmInflight, vmLimit) + if (family === 'fable' && fableLimit > 0 && counts.fableInflight >= fableLimit) { + return denied('fable_concurrency', counts.fableInflight, fableLimit) + } + if (rpmLimit > 0 && counts.rpmCount >= rpmLimit) { + return denied('rpm_limit', counts.rpmCount, rpmLimit, { + retry_after_ms: Math.max(1, counts.rpmResetAt - now.getTime()), + }) + } + const admissionId = crypto.randomUUID() + const record = await this.db + .prepare(`INSERT INTO execution_admissions (id, owner_id, request_id, account_id, vm_id, model_family, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING *`) + .get(admissionId, ownerId, requestId, accountId, vmId, family, now) + const handle = { admissionId, ownerId } + return { + ok: true, + admissionId, + ownerId, + requestId, + accountId, + vmId, + record, + release: (options) => this.release(handle, options), + markUncertain: () => this.markUncertain(handle), + } + }) + } + + async markUncertain({ admissionId, ownerId }) { + const result = await this.db + .prepare(`UPDATE execution_admissions SET uncertain_at = COALESCE(uncertain_at, clock_timestamp()) + WHERE id = $1 AND owner_id = $2 AND released_at IS NULL`) + .run(admissionId, ownerId) + return { uncertain: result.changes > 0 } + } + + async release(handle, { uncertain = false } = {}) { + if (uncertain) return this.markUncertain(handle) + const result = await this.db + .prepare(`UPDATE execution_admissions SET released_at = clock_timestamp() + WHERE id = $1 AND owner_id = $2 AND released_at IS NULL AND uncertain_at IS NULL`) + .run(handle.admissionId, handle.ownerId) + return { released: result.changes > 0 } + } +} diff --git a/test/unit/execution-admissions.test.mjs b/test/unit/execution-admissions.test.mjs index 7afd99ff9..c04ebaf72 100644 --- a/test/unit/execution-admissions.test.mjs +++ b/test/unit/execution-admissions.test.mjs @@ -1,6 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' -import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { createDatabase, getDb, withSavepoint, withTransaction } from '../../src/lib/db/database.mjs' async function replicas() { const { ExecutionAdmissionsRepo } = await import('../../src/lib/db/repos/execution-admissions-repo.mjs') @@ -196,3 +196,70 @@ test('a shared database failure never creates an in-memory fallback admission', await assert.rejects(second.tryReserve(candidate()), /database is closed/) assert.equal((await first.getCounts(candidate())).accountInflight, 0) }) + +test('simultaneous admissions atomically reserve account RPM across replicas', async () => { + const repos = await replicas() + const input = candidate({ accountLimit: 20, vmLimit: 20, rpmLimit: 3 }) + const gates = await Promise.all( + Array.from({ length: 12 }, (_, i) => repos[i % 2].tryReserve({ ...input, vmId: `vm-${i}` })), + ) + assert.equal(gates.filter((gate) => gate.ok).length, 3) + assert.ok(gates.filter((gate) => !gate.ok).every((gate) => gate.code === 'rpm_limit')) + await Promise.all(gates.filter((gate) => gate.ok).map((gate) => gate.release())) + assert.equal((await repos[1].getCounts(input)).rpmCount, 3) +}) + +test('simultaneous Fable requests share their sublimit across VMs and replicas', async () => { + const repos = await replicas() + const input = candidate({ accountLimit: 20, vmLimit: 20, fableLimit: 2, family: 'fable' }) + const gates = await Promise.all( + Array.from({ length: 12 }, (_, i) => repos[i % 2].tryReserve({ ...input, vmId: `vm-${i}` })), + ) + assert.equal(gates.filter((gate) => gate.ok).length, 2) + assert.ok(gates.filter((gate) => !gate.ok).every((gate) => gate.code === 'fable_concurrency')) + assert.equal((await repos[1].getCounts(input)).fableInflight, 2) + await Promise.all(gates.filter((gate) => gate.ok).map((gate) => gate.release())) +}) + +test('a refused seat can roll back admission in a savepoint and try another VM', async () => { + const [first, second] = await replicas() + let accepted + await withTransaction(first.db, async () => { + await assert.rejects( + withSavepoint(first.db, async () => { + const deniedSeat = await first.tryReserve(candidate({ accountLimit: 1, rpmLimit: 1 })) + assert.equal(deniedSeat.ok, true) + throw new Error('first VM has no seat') + }), + /first VM has no seat/, + ) + accepted = await first.tryReserve(candidate({ vmId: 'vm-other', accountLimit: 1, rpmLimit: 1 })) + assert.equal(accepted.ok, true) + }) + const counts = await second.getCounts(candidate({ vmId: 'vm-other' })) + assert.equal(counts.accountInflight, 1) + assert.equal(counts.rpmCount, 1) + await accepted.release() +}) + +test('resource locks compose with seat transactions and leave unrelated resources available', async () => { + const [first, second] = await replicas() + await assert.rejects(first.lockVm('vm-1'), { code: 'execution_transaction_required' }) + await withTransaction(first.db, async () => { + await first.lockResources({ accountId: 'account-1', vmIds: ['vm-2', 'vm-1'] }) + await first.lockVm('vm-1') + await withTransaction(second.db, async () => { + for (const [resource, expected] of [ + ['account:account-1', false], + ['vm:vm-1', false], + ['vm:vm-2', false], + ['vm:vm-unrelated', true], + ]) { + const result = await second.db + .prepare('SELECT pg_try_advisory_xact_lock(hashtext($1), hashtext($2)) AS locked') + .get(`vm2api:execution:${first.db.schema}`, resource) + assert.equal(result.locked, expected, resource) + } + }) + }) +}) From dc40866d7a3332e861574fb10aea67c9e1574839 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:30:00 +0800 Subject: [PATCH 076/473] test(k8s): specify cross-controller device seat admission --- test/unit/shared-execution-seats.test.mjs | 132 ++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 test/unit/shared-execution-seats.test.mjs diff --git a/test/unit/shared-execution-seats.test.mjs b/test/unit/shared-execution-seats.test.mjs new file mode 100644 index 000000000..ab864425b --- /dev/null +++ b/test/unit/shared-execution-seats.test.mjs @@ -0,0 +1,132 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' + +async function replicas(t) { + const { SharedExecutionAdmissions } = await import('../../src/lib/pool/shared-execution-admissions.mjs') + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + return [new SharedExecutionAdmissions(db, { ownerId: 'a' }), new SharedExecutionAdmissions(other, { ownerId: 'b' })] +} + +function candidate(patch = {}) { + return { + accountId: 'account-1', + vmId: 'vm-1', + accountLimit: 20, + vmLimit: 20, + rpmLimit: 0, + fableLimit: 0, + seatKey: 'platform|device-1', + seatCap: 2, + seatGraceMs: 0, + seatReservePct: 0.02, + seatHeadroom: null, + ...patch, + } +} + +test('one device cannot split across VMs on two controllers and shares its global slot index', async (t) => { + const [first, second] = await replicas(t) + const gates = await Promise.all([first.tryReserve(candidate()), second.tryReserve(candidate({ vmId: 'vm-2' }))]) + assert.equal(gates.filter((gate) => gate.ok).length, 1) + const held = gates.find((gate) => gate.ok) + const denied = gates.find((gate) => !gate.ok) + assert.equal(denied.code, 'device_seat_bound') + assert.equal(denied.vmId, held.vmId) + const another = await second.tryReserve(candidate({ vmId: held.vmId })) + assert.equal(another.ok, true) + assert.equal(another.slotIndex, held.slotIndex) + assert.equal(another.seatGeneration, held.seatGeneration) + assert.equal((await first.getSeat(candidate().seatKey)).inflight, 2) + await held.release() + assert.equal((await second.getSeat(candidate().seatKey)).inflight, 1) + await another.release() + assert.equal(await first.getSeat(candidate().seatKey), null) +}) + +test('seat capacity is atomic across devices and denied seats consume no account RPM', async (t) => { + const controllers = await replicas(t) + const gates = await Promise.all( + Array.from({ length: 12 }, (_, i) => + controllers[i % 2].tryReserve(candidate({ seatKey: `device-${i}`, seatCap: 1 })), + ), + ) + assert.equal(gates.filter((gate) => gate.ok).length, 1) + assert.ok(gates.filter((gate) => !gate.ok).every((gate) => gate.code === 'seat_limit')) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM execution_admissions').get()).n, 1) + await gates.find((gate) => gate.ok).release() +}) + +test('global seat indexes are distinct and a stale release cannot affect a newer generation', async (t) => { + const [first, second] = await replicas(t) + const a = await first.tryReserve(candidate()) + const b = await second.tryReserve(candidate({ seatKey: 'device-2' })) + assert.deepEqual(new Set([a.slotIndex, b.slotIndex]), new Set([0, 1])) + await a.release() + const next = await second.tryReserve(candidate()) + assert.equal(next.slotIndex, a.slotIndex) + assert.notEqual(next.seatGeneration, a.seatGeneration) + await a.release() + assert.equal((await first.getSeat(candidate().seatKey)).inflight, 1) + await next.release() + await b.release() +}) + +test('grace starts only after the last shared request ends and active seats cannot move', async (t) => { + const [first, second] = await replicas(t) + const a = await first.tryReserve(candidate({ seatGraceMs: 1000 })) + const b = await second.tryReserve(candidate({ seatGraceMs: 1000 })) + await a.release() + assert.equal((await first.getSeat(candidate().seatKey)).graceUntil, null) + const denied = await second.tryReserve(candidate({ vmId: 'vm-2', moveSeat: true })) + assert.equal(denied.code, 'device_seat_bound') + await b.release() + const grace = await first.getSeat(candidate().seatKey) + assert.equal(grace.inflight, 0) + assert.ok(grace.graceUntil > Date.now()) + assert.equal((await second.tryReserve(candidate({ vmId: 'vm-2' }))).code, 'device_seat_bound') + await getDb().prepare("UPDATE execution_seats SET grace_until = clock_timestamp() - interval '1 second'").run() + const moved = await second.tryReserve(candidate({ vmId: 'vm-2' })) + assert.equal(moved.ok, true) + assert.notEqual(moved.seatGeneration, grace.generation) + await moved.release() +}) + +test('new device budget uses global seat occupancy and existing devices need no new seat budget', async (t) => { + const [first, second] = await replicas(t) + const a = await first.tryReserve(candidate({ seatHeadroom: 0.03 })) + assert.equal(a.ok, true) + const denied = await second.tryReserve(candidate({ seatKey: 'device-2', seatHeadroom: 0.03 })) + assert.equal(denied.code, 'seat_budget') + const same = await second.tryReserve(candidate({ seatHeadroom: 0 })) + assert.equal(same.ok, true) + const next = await second.tryReserve(candidate({ seatKey: 'device-2', seatHeadroom: 0.04 })) + assert.equal(next.ok, true) + await Promise.all([a.release(), same.release(), next.release()]) +}) + +test('an uncertain execution keeps its device and VM capacity after release and expiry', async (t) => { + const [first, second] = await replicas(t) + const gate = await first.tryReserve(candidate({ vmLimit: 1 })) + await gate.markUncertain() + await gate.release() + await getDb().prepare("UPDATE execution_seats SET grace_until = clock_timestamp() - interval '1 hour'").run() + assert.equal((await second.getSeat(candidate().seatKey)).inflight, 1) + assert.equal((await second.tryReserve(candidate({ seatKey: 'device-2', vmLimit: 1 }))).code, 'vm_concurrency_limit') + assert.equal((await second.tryReserve(candidate({ vmId: 'vm-2', moveSeat: true }))).code, 'device_seat_bound') +}) + +test('seat release and execution release roll back together and can be retried', async (t) => { + const [first] = await replicas(t) + const gate = await first.tryReserve(candidate({ seatGraceMs: 1000 })) + await getDb().exec(`CREATE FUNCTION refuse_seat_release() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.grace_until IS NOT NULL THEN RAISE EXCEPTION 'synthetic release failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER refuse_seat_release BEFORE UPDATE ON execution_seats FOR EACH ROW EXECUTE FUNCTION refuse_seat_release()`) + await assert.rejects(gate.release()) + assert.equal((await first.getSeat(candidate().seatKey)).inflight, 1) + await getDb().exec('DROP TRIGGER refuse_seat_release ON execution_seats') + await gate.release() + assert.equal((await first.getSeat(candidate().seatKey)).inflight, 0) +}) From b9e5c7f91c25956b184f040bff1e10b6cf7fbf57 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:34:30 +0800 Subject: [PATCH 077/473] test: expose worker execution identity and termination certainty --- test/unit/worker-execution.test.mjs | 255 ++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 test/unit/worker-execution.test.mjs diff --git a/test/unit/worker-execution.test.mjs b/test/unit/worker-execution.test.mjs new file mode 100644 index 000000000..4ea590d65 --- /dev/null +++ b/test/unit/worker-execution.test.mjs @@ -0,0 +1,255 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { callGoWorker, streamGoWorker } from '../../src/lib/transport/go-worker-client.mjs' + +const unixTest = process.platform === 'win32' ? test.skip : test +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function fixture(t, handler) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-exec-')) + const socket = path.join(root, 'worker.sock') + const server = http.createServer(handler) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(socket, resolve) + }) + t.after(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + fs.rmSync(root, { recursive: true, force: true }) + }) + return { vmId: 'vm-execution', vm: { runtime: { worker_socket: socket } } } +} + +const envelope = { body: { model: 'claude-test', messages: [{ role: 'user', content: 'hi' }] }, headers: {} } + +for (const [name, invoke] of [ + ['HTTP', callGoWorker], + ['SSE', streamGoWorker], +]) { + unixTest( + `${name} exposes the actual hop ID and awaits durable binding before submitting`, + { timeout: 2000 }, + async (t) => { + let calls = 0 + let wireId + const entered = deferred() + const bound = deferred() + const exec = await fixture(t, async (req, res) => { + calls++ + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + wireId = JSON.parse(Buffer.concat(chunks).toString()).request_id + if (name === 'HTTP') res.end(JSON.stringify({ type: 'message', content: [{ type: 'text', text: 'ok' }] })) + else { + res.setHeader('content-type', 'text/event-stream') + res.end('data: {"type":"message_stop"}\n\n') + } + }) + let callbackId + const pending = invoke({ + exec, + envelope, + requestId: `hop-${name}`, + onExecutionStart: async ({ requestId }) => { + callbackId = requestId + entered.resolve() + await bound.promise + }, + }) + try { + await Promise.race([ + entered.promise, + pending.then(() => { + throw new Error('request finished without invoking durable binding') + }), + ]) + assert.equal(calls, 0) + bound.resolve() + const result = await pending + assert.equal(callbackId, `hop-${name}`) + assert.equal(wireId, callbackId) + assert.deepEqual(result.execution, { requestId: wireId, started: true, ended: true, reason: 'eof' }) + assert.equal(result.body.execution, undefined) + } finally { + bound.resolve() + await pending + } + }, + ) + + unixTest(`${name} pre-aborted requests are proven not started`, async (t) => { + let calls = 0 + const exec = await fixture(t, (_req, res) => { + calls++ + res.end('{}') + }) + const signal = AbortSignal.abort() + const result = await invoke({ + exec, + envelope, + signal, + requestId: 'pre-aborted', + onExecutionStart: () => { + calls++ + }, + }) + assert.deepEqual(result.execution, { requestId: 'pre-aborted', started: false, ended: true, reason: 'not_started' }) + assert.equal(calls, 0) + }) + + unixTest(`${name} binding failure prevents execution`, async (t) => { + let calls = 0 + const exec = await fixture(t, (_req, res) => { + calls++ + res.end('{}') + }) + const result = await invoke({ + exec, + envelope, + requestId: 'binding-failed', + onExecutionStart: () => { + throw new Error('cannot persist the attempt') + }, + }) + assert.deepEqual(result.execution, { + requestId: 'binding-failed', + started: false, + ended: true, + reason: 'not_started', + }) + assert.equal(calls, 0) + assert.equal(result.ok, false) + }) + + unixTest(`${name} request timeouts do not prove the kernel task ended`, async (t) => { + const exec = await fixture(t, async (req) => { + for await (const _chunk of req) { + } + }) + const result = await invoke({ exec, envelope, requestId: 'timed-out', timeoutMs: 25 }) + assert.deepEqual(result.execution, { requestId: 'timed-out', started: true, ended: false, reason: 'timeout' }) + assert.equal(result.transportError, true) + }) + + unixTest(`${name} truncated response bodies preserve execution uncertainty`, async (t) => { + const exec = await fixture(t, async (req, res) => { + for await (const _chunk of req) { + } + res.writeHead(200, { 'content-length': '1000' }) + res.write('{"type":"message"') + setImmediate(() => res.destroy()) + }) + const result = await invoke({ exec, envelope, requestId: 'truncated' }) + assert.deepEqual(result.execution, { + requestId: 'truncated', + started: true, + ended: false, + reason: 'transport_error', + }) + }) +} + +unixTest( + 'an abort returns before a delayed cancel response and never treats cancelled:true as termination', + { timeout: 2000 }, + async (t) => { + const started = deferred() + const sawCancel = deferred() + const respondCancel = deferred() + let cancelResponded = false + let wireId + const exec = await fixture(t, async (req, res) => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + const body = JSON.parse(Buffer.concat(chunks).toString()) + if (req.url === '/internal/v1/cancel') { + sawCancel.resolve(body.request_id) + await respondCancel.promise + cancelResponded = true + res.end('{"ok":true,"cancelled":true}') + return + } + wireId = body.request_id + res.setHeader('content-type', 'text/event-stream') + res.write('data: {"type":"message_start","message":{}}\n\n') + started.resolve() + }) + const controller = new AbortController() + const pending = streamGoWorker({ exec, envelope, signal: controller.signal, requestId: 'cancelled-hop' }) + try { + await started.promise + controller.abort() + const result = await pending + assert.equal(await sawCancel.promise, wireId) + assert.equal(cancelResponded, false) + assert.deepEqual(result.execution, { requestId: wireId, started: true, ended: false, reason: 'client_aborted' }) + respondCancel.resolve() + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(result.execution.ended, false) + } finally { + respondCancel.resolve() + } + }, +) + +unixTest('message_stop does not establish execution end before HTTP EOF', { timeout: 2000 }, async (t) => { + const eof = deferred() + const sawStop = deferred() + const exec = await fixture(t, async (req, res) => { + for await (const _chunk of req) { + } + res.setHeader('content-type', 'text/event-stream') + res.write('data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}\n\n') + res.write('data: {"type":"message_stop"}\n\n') + await eof.promise + res.end() + }) + let returned = false + const pending = streamGoWorker({ + exec, + envelope, + requestId: 'delayed-eof', + onEvent(line) { + if (line.includes('message_stop')) sawStop.resolve() + }, + }).then((result) => { + returned = true + return result + }) + try { + await sawStop.promise + assert.equal(returned, false) + eof.resolve() + assert.deepEqual((await pending).execution, { requestId: 'delayed-eof', started: true, ended: true, reason: 'eof' }) + } finally { + eof.resolve() + } +}) + +test('synthetic mock results are ended even when their assistant response is incomplete', async () => { + const before = { mock: process.env.KIN_CRS_MOCK, scenarios: process.env.KIN_MOCK_ACCOUNT_SCENARIOS } + process.env.KIN_CRS_MOCK = '1' + process.env.KIN_MOCK_ACCOUNT_SCENARIOS = JSON.stringify({ 'vm-mock': 'incomplete_stream' }) + try { + for (const invoke of [callGoWorker, streamGoWorker]) { + const result = await invoke({ exec: { vmId: 'vm-mock' }, body: envelope.body, requestId: 'mock-hop' }) + assert.deepEqual(result.execution, { requestId: 'mock-hop', started: false, ended: true, reason: 'mock' }) + } + } finally { + if (before.mock === undefined) delete process.env.KIN_CRS_MOCK + else process.env.KIN_CRS_MOCK = before.mock + if (before.scenarios === undefined) delete process.env.KIN_MOCK_ACCOUNT_SCENARIOS + else process.env.KIN_MOCK_ACCOUNT_SCENARIOS = before.scenarios + } +}) From 0481465763135674227bc709f6c77e3fc45026cc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:35:37 +0800 Subject: [PATCH 078/473] test(k8s): preserve the original seat when a move is denied --- test/unit/shared-execution-seats.test.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/unit/shared-execution-seats.test.mjs b/test/unit/shared-execution-seats.test.mjs index ab864425b..64618f9ac 100644 --- a/test/unit/shared-execution-seats.test.mjs +++ b/test/unit/shared-execution-seats.test.mjs @@ -130,3 +130,13 @@ test('seat release and execution release roll back together and can be retried', await gate.release() assert.equal((await first.getSeat(candidate().seatKey)).inflight, 0) }) + +test('a denied move preserves the original grace seat and its generation', async (t) => { + const [first, second] = await replicas(t) + const gate = await first.tryReserve(candidate({ seatGraceMs: 1000 })) + await gate.release() + const before = await first.getSeat(candidate().seatKey) + const denied = await second.tryReserve(candidate({ vmId: 'vm-2', moveSeat: true, vmLimit: 0 })) + assert.equal(denied.code, 'vm_concurrency_limit') + assert.deepEqual(await first.getSeat(candidate().seatKey), before) +}) From 50013ba197d8e2f91c8e52811ab29b923051495b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:37:04 +0800 Subject: [PATCH 079/473] feat: track worker execution identity and termination certainty --- src/lib/transport/go-worker-client.mjs | 51 +++++++++++++++++++-- test/unit/worker-execution.test.mjs | 62 ++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 8 deletions(-) diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index d0d3e5d5b..4a2ac40fc 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -530,7 +530,10 @@ export async function callGoWorker({ requestPath = '/internal/v1/messages', envelope = null, cliHop = false, + requestId = randomUUID(), + onExecutionStart, } = {}) { + const execution = { requestId, started: false, ended: true, reason: isCrsMock() ? 'mock' : 'not_started' } if (isCrsMock()) { const { body: outboundBody, headers } = finalizeWorkerPayload({ body, @@ -551,11 +554,12 @@ export async function callGoWorker({ usage: mock.body?.usage || null, model: mock.body?.model || null, stopReason: mock.body?.stop_reason || null, + execution, } } - const requestId = randomUUID() - const stopCancel = cancelOnAbort(exec, signal, requestId) + let stopCancel = () => {} try { + if (signal?.aborted) throw Object.assign(new Error('The operation was aborted'), { code: 'ABORT_ERR' }) const prepared = envelope ? { envelope, thread: null } : workerEnvelope({ @@ -571,6 +575,11 @@ export async function callGoWorker({ cliHop, requestContext, }) + // The caller can durably bind this ID before any bytes reach the kernel. + if (onExecutionStart) await onExecutionStart({ requestId }) + if (signal?.aborted) throw Object.assign(new Error('The operation was aborted'), { code: 'ABORT_ERR' }) + stopCancel = cancelOnAbort(exec, signal, requestId) + Object.assign(execution, { started: true, ended: false, reason: 'in_progress' }) const response = await workerRequest(exec, { method: 'POST', requestPath, @@ -579,6 +588,7 @@ export async function callGoWorker({ timeoutMs, }) const data = await readAll(response) + Object.assign(execution, { ended: true, reason: 'eof' }) const parsed = parseJson(data) const headers = mergeRateLimitHeaders(publicHeaders(response.headers)) const ok = response.statusCode >= 200 && response.statusCode < 300 && parsed?.type !== 'error' @@ -595,8 +605,16 @@ export async function callGoWorker({ terminalState: headers['x-terminal-state'] || null, transportError: false, threadBetas: prepared.thread, + execution, }) } catch (error) { + if (!execution.ended) { + execution.reason = signal?.aborted + ? 'client_aborted' + : error.code === 'worker_timeout' + ? 'timeout' + : 'transport_error' + } return { ok: false, status: 0, @@ -612,6 +630,7 @@ export async function callGoWorker({ headers: {}, terminalState: 'transport_error', transportError: true, + execution, } } finally { stopCancel() @@ -637,7 +656,10 @@ export async function streamGoWorker({ requestPath = '/internal/v1/messages', envelope = null, cliHop = false, + requestId = randomUUID(), + onExecutionStart, } = {}) { + const execution = { requestId, started: false, ended: true, reason: isCrsMock() ? 'mock' : 'not_started' } if (isCrsMock()) { const { body: outboundBody, headers } = finalizeWorkerPayload({ body, @@ -662,6 +684,7 @@ export async function streamGoWorker({ headers: {}, terminalState: 'incomplete', committed: false, + execution, } } if (typeof onCommit === 'function') onCommit() @@ -684,6 +707,7 @@ export async function streamGoWorker({ headers: {}, terminalState: 'incomplete', committed: true, + execution, } } const payload = mockCrsPayload({ scenario }) @@ -693,6 +717,7 @@ export async function streamGoWorker({ via: 'go-worker-mock-stream', terminalState: 'rejected', committed: false, + execution, } } let mockTtftMs = null @@ -716,15 +741,16 @@ export async function streamGoWorker({ stopReason: payload.body?.stop_reason || null, ttftMs: mockTtftMs, firstTokenAt: mockTtftMs == null ? null : mockStartedAt + mockTtftMs, + execution, } } let committed = false // Hop-relative TTFT. firstTokenAt is epoch ms so request logs subtract ingress t0. const startedAt = Date.now() let ttftMs = null - const requestId = randomUUID() - const stopCancel = cancelOnAbort(exec, signal, requestId) + let stopCancel = () => {} try { + if (signal?.aborted) throw Object.assign(new Error('The operation was aborted'), { code: 'ABORT_ERR' }) const prepared = envelope ? { envelope, thread: null } : workerEnvelope({ @@ -741,6 +767,10 @@ export async function streamGoWorker({ cliHop, requestContext, }) + if (onExecutionStart) await onExecutionStart({ requestId }) + if (signal?.aborted) throw Object.assign(new Error('The operation was aborted'), { code: 'ABORT_ERR' }) + stopCancel = cancelOnAbort(exec, signal, requestId) + Object.assign(execution, { started: true, ended: false, reason: 'in_progress' }) const response = await workerRequest(exec, { method: 'POST', requestPath, @@ -753,6 +783,7 @@ export async function streamGoWorker({ const headers = mergeRateLimitHeaders(publicHeaders(response.headers)) if ((response.statusCode || 0) < 200 || (response.statusCode || 0) >= 300) { const data = await readAll(response, 1024 * 1024) + Object.assign(execution, { ended: true, reason: 'eof' }) return restoreKernelErrorStatus({ ok: false, status: response.statusCode || 0, @@ -763,6 +794,7 @@ export async function streamGoWorker({ terminalState: headers['x-terminal-state'] || 'error', transportError: false, threadBetas: prepared.thread, + execution, }) } let buffer = '' @@ -868,6 +900,7 @@ export async function streamGoWorker({ await emitLine(line) } } + Object.assign(execution, { ended: true, reason: 'eof' }) if (buffer) { applyClaudeSSELineToMessage(buffer, assembler) await emitLine(buffer) @@ -903,17 +936,26 @@ export async function streamGoWorker({ terminalState, transportError: false, threadBetas: prepared.thread, + execution, }) } finally { if (idleTimer) clearInterval(idleTimer) } } catch (error) { + if (!execution.ended) { + execution.reason = signal?.aborted + ? 'client_aborted' + : error.code === 'worker_timeout' + ? 'timeout' + : 'transport_error' + } if (signal?.aborted) { return clientCancelledResult({ via: 'go-worker-stream', ttftMs, firstTokenAt: ttftMs == null ? null : startedAt + ttftMs, committed, + execution, }) } return { @@ -934,6 +976,7 @@ export async function streamGoWorker({ committed, terminalState: committed ? 'incomplete' : 'transport_error', transportError: true, + execution, } } finally { stopCancel() diff --git a/test/unit/worker-execution.test.mjs b/test/unit/worker-execution.test.mjs index 4ea590d65..d15ced943 100644 --- a/test/unit/worker-execution.test.mjs +++ b/test/unit/worker-execution.test.mjs @@ -132,6 +132,54 @@ for (const [name, invoke] of [ assert.equal(result.ok, false) }) + unixTest(`${name} generates a recoverable hop ID when none is provided`, async (t) => { + let wireId + let boundId + const exec = await fixture(t, async (req, res) => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + wireId = JSON.parse(Buffer.concat(chunks).toString()).request_id + res.end('{}') + }) + const result = await invoke({ + exec, + envelope, + onExecutionStart: ({ requestId }) => { + boundId = requestId + }, + }) + assert.match(boundId, /^[a-f0-9-]{36}$/) + assert.equal(result.execution.requestId, wireId) + assert.equal(wireId, boundId) + assert.equal(result.execution.ended, true) + }) + + unixTest(`${name} aborting during durable binding prevents submission`, async (t) => { + let calls = 0 + const controller = new AbortController() + const exec = await fixture(t, (_req, res) => { + calls++ + res.end('{}') + }) + const result = await invoke({ + exec, + envelope, + signal: controller.signal, + requestId: 'abort-during-bind', + onExecutionStart: async () => { + await Promise.resolve() + controller.abort() + }, + }) + assert.deepEqual(result.execution, { + requestId: 'abort-during-bind', + started: false, + ended: true, + reason: 'not_started', + }) + assert.equal(calls, 0) + }) + unixTest(`${name} request timeouts do not prove the kernel task ended`, async (t) => { const exec = await fixture(t, async (req) => { for await (const _chunk of req) { @@ -167,6 +215,7 @@ unixTest( const started = deferred() const sawCancel = deferred() const respondCancel = deferred() + const cancelCompleted = deferred() let cancelResponded = false let wireId const exec = await fixture(t, async (req, res) => { @@ -178,6 +227,7 @@ unixTest( await respondCancel.promise cancelResponded = true res.end('{"ok":true,"cancelled":true}') + cancelCompleted.resolve() return } wireId = body.request_id @@ -195,7 +245,8 @@ unixTest( assert.equal(cancelResponded, false) assert.deepEqual(result.execution, { requestId: wireId, started: true, ended: false, reason: 'client_aborted' }) respondCancel.resolve() - await new Promise((resolve) => setImmediate(resolve)) + await cancelCompleted.promise + assert.equal(cancelResponded, true) assert.equal(result.execution.ended, false) } finally { respondCancel.resolve() @@ -242,9 +293,12 @@ test('synthetic mock results are ended even when their assistant response is inc process.env.KIN_CRS_MOCK = '1' process.env.KIN_MOCK_ACCOUNT_SCENARIOS = JSON.stringify({ 'vm-mock': 'incomplete_stream' }) try { - for (const invoke of [callGoWorker, streamGoWorker]) { - const result = await invoke({ exec: { vmId: 'vm-mock' }, body: envelope.body, requestId: 'mock-hop' }) - assert.deepEqual(result.execution, { requestId: 'mock-hop', started: false, ended: true, reason: 'mock' }) + for (const scenario of ['text', 'error', 'rate_limit', 'incomplete_stream']) { + process.env.KIN_MOCK_ACCOUNT_SCENARIOS = JSON.stringify({ 'vm-mock': scenario }) + for (const invoke of [callGoWorker, streamGoWorker]) { + const result = await invoke({ exec: { vmId: 'vm-mock' }, body: envelope.body, requestId: 'mock-hop' }) + assert.deepEqual(result.execution, { requestId: 'mock-hop', started: false, ended: true, reason: 'mock' }) + } } } finally { if (before.mock === undefined) delete process.env.KIN_CRS_MOCK From f48c30c784d923c85c361ad481e0281af3acc078 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:37:04 +0800 Subject: [PATCH 080/473] feat(k8s): admit device seats and execution capacity atomically --- .../006_execution_seats.sql | 12 ++ src/lib/pool/shared-execution-admissions.mjs | 168 ++++++++++++++++++ src/server.mjs | 1 + test/unit/backup-service.test.mjs | 14 +- test/unit/database.test.mjs | 8 +- 5 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 src/lib/db/postgres-migrations/006_execution_seats.sql create mode 100644 src/lib/pool/shared-execution-admissions.mjs diff --git a/src/lib/db/postgres-migrations/006_execution_seats.sql b/src/lib/db/postgres-migrations/006_execution_seats.sql new file mode 100644 index 000000000..e4428aed5 --- /dev/null +++ b/src/lib/db/postgres-migrations/006_execution_seats.sql @@ -0,0 +1,12 @@ +CREATE TABLE execution_seats ( + seat_key text COLLATE "C" PRIMARY KEY, + generation text COLLATE "C" NOT NULL UNIQUE, + vm_id text COLLATE "C" NOT NULL, + account_id text COLLATE "C" NOT NULL, + slot_index integer NOT NULL CHECK (slot_index >= 0), + grace_ms integer NOT NULL CHECK (grace_ms >= 0 AND grace_ms <= 120000), + grace_until timestamptz, + UNIQUE (vm_id, slot_index) +); + +CREATE INDEX idx_execution_seats_vm ON execution_seats (vm_id); diff --git a/src/lib/pool/shared-execution-admissions.mjs b/src/lib/pool/shared-execution-admissions.mjs new file mode 100644 index 000000000..12c7e58df --- /dev/null +++ b/src/lib/pool/shared-execution-admissions.mjs @@ -0,0 +1,168 @@ +import crypto from 'node:crypto' +import { getDb, withTransaction } from '../db/database.mjs' +import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' +import { budgetAllows } from './seat-planner.mjs' + +function invalid(field) { + return Object.assign(new Error(`Invalid shared execution ${field}`), { code: 'invalid_execution_admission' }) +} + +/** Account/VM admission and device-seat ownership commit in the same short transaction. */ +export class SharedExecutionAdmissions { + constructor(db = getDb(), options = {}) { + this.db = db + this.executions = new ExecutionAdmissionsRepo(db, options) + } + + async lockDevice(seatKey) { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ + `vm2api:execution:${this.db.schema}`, + `device:${seatKey}`, + ]) + } + + async readSeat(seatKey) { + return this.db + .prepare(`SELECT s.*, + (SELECT count(*) FROM execution_admissions a WHERE a.seat_key = s.seat_key + AND a.seat_generation = s.generation AND a.released_at IS NULL) AS inflight, + clock_timestamp() AS database_now + FROM execution_seats s WHERE s.seat_key = $1`) + .get(seatKey) + } + + async getSeat(seatKey) { + const row = await this.readSeat(seatKey) + if (!row || (!row.inflight && (!row.grace_until || row.grace_until <= row.database_now))) return null + return { + seatKey: row.seat_key, + vmId: row.vm_id, + accountId: row.account_id, + slotIndex: row.slot_index, + generation: row.generation, + inflight: row.inflight, + graceUntil: row.grace_until?.getTime() ?? null, + } + } + + async getCounts(accountId, vmId) { + return this.executions.getCounts({ accountId, vmId }) + } + + async tryReserve(input) { + const { seatKey } = input + if (seatKey == null) return this.executions.tryReserve(input) + if (typeof seatKey !== 'string' || !seatKey.trim()) throw invalid('seatKey') + if (!Number.isSafeInteger(input.seatCap) || input.seatCap < 0 || input.seatCap > 20) throw invalid('seatCap') + if (!Number.isSafeInteger(input.seatGraceMs) || input.seatGraceMs < 0 || input.seatGraceMs > 120000) + throw invalid('seatGraceMs') + if (!Number.isFinite(input.seatReservePct) || input.seatReservePct < 0 || input.seatReservePct > 1) + throw invalid('seatReservePct') + if (input.seatHeadroom != null && !Number.isFinite(input.seatHeadroom)) throw invalid('seatHeadroom') + const gate = await withTransaction(this.db, async () => { + await this.lockDevice(seatKey) + const before = await this.readSeat(seatKey) + // A move involves two VM books. Lock both in the same order as admissions. + await this.executions.lockResources({ + accountId: input.accountId, + vmIds: [input.vmId, ...(before ? [before.vm_id] : [])], + }) + await this.db + .prepare(`DELETE FROM execution_seats s WHERE s.vm_id = ANY($1::text[]) + AND (s.grace_until IS NULL OR s.grace_until <= clock_timestamp()) + AND NOT EXISTS (SELECT 1 FROM execution_admissions a WHERE a.seat_key = s.seat_key + AND a.seat_generation = s.generation AND a.released_at IS NULL)`) + .run([input.vmId, ...(before ? [before.vm_id] : [])]) + let seat = await this.readSeat(seatKey) + let previousSeat + if (seat && (seat.vm_id !== input.vmId || seat.account_id !== input.accountId)) { + if (seat.inflight || !input.moveSeat) { + return { + ok: false, + code: 'device_seat_bound', + reason: 'device_seat_bound', + vmId: seat.vm_id, + slotIndex: seat.slot_index, + } + } + previousSeat = seat + seat = null + } + let slotIndex = seat?.slot_index + if (!seat) { + const { count } = await this.db + .prepare('SELECT count(*) AS count FROM execution_seats WHERE vm_id = $1 AND seat_key <> $2') + .get(input.vmId, seatKey) + if (count >= input.seatCap) return { ok: false, code: 'seat_limit', reason: 'seat_limit' } + if (!budgetAllows(input.seatHeadroom, count, input.seatReservePct)) + return { ok: false, code: 'seat_budget', reason: 'seat_budget' } + const free = await this.db + .prepare(`SELECT n AS slot_index FROM generate_series(0, $2::integer - 1) AS n + WHERE NOT EXISTS (SELECT 1 FROM execution_seats WHERE vm_id = $1 AND slot_index = n AND seat_key <> $3) + ORDER BY n LIMIT 1`) + .get(input.vmId, input.seatCap, seatKey) + if (!free) return { ok: false, code: 'seat_limit', reason: 'seat_limit' } + slotIndex = free.slot_index + } + const admission = await this.executions.tryReserve(input) + if (!admission.ok) return admission + if (previousSeat) { + await this.db + .prepare('DELETE FROM execution_seats WHERE seat_key = $1 AND generation = $2') + .run(seatKey, previousSeat.generation) + } + const generation = seat?.generation || crypto.randomUUID() + if (seat) { + await this.db + .prepare('UPDATE execution_seats SET grace_until = NULL, grace_ms = $2 WHERE seat_key = $1') + .run(seatKey, input.seatGraceMs) + } else { + await this.db + .prepare(`INSERT INTO execution_seats (seat_key, generation, vm_id, account_id, slot_index, grace_ms) + VALUES ($1,$2,$3,$4,$5,$6)`) + .run(seatKey, generation, input.vmId, input.accountId, slotIndex, input.seatGraceMs) + } + await this.db + .prepare('UPDATE execution_admissions SET seat_key = $2, seat_generation = $3, slot_index = $4 WHERE id = $1') + .run(admission.admissionId, seatKey, generation, slotIndex) + return { ...admission, slotIndex, seatGeneration: generation, seatKey } + }) + if (!gate.ok) return gate + return { + ...gate, + release: (options) => this.release(gate, options), + markUncertain: () => this.executions.markUncertain(gate), + } + } + + async release(handle, options = {}) { + if (options.uncertain) return this.executions.markUncertain(handle) + return withTransaction(this.db, async () => { + const row = await this.db + .prepare('SELECT * FROM execution_admissions WHERE id = $1 AND owner_id = $2') + .get(handle.admissionId, handle.ownerId) + if (!row) return { released: false } + if (row.seat_key) await this.lockDevice(row.seat_key) + const seat = row.seat_key ? await this.readSeat(row.seat_key) : null + await this.executions.lockResources({ + accountId: row.account_id, + vmIds: [row.vm_id, ...(seat ? [seat.vm_id] : [])], + }) + const result = await this.executions.release(handle) + if (!result.released || !seat || seat.generation !== row.seat_generation) return result + const after = await this.readSeat(row.seat_key) + if (after.inflight) return result + if (after.grace_ms > 0) { + await this.db + .prepare(`UPDATE execution_seats SET grace_until = clock_timestamp() + $3::integer * interval '1 millisecond' + WHERE seat_key = $1 AND generation = $2`) + .run(row.seat_key, row.seat_generation, after.grace_ms) + } else { + await this.db + .prepare('DELETE FROM execution_seats WHERE seat_key = $1 AND generation = $2') + .run(row.seat_key, row.seat_generation) + } + return result + }) + } +} diff --git a/src/server.mjs b/src/server.mjs index 23850e17d..aa2d06255 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1538,6 +1538,7 @@ function maintenanceMonitors() { } async function flushRuntimeWrites() { + await poolScheduler?.flush?.() await requestLog.flush?.() await flushPendingWrites() await accountQuota.flush() diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 7ed1de750..f20ef95c2 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,7 +79,7 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets; DELETE FROM schema_migrations WHERE version <> '001'", + "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() @@ -90,16 +90,22 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current await new SettingsRepo(db).set('older-snapshot', 'after') assert.equal((await svc.restoreBackup(rec.id)).ok, true) assert.equal(await new SettingsRepo(db).get('older-snapshot'), 'before') - for (const name of ['panel_sessions', 'api_key_admissions', 'gateway_rate_buckets']) { + for (const name of [ + 'panel_sessions', + 'api_key_admissions', + 'gateway_rate_buckets', + 'execution_admissions', + 'execution_seats', + ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 4) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 6) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets; DELETE FROM schema_migrations WHERE version <> '001'", + "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index ac062cb8c..ed1f78e06 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 33) + assert.equal(rows.length, 35) for (const name of [ 'settings', 'users', @@ -48,6 +48,8 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'panel_sessions', 'api_key_admissions', 'gateway_rate_buckets', + 'execution_admissions', + 'execution_seats', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -108,7 +110,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 4) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 6) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -124,7 +126,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 4) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 6) fs.rmSync(dir, { recursive: true, force: true }) }) From 2a38a4da89e9023dde41f711200659b2c2f8610d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:38:15 +0800 Subject: [PATCH 081/473] feat(pool): use asynchronous shared execution admission in Kubernetes --- src/lib/pool/pool-scheduler.mjs | 291 +++++++++++++++++++++-- src/lib/pool/seat-planner.mjs | 100 +++++++- test/unit/pool-scheduler-shared.test.mjs | 280 +++++++++++++++++++++- 3 files changed, 631 insertions(+), 40 deletions(-) diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 4dfec7eff..218c00e56 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -51,6 +51,9 @@ import { SeatPlanner, rankSeatCandidates } from './seat-planner.mjs' import { GATE_DRAINING } from './quota-gate.mjs' import { activeHoursOf, evaluateActiveHours } from './active-hours.mjs' import { vmNodeReady } from '../cluster/placement.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { getDb } from '../db/database.mjs' +import { SharedExecutionAdmissions } from './shared-execution-admissions.mjs' const WAIT_TIMEOUT_MIN_MS = 1000 const WAIT_TIMEOUT_MAX_MS = 120000 @@ -267,6 +270,7 @@ export const SOFT_COOLDOWN_REASONS = new Set(['rpm_limited', 'rate_limited_unkno function stickyShouldWait(waitReason, cooldownReason = null) { if ( waitReason === 'concurrency_limit' || + waitReason === 'vm_concurrency_limit' || waitReason === 'fable_concurrency' || waitReason === 'rpm_limit' || waitReason === 'slot_busy' || @@ -283,6 +287,7 @@ function stickyShouldWait(waitReason, cooldownReason = null) { /** Waits that reserve() re-checks live, so a queued grant may try them without a fresh eligibility read. */ const LIVE_RECHECKED_WAITS = new Set([ 'concurrency_limit', + 'vm_concurrency_limit', 'fable_concurrency', 'rpm_limit', 'slot_busy', @@ -326,6 +331,7 @@ export class PoolScheduler extends EventEmitter { accountQuota = null, runtimeRepo = null, workerHealth = null, + executionAdmissions = null, config = {}, } = {}) { super() @@ -336,6 +342,12 @@ export class PoolScheduler extends EventEmitter { this.accountQuota = accountQuota this.runtimeRepo = runtimeRepo this.workerHealth = workerHealth + this.shared = isKubernetesMode() + this.executionAdmissions = this.shared + ? executionAdmissions || new SharedExecutionAdmissions(accountQuota?.db || getDb()) + : null + this.tasks = new Set() + this.failure = null this.config = normalizePoolConfig(config) this.lastStickyCleared = false this.inflight = new Map() @@ -352,6 +364,7 @@ export class PoolScheduler extends EventEmitter { strategy: this.config.strategy, stickyWaitMs: () => this.config.sticky_wait_timeout_ms, onChange: () => this.emit('change'), + asyncGrants: this.shared, }) this.unitCircuit = unitCircuit if (runtimeRepo) this.unitCircuit.bindRepo(runtimeRepo) @@ -452,16 +465,23 @@ export class PoolScheduler extends EventEmitter { if (this.lastStickyCleared) stickyCleared = true const reserveMisses = [] const attempted = new Set() + const attemptKey = (candidate) => (this.shared ? candidate.vmId : candidate.accountId) // Free capacity first: a sticky or preferred miss scans every other // candidate before anything waits. Borrowing never moves the durable pin. while (selected) { - const reservation = this.reserve(selected, { skipQuota: pinned, pinned }) - if (reservation) return finishReserve(selected, reservation) + const reservation = await this.reserve(selected, { skipQuota: pinned, pinned }) + if (reservation) { + if (signal?.aborted) { + await reservation.release() + throw makeAbortError() + } + return finishReserve(selected, reservation) + } reserveMisses.push({ ...selected, busy: true, waitReason: selected.waitReason || 'concurrency_limit' }) - attempted.add(selected.accountId) + attempted.add(attemptKey(selected)) const remaining = preferred.filter( (candidate) => - !attempted.has(candidate.accountId) && !blocked.has(candidate.accountId) && !blocked.has(candidate.vmId), + !attempted.has(attemptKey(candidate)) && !blocked.has(candidate.accountId) && !blocked.has(candidate.vmId), ) if (!remaining.length) break selected = this.pick(remaining, { @@ -473,12 +493,12 @@ export class PoolScheduler extends EventEmitter { } const effectiveCandidates = reserveMisses.length ? candidates.map((candidate) => { - const missed = reserveMisses.find((item) => item.accountId === candidate.accountId) + const missed = reserveMisses.find((item) => attemptKey(item) === attemptKey(candidate)) return missed || candidate }) : candidates const effectiveAvailable = reserveMisses.length - ? available.filter((candidate) => !attempted.has(candidate.accountId)) + ? available.filter((candidate) => !attempted.has(attemptKey(candidate))) : available const waitCandidates = reserveMisses.length ? effectiveCandidates : candidates const waitAvailable = reserveMisses.length ? effectiveAvailable : available @@ -509,7 +529,10 @@ export class PoolScheduler extends EventEmitter { return fail('all_accounts_busy', waitCandidates, waitAvailable, waitPool) } const waitCap = Math.min(waitDeadline, loopDeadline) - const sliceDeadline = wakeAts.length ? Math.min(waitCap, ...wakeAts) : waitCap + const sliceDeadline = Math.min( + wakeAts.length ? Math.min(waitCap, ...wakeAts) : waitCap, + this.shared ? Date.now() + QUEUE_REFRESH_MS : Infinity, + ) const waited = await this.waitForCapacity({ signal, deadline: sliceDeadline, @@ -575,6 +598,7 @@ export class PoolScheduler extends EventEmitter { } } const finish = (granted) => { + if (signal?.aborted) throw makeAbortError() consumed = true return { ...granted.candidate, @@ -591,6 +615,7 @@ export class PoolScheduler extends EventEmitter { try { for (;;) { if (signal?.aborted) throw makeAbortError() + if (ticket?.error) throw ticket.error if (ticket?.granted) return finish(ticket.granted) candidates = await this.eligibleCandidates({ model, excluded, signal, ownerScope, keyScope, quotaRequest }) // A grant can land while eligibility was being re-read. @@ -613,7 +638,8 @@ export class PoolScheduler extends EventEmitter { this.planner.toGlobal(ticket) ticket.deadline = Math.min(failoverDeadline, now + this.config.fallback_wait_timeout_ms) } - let seat = this.planner.seatOf(seatKey) + let moveSeat = false + let seat = this.shared ? await this.executionAdmissions.getSeat(seatKey) : this.planner.seatOf(seatKey) if (seat) { const home = byVm.get(seat.vmId) const homeUsable = !!home && (!home.busy || stickyShouldWait(home.waitReason, home.cooldownReason)) @@ -621,7 +647,8 @@ export class PoolScheduler extends EventEmitter { // same-unit retry with a free seat elsewhere moves the device now, without grace. const steerAway = homeUsable && avoided(home) && this.openableCandidates(candidates, avoided).length > 0 if (!homeUsable || steerAway) { - this.planner.free(seat) + if (this.shared) moveSeat = true + else this.planner.free(seat) seatMoved = true seat = null } else if (avoided(home) && !allowWait) { @@ -631,8 +658,14 @@ export class PoolScheduler extends EventEmitter { if (seat) { if (!ticket) { if (!this.planner.hasConcWaiters(seat.vmId)) { - const granted = this.claimSeat(seatKey, byVm.get(seat.vmId), 'seat-held') - if (granted) return finish(granted) + const granted = await this.claimSeat(seatKey, byVm.get(seat.vmId), 'seat-held') + if (granted) { + if (signal?.aborted) { + await granted.reservation.release() + throw makeAbortError() + } + return finish(granted) + } } if (!allowWait || now >= failoverDeadline) return fail('all_accounts_busy') ticket = this.enqueueTicket({ @@ -649,8 +682,14 @@ export class PoolScheduler extends EventEmitter { } else { const stickyVms = [familyGated ? '' : familyVm, deviceVm, sticky.vmId || ''].filter(Boolean) if (!ticket) { - const granted = this.openNewSeat(seatKey, candidates, { avoided, stickyVms }) - if (granted) return finish(granted) + const granted = await this.openNewSeat(seatKey, candidates, { avoided, stickyVms, moveSeat }) + if (granted) { + if (signal?.aborted) { + await granted.reservation.release() + throw makeAbortError() + } + return finish(granted) + } if (!candidates.length) { const empty = emptyPoolFailure(model, candidates) return { ...fail(empty.reason), retry_after_ms: empty.retry_after_ms } @@ -680,7 +719,10 @@ export class PoolScheduler extends EventEmitter { this.planner.place(ticket, { kind: 'global', vmId: null }) ticket.deadline = Math.min(failoverDeadline, now + this.config.fallback_wait_timeout_ms) } - for (const vmId of ticket.vmId ? [ticket.vmId] : byVm.keys()) this.planner.pump(vmId) + ticket.moveSeat = moveSeat + for (const vmId of ticket.vmId ? [ticket.vmId] : byVm.keys()) { + this.planner.pump(vmId) + } } if (ticket.granted) return finish(ticket.granted) const until = ticket.kind === 'seat' ? ticket.stickyUntil : ticket.deadline @@ -697,7 +739,7 @@ export class PoolScheduler extends EventEmitter { } finally { this.planner.remove(ticket) // Granted while the caller went away: hand the seat and the slot back. - if (ticket?.granted && !consumed) ticket.granted.reservation.release() + if (ticket?.granted && !consumed) await ticket.granted.reservation.release() } } @@ -711,7 +753,7 @@ export class PoolScheduler extends EventEmitter { (candidate) => !avoided(candidate) && this.isReservable(candidate) && - this.planner.canOpen(candidate.vmId, candidate.seatCap, candidate.headroom) && + (this.shared || this.planner.canOpen(candidate.vmId, candidate.seatCap, candidate.headroom)) && !this.planner.hasSeatWaiters(candidate.vmId), ) const open = openable.filter((candidate) => !candidate.draining) @@ -719,7 +761,8 @@ export class PoolScheduler extends EventEmitter { } /** Sticky VMs first (family, device, session), then the configured strategy. */ - openNewSeat(seatKey, candidates, { avoided, stickyVms = [] }) { + openNewSeat(seatKey, candidates, { avoided, stickyVms = [], moveSeat = false }) { + if (this.shared) return this.openNewSeatShared(seatKey, candidates, { avoided, stickyVms, moveSeat }) const openable = this.openableCandidates(candidates, avoided) if (!openable.length) return null for (const vmId of stickyVms) { @@ -734,12 +777,80 @@ export class PoolScheduler extends EventEmitter { return null } + async openNewSeatShared(seatKey, candidates, { avoided, stickyVms, moveSeat }) { + const openable = this.openableCandidates(candidates, avoided) + const preferred = stickyVms.map((vmId) => openable.find((candidate) => candidate.vmId === vmId)).filter(Boolean) + const ordered = [...preferred, ...this.planner.rank(openable).filter((candidate) => !preferred.includes(candidate))] + for (const candidate of ordered) { + const granted = await this.claimSeatShared( + seatKey, + candidate, + preferred.includes(candidate) ? 'sticky' : this.planner.strategy, + { moveSeat }, + ) + if (granted) return granted + } + return null + } + + async claimSeatShared(seatKey, candidate, selectionReason, { moveSeat = false } = {}) { + if (!candidate) return null + const reservation = await this.reserve(candidate, { + seatKey, + seatCap: candidate.seatCap, + seatHeadroom: candidate.headroom, + seatGraceMs: this.config.seat_grace_ms, + seatReservePct: this.config.seat_budget_reserve_pct, + moveSeat, + }) + if (!reservation) return null + let seat = this.planner.seatOf(seatKey) + if ( + seat && + (seat.vmId !== candidate.vmId || + seat.index !== reservation.slotIndex || + seat.generation !== reservation.seatGeneration) + ) { + this.planner.detachSeat(seat) + seat = null + } + const opened = !seat + seat ||= this.planner.open(seatKey, { + vmId: candidate.vmId, + accountId: candidate.accountId, + cap: candidate.seatCap, + index: reservation.slotIndex, + }) + seat.generation = reservation.seatGeneration + this.planner.hold(seat) + const releaseRequest = reservation.release + let released = false + return { + candidate, + selectionReason, + seatOpened: opened, + reservation: { + ...reservation, + release: (options) => + this.track( + (async () => { + await releaseRequest(options) + if (released || reservation.uncertain) return + released = true + this.planner.release(seat) + })(), + ), + }, + } + } + /** * Synchronous claim: the reservation (concurrency, quota/RPM, circuit) and * the seat land together or not at all. The device's existing seat must be * on this VM; otherwise a new one opens only under cap and budget. */ claimSeat(seatKey, candidate, selectionReason) { + if (this.shared) return this.claimSeatShared(seatKey, candidate, selectionReason) if (!candidate || !grantable(candidate)) return null const current = this.planner.seatOf(seatKey) if (current && current.vmId !== candidate.vmId) return null @@ -785,6 +896,11 @@ export class PoolScheduler extends EventEmitter { accepts: (id) => ticket.kind === 'global' && ticket.candidates.has(id), vmIds: () => [...ticket.candidates.keys()], tryGrant: (id) => { + if (this.shared) { + return this.claimSeatShared(ticket.seatKey, ticket.candidates.get(id), 'queued', { + moveSeat: !!ticket.moveSeat, + }) + } const granted = this.claimSeat(ticket.seatKey, ticket.candidates.get(id), 'queued') if (!granted) return false ticket.granted = granted @@ -805,7 +921,8 @@ export class PoolScheduler extends EventEmitter { } const done = () => { cleanup() - resolve() + if (ticket.error) reject(ticket.error) + else resolve() } const onAbort = () => { cleanup() @@ -918,7 +1035,7 @@ export class PoolScheduler extends EventEmitter { eligibility.account, parseConcurrency(this.config.default_max_per_account, 2), ) - const inflight = this.inflight.get(accountId) || 0 + const inflight = this.shared ? eligibility.sharedCounts.accountInflight : this.inflight.get(accountId) || 0 const seatCap = seatCapOf(vm, this.config.default_session_slots) this.planner.noteCap(vm.id, seatCap) candidates.push({ @@ -944,6 +1061,7 @@ export class PoolScheduler extends EventEmitter { waitReason: eligibility.waitReason || null, cooldownReason: state?.cooldown_reason || null, quotaRequest, + sharedCounts: eligibility.sharedCounts, exec: this.executionContext(vm, accountId), }) } @@ -1084,6 +1202,8 @@ export class PoolScheduler extends EventEmitter { const fallbackCap = parseConcurrency(this.config.default_max_per_account, 2) const maxConcurrency = this.effectiveMaxConcurrency(vm, account, fallbackCap) if (maxConcurrency <= 0) return { ok: false, reason: 'concurrency_disabled' } + const sharedCounts = this.shared ? await this.executionAdmissions.getCounts(accountId, vm.id) : null + const sharedLimits = this.shared ? this.executionLimits({ vm, account, maxConcurrency, model }) : null let busy = false let availableAt = null let waitReason = null @@ -1115,18 +1235,21 @@ export class PoolScheduler extends EventEmitter { markWait(reason, until) } } - const inflight = this.inflight.get(accountId) || 0 + const inflight = this.shared ? sharedCounts.accountInflight : this.inflight.get(accountId) || 0 if (circuitProbeUntil) markWait('circuit_probe', circuitProbeUntil) - if (inflight >= maxConcurrency) markWait('concurrency_limit') + if (inflight >= (sharedLimits?.accountLimit ?? maxConcurrency)) markWait('concurrency_limit') + if (this.shared && sharedCounts.vmInflight >= sharedLimits.vmLimit) markWait('vm_concurrency_limit') const fableCap = Number(this.config.fable_max_per_account) if (isFableModel(modelKey) && Number.isFinite(fableCap) && fableCap > 0) { - const familyInflight = this.familyInflight(accountId, FABLE_FAMILY_KEY) + const familyInflight = this.shared ? sharedCounts.fableInflight : this.familyInflight(accountId, FABLE_FAMILY_KEY) if (familyInflight >= fableCap) markWait('fable_concurrency') } let gateZone = null if (this.accountQuota && !pinned) { const quotaGate = this.accountQuota.canAccept(accountId, { tier: vmTierOf(vm), quotaRequest }) - if (!quotaGate.ok) { + const localCapacityGate = + this.shared && ['concurrency_limit', 'rpm_limit', 'quota_band_inflight'].includes(quotaGate.reason) + if (!quotaGate.ok && !localCapacityGate) { if (quotaGate.reason === 'concurrency_limit') markWait('concurrency_limit') else if (quotaGate.reason === 'rpm_limit') markWait('rpm_limit', quotaGate.detail?.reset_at) else if (quotaGate.reason === 'quota_band_inflight') markWait('quota_band_inflight') @@ -1134,6 +1257,10 @@ export class PoolScheduler extends EventEmitter { } gateZone = this.accountQuota.gateZoneOf?.(accountId, { tier: vmTierOf(vm) }) || null } + if (this.shared && sharedLimits.rpmLimit > 0 && sharedCounts.rpmCount >= sharedLimits.rpmLimit) { + markWait('rpm_limit', sharedCounts.rpmResetAt) + } + if (this.shared && gateZone === GATE_DRAINING && inflight > 0) markWait('quota_band_inflight') // A sticky preference must not bypass exhausted kernel recovery. if (kernelFaults.has(vm.id)) return { ok: false, reason: 'kernel_faulted' } // Node offline or asleep: the relay has no link behind it, a pick would only fail. Back on reconnect. @@ -1143,7 +1270,7 @@ export class PoolScheduler extends EventEmitter { } else if (workerStatus && rustKernelProcessUp(workerStatus) && !rustKernelReachable(workerStatus)) { return { ok: false, reason: 'worker_unhealthy' } } - return { ok: true, account, workerStatus, busy, availableAt, waitReason, headroom, gateZone } + return { ok: true, account, workerStatus, busy, availableAt, waitReason, headroom, gateZone, sharedCounts } } executionContext(vm, accountId) { @@ -1414,7 +1541,9 @@ export class PoolScheduler extends EventEmitter { * release() is idempotent and hands freed concurrency to the VM's queue. * Seats are layered on top by claimSeat(). */ - reserve(candidate, { skipQuota = false, pinned = false } = {}) { + reserve(candidate, options = {}) { + if (this.shared) return this.track(this.reserveShared(candidate, options)) + const { skipQuota = false, pinned = false } = options const requestInflight = this.inflight.get(candidate.accountId) || 0 if (!candidate.maxConcurrency || requestInflight >= candidate.maxConcurrency) return null const family = isFableModel(candidate.model) ? FABLE_FAMILY_KEY : null @@ -1462,6 +1591,117 @@ export class PoolScheduler extends EventEmitter { } } + executionLimits(candidate) { + const policy = this.accountQuota?.policyFor?.(candidate.account, { tier: vmTierOf(candidate.vm) }) + let accountLimit = Number(this.accountQuota?.limitFor?.(candidate.account, policy) ?? candidate.maxConcurrency) + if (candidate.draining) accountLimit = Math.min(accountLimit, 1) + return { + accountLimit, + vmLimit: maxConcurrencyOf(candidate.vm, candidate.maxConcurrency), + rpmLimit: Number(this.accountQuota?.rpmLimitFor?.(candidate.account, policy) ?? candidate.account?.max_rpm ?? 0), + fableLimit: Number(this.config.fable_max_per_account) || 0, + family: isFableModel(candidate.model) ? FABLE_FAMILY_KEY : null, + } + } + + async reserveShared(candidate, options) { + if (!options.skipQuota) { + const gate = this.accountQuota?.canAccept?.(candidate.accountId, { + tier: vmTierOf(candidate.vm), + quotaRequest: candidate.quotaRequest, + }) + if (gate && !gate.ok && !['concurrency_limit', 'rpm_limit', 'quota_band_inflight'].includes(gate.reason)) + return null + } + const circuitHold = options.pinned ? null : this.unitCircuit?.admit?.(candidate.accountId) + if (circuitHold && !circuitHold.ok) return null + const limits = this.executionLimits(candidate) + let lease + try { + lease = await this.executionAdmissions.tryReserve({ + accountId: candidate.accountId, + vmId: candidate.vmId, + ...limits, + ...options, + }) + } catch (error) { + if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) + throw error + } + if (!lease.ok) { + if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) + candidate.busy = true + candidate.waitReason = lease.code + if (lease.retry_after_ms) candidate.availableAt = Date.now() + lease.retry_after_ms + return null + } + this.inflight.set(candidate.accountId, (this.inflight.get(candidate.accountId) || 0) + 1) + this.bumpFamily(candidate.accountId, limits.family, 1) + this.bumpVmInflight(candidate.vmId, 1) + let released = false + let releasing = null + const reservation = { + ...lease, + reserved: true, + slotIndex: lease.slotIndex ?? null, + uncertain: false, + markUncertain: () => + this.track( + (async () => { + await lease.markUncertain() + reservation.uncertain = true + })(), + ), + release: (releaseOptions) => { + if (released || reservation.uncertain) return Promise.resolve() + if (releasing) return releasing + releasing = this.track( + (async () => { + await lease.release(releaseOptions) + if (releaseOptions?.uncertain) { + reservation.uncertain = true + return + } + released = true + const next = Math.max(0, (this.inflight.get(candidate.accountId) || 1) - 1) + if (next) this.inflight.set(candidate.accountId, next) + else this.inflight.delete(candidate.accountId) + this.bumpFamily(candidate.accountId, limits.family, -1) + this.bumpVmInflight(candidate.vmId, -1) + if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) + this.planner.pump(candidate.vmId) + this.notifyCapacity(candidate.accountId) + })(), + ) + return releasing + }, + } + return reservation + } + + track(operation) { + const task = Promise.resolve(operation) + this.tasks.add(task) + task.then( + () => this.tasks.delete(task), + (error) => { + this.failure ||= error + this.tasks.delete(task) + }, + ) + return task + } + + async flush() { + while (this.tasks.size || this.planner.tasks.size) { + await Promise.allSettled([...this.tasks]) + await this.planner.flush() + } + await this.planner.flush() + await this.executionAdmissions?.flush?.() + if (this.failure) throw this.failure + } + /** Live request count per VM for the panel seat stream; every change pushes a frame. */ bumpVmInflight(vmId, delta) { if (!vmId) return @@ -1560,6 +1800,7 @@ export class PoolScheduler extends EventEmitter { isReservable(candidate) { if (!candidate) return false + if (this.shared) return grantable(candidate) const inflight = candidate.inflight || 0 const conc = Number(candidate.maxConcurrency) || 0 if (conc > 0 && inflight >= conc) return false diff --git a/src/lib/pool/seat-planner.mjs b/src/lib/pool/seat-planner.mjs index 4c0a8af99..d21e9beff 100644 --- a/src/lib/pool/seat-planner.mjs +++ b/src/lib/pool/seat-planner.mjs @@ -77,12 +77,16 @@ export class SeatPlanner { /** Read at use time: the scheduler owns `sticky_wait_timeout_ms` and hot-reloads it. */ stickyWaitMs = () => 45_000, onChange = null, + asyncGrants = false, } = {}) { this.graceMs = graceMs this.reservePct = reservePct this.strategy = normalizeSeatStrategy(strategy) this.stickyWaitMs = stickyWaitMs this.onChange = onChange + this.asyncGrants = asyncGrants + this.tasks = new Set() + this.failure = null /** @type {Map} seatKey → live seat */ this.seats = new Map() /** @type {Map>} vmId → index → seat */ @@ -147,7 +151,7 @@ export class SeatPlanner { return (this.vmQueues.get(String(vmId)) || []).some((ticket) => ticket.kind === 'conc') } - open(seatKey, { vmId, accountId = null, cap = 0 }) { + open(seatKey, { vmId, accountId = null, cap = 0, index = null }) { const key = String(seatKey) const id = String(vmId) let book = this.byVm.get(id) @@ -155,10 +159,19 @@ export class SeatPlanner { book = new Map() this.byVm.set(id, book) } - let index = 0 - while (book.has(index)) index += 1 - const seat = { seatKey: key, vmId: id, accountId, index, inflight: 0, graceUntil: null, timer: null, live: true } - book.set(index, seat) + let slotIndex = index ?? 0 + if (index == null) while (book.has(slotIndex)) slotIndex += 1 + const seat = { + seatKey: key, + vmId: id, + accountId, + index: slotIndex, + inflight: 0, + graceUntil: null, + timer: null, + live: true, + } + book.set(slotIndex, seat) this.seats.set(key, seat) if (cap) this.noteCap(id, cap) this.rehome(key, id) @@ -331,6 +344,7 @@ export class SeatPlanner { * waiting for a seat here, then the first global tickets that accept it. */ pump(vmId) { + if (this.asyncGrants) return this.pumpAsync(vmId) const id = String(vmId || '') if (!id) return this.dirty.add(id) @@ -353,6 +367,82 @@ export class SeatPlanner { for (const id of ids) this.pump(id) } + track(operation) { + const task = Promise.resolve(operation) + this.tasks.add(task) + task.then( + () => this.tasks.delete(task), + (error) => { + this.failure ||= error + this.tasks.delete(task) + }, + ) + return task + } + + async flush() { + while (this.tasks.size) await Promise.allSettled([...this.tasks]) + if (this.failure) throw this.failure + } + + pumpAsync(vmId) { + const id = String(vmId || '') + if (!id) return Promise.resolve() + this.dirty.add(id) + if (this.pumping) return this.pumpingTask + this.pumping = true + this.pumpingTask = this.track( + (async () => { + try { + while (this.dirty.size) { + const next = this.dirty.values().next().value + this.dirty.delete(next) + await this.pumpOneAsync(next) + } + } finally { + this.pumping = false + } + })(), + ) + return this.pumpingTask + } + + async pumpOneAsync(vmId) { + for (const kind of ['conc', 'seat']) { + for (const ticket of [...(this.vmQueues.get(vmId) || [])]) { + if (!ticket.queued || ticket.kind !== kind) continue + if (!(await this.grantAsync(ticket, vmId))) break + } + } + for (const ticket of [...this.globalQueue]) { + if (!ticket.queued || !ticket.accepts(vmId)) continue + await this.grantAsync(ticket, vmId) + } + } + + async grantAsync(ticket, vmId) { + let result + try { + result = await ticket.tryGrant(vmId) + } catch (error) { + ticket.error = error + ticket.wake?.() + throw error + } + if (!result) return false + const granted = typeof result === 'object' ? result : ticket.granted + if (!ticket.queued || (Number.isFinite(ticket.deadline) && Date.now() >= ticket.deadline)) { + ticket.granted = null + await granted?.reservation?.release?.() + return false + } + ticket.granted = granted + this.unlink(ticket) + this.changed() + ticket.wake?.() + return true + } + pumpOne(vmId) { for (const kind of ['conc', 'seat']) { for (const ticket of [...(this.vmQueues.get(vmId) || [])]) { diff --git a/test/unit/pool-scheduler-shared.test.mjs b/test/unit/pool-scheduler-shared.test.mjs index a0caa27d9..5c84b7e8b 100644 --- a/test/unit/pool-scheduler-shared.test.mjs +++ b/test/unit/pool-scheduler-shared.test.mjs @@ -1,7 +1,12 @@ import test from 'node:test' import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' import { SeatPlanner } from '../../src/lib/pool/seat-planner.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' function gate() { let resolve @@ -11,27 +16,38 @@ function gate() { return { promise, resolve } } -function sharedPool(admissions) { +function sharedPool( + admissions, + { + accountId = 'shared-account', + vmId = 'shared-vm', + accountLimit = 2, + vmLimit = 2, + rpmLimit = 0, + fableLimit = 4, + model = 'claude-test', + } = {}, +) { const pool = new PoolScheduler({ projectRoot: '/tmp', executionAdmissions: admissions, accountQuota: { canAccept: () => ({ ok: true }), - limitFor: () => 2, + limitFor: () => accountLimit, policyFor: () => ({}), - rpmLimitFor: () => 0, + rpmLimitFor: () => rpmLimit, }, - config: { seat_grace_ms: 0 }, + config: { seat_grace_ms: 0, fable_max_per_account: fableLimit }, }) pool.eligibleCandidates = async () => [ { ok: true, - accountId: 'shared-account', - vmId: 'shared-vm', - model: 'claude-test', - vm: { id: 'shared-vm', policy: { maxConcurrency: 2 } }, - account: { account_id: 'shared-account' }, - maxConcurrency: 2, + accountId, + vmId, + model, + vm: { id: vmId, policy: { maxConcurrency: vmLimit } }, + account: { account_id: accountId }, + maxConcurrency: vmLimit, inflight: 0, loadRatio: 0, priority: 0, @@ -150,3 +166,247 @@ test('a cancelled queued request awaits release of a late asynchronous seat gran released.resolve() } }) + +test('asynchronous seat failures reach the waiting request and shutdown flush', async () => { + const error = new Error('synthetic PostgreSQL admission failed') + const planner = new SeatPlanner({ asyncGrants: true }) + const ticket = { + seatKey: 'device-error', + vmId: 'vm-error', + kind: 'conc', + enqueuedAt: Date.now(), + tryGrant: async () => { + throw error + }, + } + planner.enqueue(ticket) + await assert.rejects(planner.pump('vm-error'), error) + assert.equal(ticket.error, error) + await assert.rejects(planner.flush(), error) + planner.remove(ticket) +}) + +async function replicas(t, options = {}) { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + const services = [db, other].map((connection) => { + const service = new SharedExecutionAdmissions(connection) + return { + repo: service.executions, + tryReserve: (input) => service.tryReserve(input), + getCounts: (accountId, vmId) => service.getCounts(accountId, vmId), + getSeat: (seatKey) => service.getSeat(seatKey), + } + }) + const pools = services.map((service, index) => sharedPool(service, { ...options, vmId: `shared-vm-${index}` })) + return { services, pools } +} + +test('two Kubernetes schedulers execute overlapping requests under one PostgreSQL account cap', async (t) => { + const { services, pools } = await replicas(t, { accountLimit: 2, vmLimit: 1 }) + const held = await Promise.all(pools.map((pool) => pool.selectAndReserve({ model: 'claude-test', allowWait: false }))) + try { + assert.deepEqual( + held.map((item) => item.ok), + [true, true], + 'both replicas may execute simultaneously', + ) + assert.equal((await services[0].repo.getCounts({ accountId: 'shared-account' })).accountInflight, 2) + await held[0].release() + const fullVm = await pools[1].selectAndReserve({ model: 'claude-test', allowWait: false }) + assert.equal(fullVm.ok, false) + assert.ok(fullVm.wait_reasons.includes('vm_concurrency_limit')) + const reused = await pools[0].selectAndReserve({ model: 'claude-test', allowWait: false }) + assert.equal(reused.ok, true) + await reused.release() + } finally { + for (const item of held) await item.release?.() + for (const pool of pools) await pool.flush() + } +}) + +test('independent schedulers cannot oversell an account while choosing different VMs', async (t) => { + const { services, pools } = await replicas(t, { accountLimit: 1, vmLimit: 2 }) + const held = await Promise.all(pools.map((pool) => pool.selectAndReserve({ model: 'claude-test', allowWait: false }))) + try { + assert.equal(held.filter((item) => item.ok).length, 1) + assert.equal((await services[1].repo.getCounts({ accountId: 'shared-account' })).accountInflight, 1) + } finally { + for (const item of held) await item.release?.() + for (const pool of pools) await pool.flush() + } +}) + +test('released executions still consume the shared account RPM window', async (t) => { + const { services, pools } = await replicas(t, { accountLimit: 2, vmLimit: 2, rpmLimit: 1 }) + const first = await pools[0].selectAndReserve({ model: 'claude-test', allowWait: false }) + try { + assert.equal(first.ok, true) + await first.release() + const limited = await pools[1].selectAndReserve({ model: 'claude-test', allowWait: false }) + assert.equal(limited.ok, false) + assert.ok(limited.wait_reasons.includes('rpm_limit')) + const counts = await services[1].repo.getCounts({ accountId: 'shared-account' }) + assert.equal(counts.accountInflight, 0) + assert.equal(counts.rpmCount, 1) + } finally { + await first.release?.() + for (const pool of pools) await pool.flush() + } +}) + +test('the PostgreSQL Fable cap spans schedulers without blocking another model family', async (t) => { + const { services, pools } = await replicas(t, { accountLimit: 4, vmLimit: 2, fableLimit: 1, model: 'claude-fable-5' }) + const first = await pools[0].selectAndReserve({ model: 'claude-fable-5', allowWait: false }) + let regular + try { + assert.equal(first.ok, true) + const limited = await pools[1].selectAndReserve({ model: 'claude-fable-5', allowWait: false }) + assert.equal(limited.ok, false) + assert.ok(limited.wait_reasons.includes('fable_concurrency')) + const otherFamily = sharedPool(services[1], { + accountLimit: 4, + vmLimit: 2, + fableLimit: 1, + model: 'claude-sonnet-5', + vmId: 'shared-vm-1', + }) + regular = await otherFamily.selectAndReserve({ model: 'claude-sonnet-5', allowWait: false }) + assert.equal(regular.ok, true) + await regular.release() + await otherFamily.flush() + } finally { + await first.release?.() + await regular?.release?.() + for (const pool of pools) await pool.flush() + } +}) + +test('a second scheduler finds the PostgreSQL device home and retains its shared slot index', async (t) => { + const { services, pools } = await replicas(t, { accountLimit: 4, vmLimit: 2 }) + const candidates = [...(await pools[0].eligibleCandidates()), ...(await pools[1].eligibleCandidates())] + pools[0].eligibleCandidates = async () => candidates + pools[1].eligibleCandidates = async () => [...candidates].reverse() + const first = await pools[0].selectAndReserve({ + model: 'claude-test', + seatKey: 'device-cross-replica', + allowWait: false, + }) + let second + try { + assert.equal(first.ok, true) + assert.equal(pools[1].planner.seats.size, 0, 'the second replica starts without a local seat mirror') + second = await pools[1].selectAndReserve({ + model: 'claude-test', + seatKey: 'device-cross-replica', + allowWait: false, + }) + assert.equal(second.ok, true) + assert.equal(second.vmId, first.vmId) + assert.equal(second.slotIndex, first.slotIndex) + assert.equal(second.seatGeneration, first.seatGeneration) + const shared = await services[1].getSeat('platform|device-cross-replica') + assert.equal(shared.vmId, first.vmId) + assert.equal(shared.inflight, 2) + } finally { + await first.release?.() + await second?.release?.() + for (const pool of pools) await pool.flush() + } +}) + +test('cancellation during a PostgreSQL admission waits for its late lease to be released', async (t) => { + const { services } = await replicas(t) + const reserved = gate() + const deliver = gate() + const pool = sharedPool({ + ...services[0], + async tryReserve(input) { + const lease = await services[0].tryReserve(input) + reserved.resolve() + await deliver.promise + return lease + }, + }) + const controller = new AbortController() + const pending = pool.selectAndReserve({ model: 'claude-test', signal: controller.signal, allowWait: false }) + const rejected = assert.rejects(pending, { code: 'selection_cancelled' }) + try { + await reserved.promise + controller.abort() + assert.equal((await services[0].repo.getCounts({ accountId: 'shared-account' })).accountInflight, 1) + deliver.resolve() + await rejected + await pool.flush() + assert.equal((await services[0].repo.getCounts({ accountId: 'shared-account' })).accountInflight, 0) + } finally { + deliver.resolve() + await rejected + await pool.flush() + } +}) + +test('default Kubernetes selection reads PostgreSQL capacity despite stale local quota and inflight caches', async (t) => { + const { services } = await replicas(t) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-pool-real-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const vms = path.join(root, 'vms') + fs.mkdirSync(vms) + fs.writeFileSync( + path.join(vms, 'vm-real.json'), + JSON.stringify({ + id: 'vm-real', + status: 'running', + schedulable: true, + proxy_cli_enabled: true, + proxy: { id: 'synthetic-proxy', url: 'socks5h://127.0.0.1:11001' }, + runtime: { worker_socket: path.join(vms, 'vm-real', 'run', 'worker.sock') }, + policy: { maxConcurrency: 1, concurrencyOverride: true }, + claude: { + account_uuid: 'account-real', + account_tier: 'max', + access_token: 'synthetic-access', + refresh_token: 'synthetic-refresh', + expires_at: Math.floor(Date.now() / 1000) + 3600, + }, + }), + ) + const pools = [getDb(), services[1].repo.db].map( + (db) => + new PoolScheduler({ + projectRoot: root, + accountQuota: { + db, + canAccept: () => ({ ok: false, reason: 'concurrency_limit' }), + limitFor: () => 2, + policyFor: () => ({}), + rpmLimitFor: () => 0, + }, + workerHealth: async () => ({ ok: true, credential: { generation: 1, has_access: true } }), + }), + ) + pools[0].inflight.set('account-real', 99) + pools[1].inflight.set('account-real', 99) + const first = await pools[0].selectAndReserve({ model: 'claude-test', allowWait: false }) + let next + try { + assert.equal(first.ok, true, 'stale local capacity cannot reject a PostgreSQL-granted request') + const full = await pools[1].selectAndReserve({ model: 'claude-test', allowWait: false }) + assert.equal(full.ok, false) + assert.ok(full.wait_reasons.includes('vm_concurrency_limit')) + await first.release() + next = await pools[1].selectAndReserve({ model: 'claude-test', allowWait: false }) + assert.equal(next.ok, true) + } finally { + await first.release?.() + await next?.release?.() + for (const pool of pools) await pool.flush() + } +}) From 68c017b50874d1647b3783cc86fc669dde08172f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:39:09 +0800 Subject: [PATCH 082/473] test(k8s): reproduce account seat budget oversubscription across VMs --- test/unit/shared-execution-seats.test.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/unit/shared-execution-seats.test.mjs b/test/unit/shared-execution-seats.test.mjs index 64618f9ac..d077256d1 100644 --- a/test/unit/shared-execution-seats.test.mjs +++ b/test/unit/shared-execution-seats.test.mjs @@ -140,3 +140,13 @@ test('a denied move preserves the original grace seat and its generation', async assert.equal(denied.code, 'vm_concurrency_limit') assert.deepEqual(await first.getSeat(candidate().seatKey), before) }) + +test('seat budget is shared by one account across different VMs', async (t) => { + const [first, second] = await replicas(t) + const a = await first.tryReserve(candidate({ seatHeadroom: 0.03 })) + assert.equal(a.ok, true) + const b = await second.tryReserve(candidate({ vmId: 'vm-2', seatKey: 'device-2', seatHeadroom: 0.03 })) + assert.equal(b.ok, false) + assert.equal(b.code, 'seat_budget') + await a.release() +}) From 2ffb8fb43bb6bd578bf86599a2a32d07d1302db8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:39:36 +0800 Subject: [PATCH 083/473] fix(k8s): share device seat budgets across an account VMs --- src/lib/pool/shared-execution-admissions.mjs | 7 ++++++- test/unit/shared-execution-seats.test.mjs | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/lib/pool/shared-execution-admissions.mjs b/src/lib/pool/shared-execution-admissions.mjs index 12c7e58df..31fafcd7f 100644 --- a/src/lib/pool/shared-execution-admissions.mjs +++ b/src/lib/pool/shared-execution-admissions.mjs @@ -94,7 +94,12 @@ export class SharedExecutionAdmissions { .prepare('SELECT count(*) AS count FROM execution_seats WHERE vm_id = $1 AND seat_key <> $2') .get(input.vmId, seatKey) if (count >= input.seatCap) return { ok: false, code: 'seat_limit', reason: 'seat_limit' } - if (!budgetAllows(input.seatHeadroom, count, input.seatReservePct)) + const { count: accountSeats } = await this.db + .prepare(`SELECT count(*) AS count FROM execution_seats s WHERE s.account_id = $1 AND s.seat_key <> $2 + AND (s.grace_until > clock_timestamp() OR EXISTS (SELECT 1 FROM execution_admissions a + WHERE a.seat_key = s.seat_key AND a.seat_generation = s.generation AND a.released_at IS NULL))`) + .get(input.accountId, seatKey) + if (!budgetAllows(input.seatHeadroom, accountSeats, input.seatReservePct)) return { ok: false, code: 'seat_budget', reason: 'seat_budget' } const free = await this.db .prepare(`SELECT n AS slot_index FROM generate_series(0, $2::integer - 1) AS n diff --git a/test/unit/shared-execution-seats.test.mjs b/test/unit/shared-execution-seats.test.mjs index d077256d1..5bef77be3 100644 --- a/test/unit/shared-execution-seats.test.mjs +++ b/test/unit/shared-execution-seats.test.mjs @@ -150,3 +150,13 @@ test('seat budget is shared by one account across different VMs', async (t) => { assert.equal(b.code, 'seat_budget') await a.release() }) + +test('expired account seats on another VM do not consume the new-device budget', async (t) => { + const [first, second] = await replicas(t) + const a = await first.tryReserve(candidate({ seatHeadroom: 0.03, seatGraceMs: 1000 })) + await a.release() + await getDb().prepare("UPDATE execution_seats SET grace_until = clock_timestamp() - interval '1 second'").run() + const b = await second.tryReserve(candidate({ vmId: 'vm-2', seatKey: 'device-2', seatHeadroom: 0.03 })) + assert.equal(b.ok, true) + await b.release() +}) From cb927ad4d87d71677e33d00e6256e93ec71df40f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:41:15 +0800 Subject: [PATCH 084/473] test(kernel): reproduce unsafe retries without execution completion --- test/unit/kernel-router.test.mjs | 147 +++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/test/unit/kernel-router.test.mjs b/test/unit/kernel-router.test.mjs index 4d6706486..3ec21e10c 100644 --- a/test/unit/kernel-router.test.mjs +++ b/test/unit/kernel-router.test.mjs @@ -198,6 +198,153 @@ async function kernelFixture(handler) { } } +for (const [mode, dispatch] of [ + ['call', dispatchCallInference], + ['stream', dispatchStreamInference], +]) { + unixTest(`shared ${mode} execution does not retry after a truncated Unix socket response`, async () => { + const previous = process.env.KIN_KERNEL_BIN + process.env.KIN_KERNEL_BIN = '/bin/true' + const requests = [] + const started = [] + const ended = [] + const fx = await kernelFixture(async (req, res) => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + requests.push(JSON.parse(Buffer.concat(chunks).toString('utf8')).request_id) + res.destroy() + }) + try { + const result = await dispatch({ + exec: fx.exec, + body: { model: 'claude-test', messages: [{ role: 'user', content: 'synthetic request' }] }, + ensureRust: async () => ({ ok: true }), + timeoutMs: 1000, + onExecutionStart: async (execution) => started.push(execution), + onExecutionEnd: async (execution) => ended.push(execution), + }) + assert.equal(requests.length, 1, 'an unconfirmed execution must not create another kernel job') + assert.equal(started.length, 1) + assert.equal(ended.length, 1) + assert.equal(ended[0].requestId, requests[0]) + assert.equal(ended[0].started, true) + assert.equal(ended[0].ended, false) + assert.equal(result.execution.ended, false) + assert.equal(result.rust_transport_retried, undefined) + } finally { + await fx.close() + if (previous == null) delete process.env.KIN_KERNEL_BIN + else process.env.KIN_KERNEL_BIN = previous + } + }) +} + +unixTest('shared credential retry awaits the first execution end and uses a new job ID', async () => { + const previous = process.env.KIN_KERNEL_BIN + process.env.KIN_KERNEL_BIN = '/bin/true' + const requests = [] + const ended = [] + let ensures = 0 + const fx = await kernelFixture(async (req, res) => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + requests.push(JSON.parse(Buffer.concat(chunks).toString('utf8')).request_id) + res.writeHead(requests.length === 1 ? 401 : 200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify( + requests.length === 1 + ? { type: 'error', error: { code: 'needs_refresh', message: 'synthetic refresh' } } + : { type: 'message', role: 'assistant', content: [] }, + ), + ) + }) + try { + const result = await dispatchCallInference({ + exec: fx.exec, + body: { model: 'claude-test', messages: [{ role: 'user', content: 'synthetic request' }] }, + requestId: 'fixed-first-job', + ensureRust: async () => ({ ok: true }), + onExecutionStart: async () => {}, + onExecutionEnd: async (execution) => { + await new Promise((resolve) => setImmediate(resolve)) + ended.push(execution) + }, + ensureCredential: async () => { + assert.equal(ended.length, 1, 'credential refresh follows the awaited execution-end callback') + assert.equal(ended[0].ended, true) + ensures++ + return { ok: true } + }, + timeoutMs: 1000, + }) + assert.equal(result.credential_retried, true) + assert.equal(ensures, 1) + assert.equal(requests.length, 2) + assert.equal(requests[0], 'fixed-first-job') + assert.notEqual(requests[1], requests[0], 'a retry cannot reuse a cancelled or completed job ID') + assert.deepEqual( + ended.map((execution) => execution.requestId), + requests, + ) + assert.equal(ended[1].ended, true) + } finally { + await fx.close() + if (previous == null) delete process.env.KIN_KERNEL_BIN + else process.env.KIN_KERNEL_BIN = previous + } +}) + +for (const callback of ['onExecutionStart', 'onExecutionEnd']) { + unixTest(`a failing ${callback} prevents further jobs and preserves its original error`, async () => { + const previous = process.env.KIN_KERNEL_BIN + process.env.KIN_KERNEL_BIN = '/bin/true' + let requests = 0 + let ensures = 0 + const error = new Error(`synthetic ${callback} persistence failure`) + const fx = await kernelFixture((req, res) => { + req.resume() + requests++ + res.writeHead(401, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ type: 'error', error: { code: 'needs_refresh' } })) + }) + try { + await assert.rejects( + dispatchCallInference({ + exec: fx.exec, + body: { model: 'claude-test', messages: [{ role: 'user', content: 'synthetic request' }] }, + ensureRust: async () => ({ ok: true }), + onExecutionStart: async () => {}, + onExecutionEnd: async () => {}, + [callback]: async () => { + throw error + }, + ensureCredential: async () => { + ensures++ + return { ok: true } + }, + timeoutMs: 1000, + }), + (actual) => actual === error, + ) + assert.equal(requests, callback === 'onExecutionStart' ? 0 : 1) + assert.equal(ensures, 0) + } finally { + await fx.close() + if (previous == null) delete process.env.KIN_KERNEL_BIN + else process.env.KIN_KERNEL_BIN = previous + } + }) +} + +unixTest('preparation rejection identifies that no kernel execution started', async () => { + const result = await dispatchCallInference({ + exec: { vmId: 'synthetic-unavailable', vm: { id: 'synthetic-unavailable' } }, + ensureRust: async () => ({ ok: false, reason: 'kernel_unavailable' }), + onExecutionStart: async () => assert.fail('preparation cannot create a kernel job'), + }) + assert.deepEqual(result.execution, { requestId: null, started: false, ended: true, reason: 'not_started' }) +}) + unixTest('dispatchStreamInference uses rust socket when ready', async () => { const prev = process.env.KIN_KERNEL_BIN process.env.KIN_KERNEL_BIN = '/bin/true' From 1aca66f7cd16434dda393643f2c012e64b705cbe Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:41:30 +0800 Subject: [PATCH 085/473] test(k8s): retain capacity when execution termination is unknown --- test/unit/failover-shared-execution.test.mjs | 105 +++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 test/unit/failover-shared-execution.test.mjs diff --git a/test/unit/failover-shared-execution.test.mjs b/test/unit/failover-shared-execution.test.mjs new file mode 100644 index 000000000..12449e2db --- /dev/null +++ b/test/unit/failover-shared-execution.test.mjs @@ -0,0 +1,105 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' + +function fixture() { + const events = [] + const selected = { + ok: true, + accountId: 'shared-account', + vmId: 'shared-vm', + bindExecution: async () => {}, + markUncertain: async () => { + events.push('uncertain') + }, + release: async () => { + events.push('release') + }, + } + const runner = new FailoverRunner({ + scheduler: { + async selectAndReserve() { + const another = events.includes('select') + events.push('select') + if (another) return { ok: false, reason: 'no_eligible_accounts' } + return selected + }, + markSuccess() {}, + markCooldown() {}, + }, + config: { max_same_account_retries: 0, max_account_switches: 2, max_total_attempts: 2 }, + }) + return { runner, events } +} + +function args(callAttempt, extra = {}) { + return { + requestId: 'shared-execution', + model: 'claude-sonnet-4-6', + canonicalBody: { model: 'claude-sonnet-4-6', messages: [] }, + callAttempt, + ...extra, + } +} + +for (const execution of [undefined, { requestId: 'job-1', started: true, ended: false, reason: 'transport_error' }]) { + test(`shared execution with ${execution ? 'an unfinished job' : 'missing termination metadata'} does not fail over`, async () => { + const { runner, events } = fixture() + const result = await runner.run( + args(async () => { + events.push('call') + return { ok: false, status: 0, transportError: true, terminalState: 'transport_error', execution } + }), + ) + assert.equal(result.ok, false) + assert.deepEqual(events, ['select', 'call', 'uncertain', 'release']) + }) +} + +test('an exception after entering a shared execution retains capacity and stops retries', async () => { + const { runner, events } = fixture() + const result = await runner.run( + args(async () => { + events.push('call') + throw new Error('connection interrupted') + }), + ) + assert.equal(result.ok, false) + assert.deepEqual(events, ['select', 'call', 'uncertain', 'release']) +}) + +test('request preparation failure releases a shared reservation without marking execution uncertain', async () => { + const { runner, events } = fixture() + const result = await runner.run( + args(() => assert.fail('preparation must precede submission'), { + applyAttempt: async () => { + throw Object.assign(new Error('invalid cache control'), { + status: 400, + body: { error: { code: 'invalid_cache_control' } }, + }) + }, + }), + ) + assert.equal(result.status, 400) + assert.deepEqual(events, ['select', 'release']) +}) + +test('a proven terminal shared execution releases capacity normally', async () => { + const { runner, events } = fixture() + const result = await runner.run( + args(async () => ({ + ok: true, + status: 200, + terminalState: 'verified', + execution: { requestId: 'job-1', started: true, ended: true, reason: 'eof' }, + body: { + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'done' }], + stop_reason: 'end_turn', + }, + })), + ) + assert.equal(result.ok, true) + assert.deepEqual(events, ['select', 'release']) +}) From 5efbda8736a6e2a8b2863de5fe652f28300a9f8f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:43:14 +0800 Subject: [PATCH 086/473] fix(kernel): await execution completion before shared retries --- src/lib/transport/kernel-router.mjs | 39 +++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/lib/transport/kernel-router.mjs b/src/lib/transport/kernel-router.mjs index 0a5a95f88..55e9803d3 100644 --- a/src/lib/transport/kernel-router.mjs +++ b/src/lib/transport/kernel-router.mjs @@ -4,6 +4,7 @@ * Credential import/ensure 仍可走 Go 客户端，但不参与推理 hop。 */ import { clientCancelledResult, isClientCancelledResult } from '../core/errors.mjs' +import { randomUUID } from 'node:crypto' import { ensureWorkerCredential } from './go-worker-client.mjs' import { ensureOfficialCredentialLink, slotUidGidFromHomeDir } from '../oauth/oauth-credentials.mjs' import { slotHost } from '../vm/slot-host.mjs' @@ -142,6 +143,7 @@ function rustUnavailableResult(ready) { headers: {}, terminalState: answered ? 'rejected' : 'transport_error', transportError: !answered, + execution: { requestId: null, started: false, ended: true, reason: 'not_started' }, } } @@ -239,6 +241,7 @@ async function runHop({ mode, opts }) { status: 400, terminalState: 'rejected', upstreamExecutions: 0, + execution: { requestId: null, started: false, ended: true, reason: 'not_started' }, body: { type: 'error', error: { @@ -265,26 +268,52 @@ async function runHop({ mode, opts }) { } } const send = mode === 'stream' ? streamRustKernel : callRustKernel + const sharedExecution = typeof opts.onExecutionStart === 'function' || typeof opts.onExecutionEnd === 'function' + const canRetry = (value) => !sharedExecution || value?.execution?.ended === true + let hop = 0 + const sendHop = async () => { + let startError + const value = await send({ + ...opts, + requestId: hop++ === 0 && opts.requestId != null ? opts.requestId : randomUUID(), + onExecutionStart: + typeof opts.onExecutionStart === 'function' + ? async (execution) => { + try { + await opts.onExecutionStart(execution) + } catch (error) { + startError = error + throw error + } + } + : opts.onExecutionStart, + }) + // The worker returns transport results for errors, including callback errors. + // A failed durable start must retain the original exception and cannot retry. + if (startError) throw startError + if (typeof opts.onExecutionEnd === 'function') await opts.onExecutionEnd(value?.execution) + return value + } beginWrapHop(opts.exec) let result try { - result = await send(opts) + result = await sendHop() noteWrapHop(opts.exec) if (opts.signal?.aborted || isClientCancelledResult(result)) { result = clientCancelledResult(result) - } else if (result.transportError === true && result.committed !== true) { - result = await send(opts) + } else if (result.transportError === true && result.committed !== true && canRetry(result)) { + result = await sendHop() result = { ...result, rust_transport_retried: true } noteWrapHop(opts.exec) } - if (!isClientCancelledResult(result) && isNeedsRefreshResult(result)) { + if (!isClientCancelledResult(result) && isNeedsRefreshResult(result) && canRetry(result)) { const ensure = opts.ensureCredential || ensureWorkerCredential const ensured = await ensure(opts.exec, { force: true }) if (ensured?.ok !== true) result = credentialEnsureFailure(result, ensured) else { // The CLI invalidates its OAuth cache from the shared credential file // before creating each API client. Recycling here kills sibling hops. - result = await send(opts) + result = await sendHop() result = { ...result, credential_retried: true } noteWrapHop(opts.exec) } From 64acdcc5fb50aa71efe0587c3ca551b26e874e39 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:44:53 +0800 Subject: [PATCH 087/473] fix(k8s): retain shared capacity until execution termination is known --- src/lib/pool/failover-runner.mjs | 29 ++++++- src/lib/protocol/handle-protocol.mjs | 7 +- test/unit/failover-shared-execution.test.mjs | 2 + .../protocol-execution-uncertain.test.mjs | 78 +++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 test/unit/protocol-execution-uncertain.test.mjs diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index f6b1dc7d4..f48731e60 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -803,6 +803,9 @@ export class FailoverRunner { let policy let committed = false let hopNoted = false + let callEntered = false + let executionEnded = false + let unknownExecution = false const noteHop = (value) => { if (hopNoted) return hopNoted = true @@ -844,6 +847,7 @@ export class FailoverRunner { }, ) } + callEntered = true result = await callAttempt({ candidate: selected, body, @@ -856,6 +860,12 @@ export class FailoverRunner { committed = true }, }) + executionEnded = result?.execution?.ended === true + if (selected.bindExecution && !executionEnded) { + await selected.markUncertain() + unknownExecution = true + if (result) result.executionUncertain = true + } noteHop(result) if (result) result.committed = result.committed || committed if (signal?.aborted || isClientCancelledResult(result)) { @@ -923,6 +933,8 @@ export class FailoverRunner { if (typeof onAttempt === 'function') { await onAttempt({ attemptNo, selected, result, policy }) } + if (unknownExecution) + return incompleteHopResult(result, policy, { ...attribution(), executionUncertain: true }) if (policy.reason === 'content_filter_refusal' || policy.reason === 'usage_policy_refusal') { this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, countUsage }) bindAll({ accountId: selected.accountId, vmId: selected.vmId }) @@ -986,6 +998,13 @@ export class FailoverRunner { } budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) } catch (error) { + // A submitted job can outlive its transport. Never replay it or free + // its shared capacity without proof that the execution ended. + if (selected.bindExecution && callEntered) { + if (executionEnded) throw error + if (!unknownExecution) await selected.markUncertain() + unknownExecution = true + } const invalidCache = error?.status === 400 && error.body?.error?.code === 'invalid_cache_control' noteHop(invalidCache ? { upstreamExecutions: 0 } : null) if (error?.code === 'response_sent') { @@ -1035,10 +1054,16 @@ export class FailoverRunner { terminalState: 'cancelled', latencyMs: Date.now() - attemptStarted, }) - return { ...clientCancelledResult({ committed }), via: 'pool-failover', ...attribution() } + return { + ...clientCancelledResult({ committed }), + ...(unknownExecution ? { executionUncertain: true } : {}), + via: 'pool-failover', + ...attribution(), + } } result = { ok: false, + ...(unknownExecution ? { executionUncertain: true } : {}), status: 0, transportError: true, committed, @@ -1076,7 +1101,7 @@ export class FailoverRunner { terminalState: result.terminalState, latencyMs: Date.now() - attemptStarted, }) - if (committed || !shouldContinue(policy)) { + if (unknownExecution || committed || !shouldContinue(policy)) { return { ...result, ...attribution(), finalState: result.terminalState, policy } } applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId }) diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 9c7404e79..0b4cab7b0 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -449,6 +449,8 @@ export function createHandleProtocol(deps) { const assembler = createClaudeMessageAssembler() const workerResult = await dispatchStreamInference({ exec: candidate.exec, + onExecutionStart: candidate.bindExecution, + onExecutionEnd: candidate.completeExecution, body, reqHeaders, timeoutMs, @@ -486,7 +488,7 @@ export function createHandleProtocol(deps) { } throw error } finally { - if (!admission.lease?.billingFailed) await admission.lease?.release?.() + if (!admission.lease?.billingFailed && !admission.lease?.executionUncertain) await admission.lease?.release?.() } } @@ -1539,6 +1541,8 @@ export function createHandleProtocol(deps) { try { return await dispatchStreamInference({ exec: candidate.exec, + onExecutionStart: candidate.bindExecution, + onExecutionEnd: candidate.completeExecution, cacheTtl, preserveCacheBreakpoints, cliHop: attemptMeta?.cliHop === true, @@ -1597,6 +1601,7 @@ export function createHandleProtocol(deps) { } }, }) + if (result?.executionUncertain && admission.lease?.shared) admission.lease.executionUncertain = true } finally { anthropicKeepalive?.stop() clientAbort.settle() diff --git a/test/unit/failover-shared-execution.test.mjs b/test/unit/failover-shared-execution.test.mjs index 12449e2db..623e91b3d 100644 --- a/test/unit/failover-shared-execution.test.mjs +++ b/test/unit/failover-shared-execution.test.mjs @@ -52,6 +52,7 @@ for (const execution of [undefined, { requestId: 'job-1', started: true, ended: }), ) assert.equal(result.ok, false) + assert.equal(result.executionUncertain, true) assert.deepEqual(events, ['select', 'call', 'uncertain', 'release']) }) } @@ -65,6 +66,7 @@ test('an exception after entering a shared execution retains capacity and stops }), ) assert.equal(result.ok, false) + assert.equal(result.executionUncertain, true) assert.deepEqual(events, ['select', 'call', 'uncertain', 'release']) }) diff --git a/test/unit/protocol-execution-uncertain.test.mjs b/test/unit/protocol-execution-uncertain.test.mjs new file mode 100644 index 000000000..aa24b83d0 --- /dev/null +++ b/test/unit/protocol-execution-uncertain.test.mjs @@ -0,0 +1,78 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Readable } from 'node:stream' +import { EventEmitter } from 'node:events' +import { createHandleProtocol } from '../../src/lib/protocol/handle-protocol.mjs' +import { readBody } from '../../src/lib/http/respond.mjs' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { getDb } from '../../src/lib/db/database.mjs' + +test('a returned uncertain execution keeps API key concurrency after usage settlement', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-protocol-uncertain-')) + const store = new ApiKeyStore({ db: getDb(), sharedAdmission: true }) + const key = await store.create({ max_concurrency: 1 }) + let executed = false + const handler = createHandleProtocol({ + cfg: { + limits: { max_body_bytes: 1 << 20 }, + rewrite: { enabled: false }, + intercept: { rules: [] }, + distill: { enabled: false }, + paths: { project: root, data: root }, + }, + json(res, status, body) { + res.statusCode = status + res.body = body + }, + readBody, + requireAuth(req) { + req.apiKeyRecord = key + req.apiKeyKind = 'managed' + return true + }, + requestLog: { start: () => ({ request_id: 'uncertain-key', t0: Date.now() }) }, + stats: { errors: 0, requests: 0, by_route: {} }, + groupsRepo: { rateMultiplier: () => 1 }, + apiKeyStore: store, + stickyRouter: { resolve: () => null }, + accountQuota: { ingestHeaders() {} }, + failoverRunner: { + async run() { + executed = true + return { + ok: false, + status: 502, + finalState: 'incomplete', + executionUncertain: true, + accountId: 'uncertain-account', + vmId: 'uncertain-vm', + body: { type: 'error', error: { type: 'api_error', message: 'upstream connection interrupted' } }, + } + }, + }, + routingConfig: { compatibility: { claude_code_only: false } }, + }) + const req = Readable.from([ + Buffer.from( + JSON.stringify({ model: 'claude-sonnet-4-6', max_tokens: 20, messages: [{ role: 'user', content: 'x' }] }), + ), + ]) + req.headers = { 'user-agent': 'claude-cli/2.1.0 (external, cli)', 'anthropic-version': '2023-06-01' } + const res = new EventEmitter() + res.setHeader = () => {} + try { + await handler.handleProtocol(req, res, 'anthropic.messages', '/v1/messages') + assert.equal(executed, true, JSON.stringify(res.body)) + const row = await getDb() + .prepare('SELECT released_at, settled_at FROM api_key_admissions WHERE key_id = $1') + .get(key.id) + assert.ok(row.settled_at, 'usage settlement remains independent of execution termination') + assert.equal(row.released_at, null) + assert.equal((await store.reserve(key)).code, 'api_key_concurrency_limit') + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) From 3098648f178770cc3e22b8ff500b31106f33297b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:45:10 +0800 Subject: [PATCH 088/473] test(pool): reproduce shared release retry and uncertainty races --- test/unit/pool-scheduler-release.test.mjs | 163 ++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 test/unit/pool-scheduler-release.test.mjs diff --git a/test/unit/pool-scheduler-release.test.mjs b/test/unit/pool-scheduler-release.test.mjs new file mode 100644 index 000000000..bd0978609 --- /dev/null +++ b/test/unit/pool-scheduler-release.test.mjs @@ -0,0 +1,163 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setImmediate } from 'node:timers/promises' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' + +function gate() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function reservation(t) { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) + const db = getDb() + const service = new SharedExecutionAdmissions(db) + const pool = new PoolScheduler({ + projectRoot: '/tmp', + executionAdmissions: service, + accountQuota: { + canAccept: () => ({ ok: true }), + limitFor: () => 2, + policyFor: () => ({}), + rpmLimitFor: () => 0, + }, + }) + pool.unitCircuit = null + const selected = await pool.reserve({ + accountId: 'release-account', + vmId: 'release-vm', + model: 'claude-test', + maxConcurrency: 2, + account: {}, + vm: { policy: { maxConcurrency: 2 } }, + }) + const row = () => db.prepare('SELECT * FROM execution_admissions WHERE id = $1').get(selected.admissionId) + return { db, service, pool, selected, row } +} + +test('a rolled-back PostgreSQL release can be retried through the scheduler reservation', async (t) => { + const { db, service, pool, selected, row } = await reservation(t) + await db.exec(`CREATE FUNCTION refuse_scheduler_release() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.released_at IS NOT NULL THEN RAISE EXCEPTION 'synthetic release failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER refuse_scheduler_release BEFORE UPDATE ON execution_admissions + FOR EACH ROW EXECUTE FUNCTION refuse_scheduler_release()`) + try { + await assert.rejects(selected.release(), { code: 'P0001' }) + assert.equal((await row()).released_at, null) + assert.equal(pool.inflight.get('release-account'), 1) + await db.exec('DROP TRIGGER refuse_scheduler_release ON execution_admissions') + await assert.doesNotReject(selected.release()) + assert.ok((await row()).released_at) + assert.equal(pool.inflight.has('release-account'), false) + await selected.release() + assert.equal(pool.vmInflight.has('release-vm'), false) + } finally { + await db.exec('DROP TRIGGER IF EXISTS refuse_scheduler_release ON execution_admissions') + await service.release(selected) + } +}) + +test('a PostgreSQL refusal to release preserves local counters and allows a later retry', async (t) => { + const { db, service, pool, selected, row } = await reservation(t) + await db.exec(`CREATE FUNCTION skip_scheduler_release() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.released_at IS NOT NULL THEN RETURN NULL; END IF; RETURN NEW; END $$; + CREATE TRIGGER skip_scheduler_release BEFORE UPDATE ON execution_admissions + FOR EACH ROW EXECUTE FUNCTION skip_scheduler_release()`) + try { + await selected.release() + assert.equal((await row()).released_at, null) + assert.equal(pool.inflight.get('release-account'), 1) + assert.equal(pool.vmInflight.get('release-vm'), 1) + await db.exec('DROP TRIGGER skip_scheduler_release ON execution_admissions') + await selected.release() + assert.ok((await row()).released_at) + assert.equal(pool.inflight.has('release-account'), false) + assert.equal(pool.vmInflight.has('release-vm'), false) + } finally { + await db.exec('DROP TRIGGER IF EXISTS skip_scheduler_release ON execution_admissions') + await service.release(selected) + } +}) + +test('release waits for an already requested uncertainty mark and retains PostgreSQL capacity', async (t) => { + const { db, pool, selected, row } = await reservation(t) + const entered = gate() + const finishMark = gate() + const query = db.query + db.query = async function (sql, params) { + if (sql.includes('UPDATE execution_admissions SET uncertain_at')) { + entered.resolve() + await finishMark.promise + } + return query.call(this, sql, params) + } + const marking = selected.markUncertain() + await entered.promise + let releaseSettled = false + const releasing = selected.release().then(() => { + releaseSettled = true + }) + try { + assert.equal(selected.uncertain, true, 'uncertainty is conservative as soon as marking is requested') + await setImmediate() + assert.equal(releaseSettled, false) + assert.equal((await row()).released_at, null) + } finally { + finishMark.resolve() + await Promise.all([marking, releasing]) + db.query = query + } + const stored = await row() + assert.ok(stored.uncertain_at) + assert.equal(stored.released_at, null) + assert.equal(pool.inflight.get('release-account'), 1) + assert.equal(pool.vmInflight.get('release-vm'), 1) +}) + +test('a failed uncertainty mark is retried instead of falling back to ordinary release', async (t) => { + const { db, pool, selected, row } = await reservation(t) + await db.exec(`CREATE FUNCTION refuse_scheduler_uncertainty() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.uncertain_at IS NOT NULL THEN RAISE EXCEPTION 'synthetic uncertainty failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER refuse_scheduler_uncertainty BEFORE UPDATE ON execution_admissions + FOR EACH ROW EXECUTE FUNCTION refuse_scheduler_uncertainty()`) + try { + await assert.rejects(selected.markUncertain(), { code: 'P0001' }) + assert.equal(selected.uncertain, true) + await db.exec('DROP TRIGGER refuse_scheduler_uncertainty ON execution_admissions') + await selected.release() + const stored = await row() + assert.ok(stored.uncertain_at) + assert.equal(stored.released_at, null) + assert.equal(pool.inflight.get('release-account'), 1) + } finally { + await db.exec('DROP TRIGGER IF EXISTS refuse_scheduler_uncertainty ON execution_admissions') + } +}) + +test('concurrent and repeated releases update PostgreSQL and local counters once', async (t) => { + const { service, pool, selected, row } = await reservation(t) + const release = service.executions.release.bind(service.executions) + let calls = 0 + service.executions.release = async (...args) => { + calls++ + return release(...args) + } + await Promise.all([selected.release(), selected.release()]) + const releasedAt = (await row()).released_at + await selected.release() + assert.ok(releasedAt) + assert.deepEqual((await row()).released_at, releasedAt) + assert.equal(calls, 1) + assert.equal(pool.inflight.has('release-account'), false) + assert.equal(pool.vmInflight.has('release-vm'), false) +}) From 62e2662e99dde6d5cd94527c655db7112483eaae Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:47:28 +0800 Subject: [PATCH 089/473] fix(pool): retry shared release and preserve uncertainty intent --- src/lib/pool/pool-scheduler.mjs | 55 ++++++++++++++++------- test/unit/pool-scheduler-release.test.mjs | 16 +++++-- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 218c00e56..8c460c95c 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -834,10 +834,11 @@ export class PoolScheduler extends EventEmitter { release: (options) => this.track( (async () => { - await releaseRequest(options) - if (released || reservation.uncertain) return + const result = await releaseRequest(options) + if (released || reservation.uncertain || result?.released === false) return result released = true this.planner.release(seat) + return result })(), ), }, @@ -1640,28 +1641,38 @@ export class PoolScheduler extends EventEmitter { this.bumpVmInflight(candidate.vmId, 1) let released = false let releasing = null + let marking = null const reservation = { ...lease, reserved: true, slotIndex: lease.slotIndex ?? null, uncertain: false, - markUncertain: () => - this.track( - (async () => { - await lease.markUncertain() - reservation.uncertain = true - })(), - ), + markUncertain: () => { + if (released) return Promise.resolve({ uncertain: false }) + // Stop ordinary release immediately, including while persistence is pending. + reservation.uncertain = true + if (marking) return marking + const task = this.track(Promise.resolve().then(() => lease.markUncertain())) + marking = task + task.then( + (result) => { + if (result?.uncertain === false && marking === task) marking = null + }, + () => { + if (marking === task) marking = null + }, + ) + return task + }, release: (releaseOptions) => { - if (released || reservation.uncertain) return Promise.resolve() + if (releaseOptions?.uncertain) return reservation.markUncertain() + if (released) return Promise.resolve({ released: false }) + if (reservation.uncertain) return reservation.markUncertain() if (releasing) return releasing - releasing = this.track( + const task = this.track( (async () => { - await lease.release(releaseOptions) - if (releaseOptions?.uncertain) { - reservation.uncertain = true - return - } + const result = await lease.release(releaseOptions) + if (result?.released === false) return result released = true const next = Math.max(0, (this.inflight.get(candidate.accountId) || 1) - 1) if (next) this.inflight.set(candidate.accountId, next) @@ -1671,9 +1682,19 @@ export class PoolScheduler extends EventEmitter { if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) this.planner.pump(candidate.vmId) this.notifyCapacity(candidate.accountId) + return result })(), ) - return releasing + releasing = task + task.then( + () => { + if (!released && releasing === task) releasing = null + }, + () => { + if (releasing === task) releasing = null + }, + ) + return task }, } return reservation diff --git a/test/unit/pool-scheduler-release.test.mjs b/test/unit/pool-scheduler-release.test.mjs index bd0978609..21e3653d1 100644 --- a/test/unit/pool-scheduler-release.test.mjs +++ b/test/unit/pool-scheduler-release.test.mjs @@ -13,7 +13,7 @@ function gate() { return { promise, resolve } } -async function reservation(t) { +async function reservation(t, { seat = false } = {}) { const previous = process.env.VM2API_CONTROL_PLANE_MODE process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' t.after(() => { @@ -25,6 +25,7 @@ async function reservation(t) { const pool = new PoolScheduler({ projectRoot: '/tmp', executionAdmissions: service, + config: { seat_grace_ms: 0 }, accountQuota: { canAccept: () => ({ ok: true }), limitFor: () => 2, @@ -33,14 +34,19 @@ async function reservation(t) { }, }) pool.unitCircuit = null - const selected = await pool.reserve({ + const candidate = { accountId: 'release-account', vmId: 'release-vm', model: 'claude-test', maxConcurrency: 2, + seatCap: 2, + headroom: null, account: {}, vm: { policy: { maxConcurrency: 2 } }, - }) + } + const selected = seat + ? (await pool.claimSeatShared('release-seat', candidate, 'test')).reservation + : await pool.reserve(candidate) const row = () => db.prepare('SELECT * FROM execution_admissions WHERE id = $1').get(selected.admissionId) return { db, service, pool, selected, row } } @@ -68,7 +74,7 @@ test('a rolled-back PostgreSQL release can be retried through the scheduler rese }) test('a PostgreSQL refusal to release preserves local counters and allows a later retry', async (t) => { - const { db, service, pool, selected, row } = await reservation(t) + const { db, service, pool, selected, row } = await reservation(t, { seat: true }) await db.exec(`CREATE FUNCTION skip_scheduler_release() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.released_at IS NOT NULL THEN RETURN NULL; END IF; RETURN NEW; END $$; CREATE TRIGGER skip_scheduler_release BEFORE UPDATE ON execution_admissions @@ -78,11 +84,13 @@ test('a PostgreSQL refusal to release preserves local counters and allows a late assert.equal((await row()).released_at, null) assert.equal(pool.inflight.get('release-account'), 1) assert.equal(pool.vmInflight.get('release-vm'), 1) + assert.equal(pool.planner.seatOf('release-seat').inflight, 1) await db.exec('DROP TRIGGER skip_scheduler_release ON execution_admissions') await selected.release() assert.ok((await row()).released_at) assert.equal(pool.inflight.has('release-account'), false) assert.equal(pool.vmInflight.has('release-vm'), false) + assert.equal(pool.planner.seatOf('release-seat'), null) } finally { await db.exec('DROP TRIGGER IF EXISTS skip_scheduler_release ON execution_admissions') await service.release(selected) From cdd5b886eb8b7f5cbefb37c5cc6e8705b51e28eb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:48:38 +0800 Subject: [PATCH 090/473] test(kernel): reproduce completion of unbound execution --- test/unit/kernel-router.test.mjs | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/unit/kernel-router.test.mjs b/test/unit/kernel-router.test.mjs index 3ec21e10c..a397d4281 100644 --- a/test/unit/kernel-router.test.mjs +++ b/test/unit/kernel-router.test.mjs @@ -345,6 +345,46 @@ unixTest('preparation rejection identifies that no kernel execution started', as assert.deepEqual(result.execution, { requestId: null, started: false, ended: true, reason: 'not_started' }) }) +for (const [mode, dispatch] of [ + ['call', dispatchCallInference], + ['stream', dispatchStreamInference], +]) { + for (const scenario of ['pre-aborted', 'mock']) { + for (const durableStart of [true, false]) { + test(`${mode} ${scenario} ${durableStart ? 'does not complete an unbound job' : 'keeps end-only observation'}`, async () => { + const env = ['KIN_KERNEL_BIN', 'KIN_CRS_MOCK', 'KIN_CRS_TRACE_FILE', 'KIN_MOCK_TRACE_FILE'] + const previous = Object.fromEntries(env.map((key) => [key, process.env[key]])) + process.env.KIN_KERNEL_BIN = '/bin/true' + process.env.KIN_CRS_MOCK = scenario === 'mock' ? '1' : '0' + delete process.env.KIN_CRS_TRACE_FILE + delete process.env.KIN_MOCK_TRACE_FILE + let starts = 0 + const ended = [] + try { + const result = await dispatch({ + exec: { vmId: 'synthetic-unbound', vm: { id: 'synthetic-unbound' } }, + body: { model: 'claude-test', messages: [{ role: 'user', content: 'synthetic request' }] }, + signal: scenario === 'pre-aborted' ? AbortSignal.abort() : undefined, + ensureRust: async () => ({ ok: true }), + ...(durableStart ? { onExecutionStart: async () => starts++ } : {}), + onExecutionEnd: async (execution) => ended.push(execution), + }) + assert.equal(starts, 0) + assert.equal(result.execution.started, false) + assert.equal(result.execution.ended, true) + assert.equal(ended.length, durableStart ? 0 : 1) + if (!durableStart) assert.deepEqual(ended[0], result.execution) + } finally { + for (const key of env) { + if (previous[key] == null) delete process.env[key] + else process.env[key] = previous[key] + } + } + }) + } + } +} + unixTest('dispatchStreamInference uses rust socket when ready', async () => { const prev = process.env.KIN_KERNEL_BIN process.env.KIN_KERNEL_BIN = '/bin/true' From db2d1806051e220561434da98b7310e258cc04d2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:49:11 +0800 Subject: [PATCH 091/473] fix(kernel): complete only registered shared jobs --- src/lib/transport/kernel-router.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/transport/kernel-router.mjs b/src/lib/transport/kernel-router.mjs index 55e9803d3..f40ba9e5f 100644 --- a/src/lib/transport/kernel-router.mjs +++ b/src/lib/transport/kernel-router.mjs @@ -273,6 +273,7 @@ async function runHop({ mode, opts }) { let hop = 0 const sendHop = async () => { let startError + let bound = false const value = await send({ ...opts, requestId: hop++ === 0 && opts.requestId != null ? opts.requestId : randomUUID(), @@ -281,6 +282,7 @@ async function runHop({ mode, opts }) { ? async (execution) => { try { await opts.onExecutionStart(execution) + bound = true } catch (error) { startError = error throw error @@ -291,7 +293,9 @@ async function runHop({ mode, opts }) { // The worker returns transport results for errors, including callback errors. // A failed durable start must retain the original exception and cannot retry. if (startError) throw startError - if (typeof opts.onExecutionEnd === 'function') await opts.onExecutionEnd(value?.execution) + if (typeof opts.onExecutionEnd === 'function' && (typeof opts.onExecutionStart !== 'function' || bound)) { + await opts.onExecutionEnd(value?.execution) + } return value } beginWrapHop(opts.exec) From bbf52c8e9fd6d3a2444e50bb2479a6636bd40adb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:52:07 +0800 Subject: [PATCH 092/473] test: require durable worker job binding before releasing execution --- test/unit/execution-jobs.test.mjs | 245 ++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 test/unit/execution-jobs.test.mjs diff --git a/test/unit/execution-jobs.test.mjs b/test/unit/execution-jobs.test.mjs new file mode 100644 index 000000000..1d4f537b3 --- /dev/null +++ b/test/unit/execution-jobs.test.mjs @@ -0,0 +1,245 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' + +async function replicas() { + const db = getDb() + const other = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: db.schema }) + return [ + new ExecutionAdmissionsRepo(db, { ownerId: 'controller-a' }), + new ExecutionAdmissionsRepo(other, { ownerId: 'controller-b' }), + ] +} +function candidate(patch = {}) { + return { accountId: 'account-1', vmId: 'vm-1', accountLimit: 8, vmLimit: 8, rpmLimit: 0, fableLimit: 0, ...patch } +} +function completed(requestId, patch = {}) { + return { requestId, started: true, ended: true, reason: 'eof', ...patch } +} + +test('a persisted pending job prevents release until matching execution completion', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate()) + assert.deepEqual(await held.bindExecution({ requestId: 'hop-1' }), { + bound: true, + admissionId: held.admissionId, + requestId: 'hop-1', + }) + const job = await second.db.prepare('SELECT * FROM execution_jobs WHERE request_id = $1').get('hop-1') + assert.equal(job.admission_id, held.admissionId) + assert.equal(job.ended_at, null) + assert.equal(job.started, null, 'binding precedes submission and cannot prove that execution has started') + assert.equal((await held.release()).released, false) + assert.equal((await second.getCounts(candidate())).accountInflight, 1) + assert.deepEqual(await held.completeExecution(completed('hop-1')), { + ended: true, + uncertain: false, + requestId: 'hop-1', + }) + assert.equal((await held.release()).released, true) + assert.equal((await second.getCounts(candidate())).accountInflight, 0) +}) + +test('independent connections cannot bind two pending jobs to one admission', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate()) + const results = await Promise.allSettled([ + first.bindExecution(held, { requestId: 'hop-a' }), + second.bindExecution(held, { requestId: 'hop-b' }), + ]) + assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1) + assert.equal(results.find((result) => result.status === 'rejected').reason.code, 'execution_job_pending') + assert.equal((await first.db.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 1) + assert.equal((await held.release()).released, false) +}) + +test('a hop ID is globally unique across admissions and owners', async () => { + const [first, second] = await replicas() + const a = await first.tryReserve(candidate()) + const b = await second.tryReserve(candidate()) + await a.bindExecution({ requestId: 'shared-hop' }) + await assert.rejects(b.bindExecution({ requestId: 'shared-hop' }), { code: 'execution_job_exists' }) + assert.equal((await b.release()).released, true) + assert.equal((await a.release()).released, false) +}) + +test('uncertain execution persists its job and admission and cannot be automatically cleared by late completion', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate({ accountLimit: 1 })) + await held.bindExecution({ requestId: 'uncertain-hop' }) + assert.deepEqual( + await held.completeExecution(completed('uncertain-hop', { ended: false, reason: 'transport_error' })), + { ended: false, uncertain: true, requestId: 'uncertain-hop' }, + ) + assert.equal((await held.release()).released, false) + assert.equal((await second.tryReserve(candidate({ accountLimit: 1 }))).code, 'concurrency_limit') + const late = await held.completeExecution(completed('uncertain-hop')) + assert.equal(late.ended, true) + assert.equal(late.uncertain, true) + assert.equal((await held.release()).released, false) + await assert.rejects(held.bindExecution({ requestId: 'late-next-hop' }), { code: 'execution_admission_inactive' }) +}) + +test('missing or malformed completion metadata preserves the pending job and marks uncertainty', async () => { + const [first, second] = await replicas() + for (const [i, metadata] of [ + undefined, + {}, + { requestId: 'missing' }, + { requestId: 'missing', started: true, ended: 1, reason: 'eof' }, + ].entries()) { + const held = await first.tryReserve(candidate({ accountId: `account-${i}`, vmId: `vm-${i}` })) + await held.bindExecution({ requestId: `hop-${i}` }) + await assert.rejects(held.completeExecution(metadata), { code: 'invalid_execution_job' }) + assert.equal((await held.release()).released, false) + const row = await second.db + .prepare('SELECT uncertain_at FROM execution_admissions WHERE id = $1') + .get(held.admissionId) + assert.ok(row.uncertain_at) + assert.equal( + (await second.db.prepare('SELECT ended_at FROM execution_jobs WHERE request_id = $1').get(`hop-${i}`)).ended_at, + null, + ) + } +}) + +test('completion cannot close a job belonging to another admission', async () => { + const [first, second] = await replicas() + const a = await first.tryReserve(candidate()) + const b = await second.tryReserve(candidate()) + await a.bindExecution({ requestId: 'hop-a' }) + await b.bindExecution({ requestId: 'hop-b' }) + await assert.rejects(a.completeExecution(completed('hop-b')), { code: 'execution_job_mismatch' }) + assert.equal( + (await second.db.prepare('SELECT ended_at FROM execution_jobs WHERE request_id = $1').get('hop-b')).ended_at, + null, + ) + assert.equal((await b.release()).released, false) + assert.equal( + (await second.db.prepare('SELECT uncertain_at FROM execution_admissions WHERE id = $1').get(b.admissionId)) + .uncertain_at, + null, + ) +}) + +test('late completion of an earlier job cannot release or change a later pending job', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate()) + await held.bindExecution({ requestId: 'hop-first' }) + await held.completeExecution(completed('hop-first')) + await held.bindExecution({ requestId: 'hop-next' }) + assert.equal((await held.completeExecution(completed('hop-first'))).ended, true) + assert.equal((await held.release()).released, false) + assert.equal( + (await second.db.prepare('SELECT ended_at FROM execution_jobs WHERE request_id = $1').get('hop-next')).ended_at, + null, + ) + await held.completeExecution(completed('hop-next')) + assert.equal((await held.release()).released, true) +}) + +test('a wrong owner cannot bind, complete or release another controller execution', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate()) + const wrong = { admissionId: held.admissionId, ownerId: 'controller-b' } + await assert.rejects(second.bindExecution(wrong, { requestId: 'hop-wrong' }), { + code: 'execution_admission_inactive', + }) + await held.bindExecution({ requestId: 'hop-owner' }) + await assert.rejects(second.completeExecution(wrong, completed('hop-owner')), { + code: 'execution_admission_inactive', + }) + assert.equal((await second.release(wrong)).released, false) + assert.equal( + (await second.db.prepare('SELECT uncertain_at FROM execution_admissions WHERE id = $1').get(held.admissionId)) + .uncertain_at, + null, + ) +}) + +test('mock and explicitly unstarted jobs can complete only through their matching hop ID', async () => { + const [first] = await replicas() + for (const reason of ['mock', 'not_started']) { + const held = await first.tryReserve(candidate()) + await held.bindExecution({ requestId: `hop-${reason}` }) + assert.equal((await held.completeExecution(completed(`hop-${reason}`, { started: false, reason }))).ended, true) + const row = await first.db + .prepare('SELECT started, ended_at FROM execution_jobs WHERE request_id = $1') + .get(`hop-${reason}`) + assert.equal(row.started, false) + assert.ok(row.ended_at) + assert.equal((await held.release()).released, true) + } +}) + +test('aging and closing the binding connection cannot remove or release a pending job', async () => { + const [first, second] = await replicas() + const held = await second.tryReserve(candidate({ accountLimit: 1 })) + await held.bindExecution({ requestId: 'old-hop' }) + await first.db.exec("UPDATE execution_jobs SET created_at = clock_timestamp() - interval '2 days'") + await first.db.exec("UPDATE execution_admissions SET created_at = clock_timestamp() - interval '2 days'") + await second.db.close() + assert.equal((await first.tryReserve(candidate({ accountLimit: 1 }))).code, 'concurrency_limit') + assert.equal((await first.release(held)).released, false) + assert.equal( + (await first.db.prepare('SELECT count(*) AS count FROM execution_jobs WHERE ended_at IS NULL').get()).count, + 1, + ) +}) + +test('a database failure during completion retains both the pending job and admission', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate()) + await held.bindExecution({ requestId: 'failure-hop' }) + await first.db.exec(`CREATE FUNCTION reject_job_completion() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'simulated completion database failure' USING ERRCODE = 'P0001'; END $$; + CREATE TRIGGER reject_job_completion BEFORE UPDATE OF ended_at ON execution_jobs + FOR EACH ROW EXECUTE FUNCTION reject_job_completion();`) + await assert.rejects(held.completeExecution(completed('failure-hop')), { code: 'P0001' }) + assert.equal( + (await second.db.prepare('SELECT ended_at FROM execution_jobs WHERE request_id = $1').get('failure-hop')).ended_at, + null, + ) + assert.equal((await held.release()).released, false) + assert.equal((await second.getCounts(candidate())).accountInflight, 1) +}) + +test('release waits for a concurrent binding transaction before checking pending jobs', async () => { + const [first, second] = await replicas() + const held = await first.tryReserve(candidate()) + let bound + const waiting = new Promise((resolve) => { + bound = resolve + }) + let commit + const proceed = new Promise((resolve) => { + commit = resolve + }) + const binding = withTransaction(first.db, async () => { + await held.bindExecution({ requestId: 'race-hop' }) + bound() + await proceed + }) + await waiting + let releaseEntered + const entered = new Promise((resolve) => { + releaseEntered = resolve + }) + const query = second.db.query.bind(second.db) + second.db.query = (...args) => { + const result = query(...args) + if (args[0].includes('execution_admissions')) releaseEntered() + return result + } + const release = second.release(held) + await entered + commit() + await binding + assert.equal((await release).released, false) + assert.equal( + (await first.db.prepare('SELECT released_at FROM execution_admissions WHERE id = $1').get(held.admissionId)) + .released_at, + null, + ) +}) From 0c19de183db87e5b5b2836b29e12540e965b0306 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:52:52 +0800 Subject: [PATCH 093/473] test(k8s): verify shared kernel hop lifecycle with PostgreSQL --- test/unit/shared-hop-lifecycle.test.mjs | 289 ++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 test/unit/shared-hop-lifecycle.test.mjs diff --git a/test/unit/shared-hop-lifecycle.test.mjs b/test/unit/shared-hop-lifecycle.test.mjs new file mode 100644 index 000000000..00a0a22f9 --- /dev/null +++ b/test/unit/shared-hop-lifecycle.test.mjs @@ -0,0 +1,289 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' +import { dispatchStreamInference } from '../../src/lib/transport/kernel-router.mjs' + +const unixTest = process.platform === 'win32' ? test.skip : test +const accountId = 'account-lifecycle' +const vmId = 'vm-lifecycle' +const model = 'claude-test' +const seatKey = 'device-lifecycle' +const scopedSeatKey = `platform|${seatKey}` + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +function success(res) { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + for (const event of [ + { type: 'message_start', message: { id: 'msg-lifecycle', type: 'message', role: 'assistant', model, content: [] } }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'done' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 1 } }, + { type: 'message_stop' }, + ]) { + res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) + } + res.end() +} + +/** Two controller instances share PostgreSQL; the kernel is a local Unix socket mock. */ +async function fixture(t, handleMessage, handleCancel = (_job, _req, res) => res.end('{}')) { + const env = ['VM2API_CONTROL_PLANE_MODE', 'KIN_KERNEL_BIN', 'KIN_CRS_MOCK'] + const previous = Object.fromEntries(env.map((key) => [key, process.env[key]])) + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + process.env.KIN_KERNEL_BIN = '/bin/true' + process.env.KIN_CRS_MOCK = '0' + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-hop-')) + const runDir = path.join(root, 'vms', vmId, 'run') + fs.mkdirSync(runDir, { recursive: true }) + fs.mkdirSync(path.join(root, 'vms', vmId, 'cli-home')) + const socket = path.join(runDir, 'kernel.sock') + const token = path.join(runDir, 'internal.token') + fs.writeFileSync(token, 'synthetic-internal-token\n', { mode: 0o600 }) + fs.writeFileSync( + path.join(root, 'vms', `${vmId}.json`), + JSON.stringify({ + id: vmId, + status: 'running', + schedulable: true, + proxy_cli_enabled: true, + proxy: { id: 'synthetic-proxy', url: 'socks5h://127.0.0.1:11001' }, + runtime: { kernel_socket: socket, worker_run_dir: runDir, worker_token_file: token }, + policy: { maxConcurrency: 1, concurrencyOverride: true }, + claude: { + account_uuid: accountId, + account_tier: 'max', + access_token: 'synthetic-access', + refresh_token: 'synthetic-refresh', + expires_at: Math.floor(Date.now() / 1000) + 3600, + }, + }), + ) + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const pools = [db, other].map( + (connection) => + new PoolScheduler({ + projectRoot: root, + accountQuota: { + db: connection, + canAccept: () => ({ ok: true }), + limitFor: () => 2, + policyFor: () => ({}), + rpmLimitFor: () => 0, + }, + workerHealth: async () => ({ ok: true, credential: { generation: 1, has_access: true } }), + config: { seat_grace_ms: 0, default_session_slots: 1 }, + }), + ) + const runners = pools.map( + (scheduler) => + new FailoverRunner({ + scheduler, + config: { max_total_attempts: 3, max_account_switches: 2, total_retry_deadline_ms: 1000 }, + }), + ) + const messages = [] + const cancellations = [] + const server = http.createServer((req, res) => { + void (async () => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + const job = JSON.parse(Buffer.concat(chunks).toString('utf8')) + if (req.url === '/internal/v1/messages') { + messages.push(job) + await handleMessage(job, req, res) + } else if (req.url === '/internal/v1/cancel') { + cancellations.push(job) + await handleCancel(job, req, res) + } else { + res.writeHead(404).end() + } + })().catch((error) => res.destroy(error)) + }) + t.after(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + try { + for (const pool of pools) await pool.flush() + } finally { + await other.close() + fs.rmSync(root, { recursive: true, force: true }) + for (const key of env) { + if (previous[key] == null) delete process.env[key] + else process.env[key] = previous[key] + } + } + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(socket, resolve) + }) + const selected = [] + return { + db, + pools, + messages, + cancellations, + selected, + counts: () => pools[1].executionAdmissions.getCounts(accountId, vmId), + async peerIsBlocked() { + const candidate = await pools[1].selectAndReserve({ model, allowWait: false }) + try { + assert.equal(candidate.ok, false, 'the other controller cannot reuse an unfinished VM execution') + assert.ok(candidate.wait_reasons.includes('vm_concurrency_limit')) + } finally { + await candidate.release?.() + } + }, + run(index = 0, signal) { + return runners[index].run({ + requestId: `synthetic-request-${selected.length}`, + model, + seatKey, + canonicalBody: { model, messages: [{ role: 'user', content: 'synthetic request' }] }, + stream: true, + signal, + callAttempt({ candidate, body, signal, onCommit }) { + selected.push(candidate) + assert.equal(typeof candidate.bindExecution, 'function') + assert.equal(typeof candidate.completeExecution, 'function') + return dispatchStreamInference({ + exec: candidate.exec, + body, + signal, + onCommit, + onEvent: async () => {}, + timeoutMs: 1000, + ensureRust: async () => ({ ok: true }), + onExecutionStart: candidate.bindExecution, + onExecutionEnd: candidate.completeExecution, + }) + }, + }) + }, + } +} + +unixTest('a shared streamed execution completes before releasing capacity and its device slot', async (t) => { + const fx = await fixture(t, (_job, _req, res) => success(res)) + const result = await fx.run() + assert.equal(result.ok, true) + assert.equal(result.execution.ended, true) + assert.equal(fx.messages.length, 1) + assert.equal((await fx.counts()).vmInflight, 0) + assert.equal(await fx.pools[1].executionAdmissions.getSeat(scopedSeatKey), null) + const completed = await fx.db + .prepare(`SELECT j.request_id, j.ended_at, a.released_at FROM execution_jobs j + JOIN execution_admissions a ON a.id = j.admission_id`) + .get() + assert.equal(completed.request_id, fx.messages[0].request_id) + assert.ok(completed.ended_at) + assert.ok(completed.released_at >= completed.ended_at) + const peerResult = await fx.run(1) + assert.equal(peerResult.ok, true) + assert.equal(fx.messages.length, 2) + assert.deepEqual( + fx.selected.map((candidate) => candidate.slotIndex), + [0, 0], + ) + assert.equal((await fx.counts()).accountInflight, 0) +}) + +unixTest('a truncated shared kernel socket cannot retry or release capacity to another controller', async (t) => { + const fx = await fixture(t, (_job, _req, res) => res.destroy()) + const result = await fx.run() + assert.equal(result.ok, false) + assert.equal(result.executionUncertain, true) + assert.equal(fx.messages.length, 1, 'neither the router nor the failover runner may submit a second job') + assert.equal((await fx.counts()).vmInflight, 1) + assert.equal((await fx.pools[1].executionAdmissions.getSeat(scopedSeatKey)).inflight, 1) + await fx.peerIsBlocked() +}) + +for (const [phase, trigger, code, expectedExecutions] of [ + ['binding', 'INSERT', 'P6481', 0], + ['completion', 'UPDATE OF ended_at', 'P6482', 1], +]) { + unixTest(`PostgreSQL ${phase} failure prevents replay and retains shared capacity`, async (t) => { + const fx = await fixture(t, (_job, _req, res) => success(res)) + // This trigger exists only in the per-test schema and leaves admission uncertainty writes available. + await fx.db.exec(`CREATE FUNCTION reject_lifecycle_job() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic execution ${phase} failure' USING ERRCODE = '${code}'; END; + $$; + CREATE TRIGGER reject_lifecycle_job BEFORE ${trigger} ON execution_jobs + FOR EACH ROW EXECUTE FUNCTION reject_lifecycle_job();`) + const result = await fx.run() + assert.equal(result.ok, false) + assert.equal(result.executionUncertain, true) + assert.equal(result.body.error.code, code, 'the original PostgreSQL failure reaches the request result') + assert.equal(fx.messages.length, expectedExecutions) + assert.equal(fx.selected.length, 1, 'persistence failure must not select a new attempt') + assert.equal((await fx.counts()).vmInflight, 1) + const rows = await fx.db.prepare('SELECT * FROM execution_jobs').all() + assert.equal(rows.length, expectedExecutions) + if (rows.length) { + assert.equal(rows[0].request_id, fx.messages[0].request_id) + assert.equal(rows[0].ended_at, null, 'failed completion does not create terminal evidence') + } + await fx.peerIsBlocked() + }) +} + +unixTest( + 'a delayed successful cancel response does not prove a shared execution ended', + { timeout: 5000 }, + async (t) => { + const received = deferred() + const cancelReceived = deferred() + const answerCancel = deferred() + const cancelAnswered = deferred() + t.after(() => answerCancel.resolve()) + const fx = await fixture( + t, + (_job, _req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.flushHeaders() + received.resolve() + }, + async (_job, _req, res) => { + cancelReceived.resolve() + await answerCancel.promise + res.end(JSON.stringify({ ok: true, cancelled: true }), () => cancelAnswered.resolve()) + }, + ) + const controller = new AbortController() + const pending = fx.run(0, controller.signal) + try { + await Promise.race([received.promise, pending.then(() => assert.fail('execution did not reach the kernel'))]) + controller.abort() + await cancelReceived.promise + const result = await pending + assert.equal(result.clientCancelled, true) + assert.equal(fx.messages.length, 1) + assert.equal(fx.cancellations[0].request_id, fx.messages[0].request_id) + assert.equal((await fx.counts()).vmInflight, 1, 'capacity is retained while cancellation is pending') + await fx.peerIsBlocked() + answerCancel.resolve() + await cancelAnswered.promise + assert.equal((await fx.counts()).vmInflight, 1, 'HTTP cancel acceptance is not a native terminal acknowledgement') + await fx.peerIsBlocked() + } finally { + controller.abort() + answerCancel.resolve() + await pending + } + }, +) From 4dd1933f8c0261e4ca69f44e4410606ecf7fd5d2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:52:57 +0800 Subject: [PATCH 094/473] feat: persist worker job bindings until execution ends --- .../007_execution_jobs.sql | 13 +++ .../db/repos/execution-admissions-repo.mjs | 91 ++++++++++++++++++- 2 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 src/lib/db/postgres-migrations/007_execution_jobs.sql diff --git a/src/lib/db/postgres-migrations/007_execution_jobs.sql b/src/lib/db/postgres-migrations/007_execution_jobs.sql new file mode 100644 index 000000000..da99d990f --- /dev/null +++ b/src/lib/db/postgres-migrations/007_execution_jobs.sql @@ -0,0 +1,13 @@ +CREATE TABLE execution_jobs ( + request_id text COLLATE "C" PRIMARY KEY, + admission_id text COLLATE "C" NOT NULL REFERENCES execution_admissions (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ended_at timestamptz, + started boolean, + reason text COLLATE "C" +); + +CREATE INDEX idx_execution_jobs_admission_pending ON execution_jobs (admission_id) WHERE ended_at IS NULL; + +-- Binding happens before submission; NULL started means it is not yet known. +-- Pending jobs have no expiry. Admission history is removed only after release. diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index d6a86d80b..5563697ed 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -19,6 +19,10 @@ function denied(code, inflight, max, extra = {}) { return { ok: false, code, reason: code, detail: { inflight, max }, ...extra } } +function jobError(code) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} + /** Short PostgreSQL transactions for global account/VM execution limits. */ export class ExecutionAdmissionsRepo { constructor(db, { ownerId = 'controller_' + crypto.randomUUID() } = {}) { @@ -122,6 +126,8 @@ export class ExecutionAdmissionsRepo { record, release: (options) => this.release(handle, options), markUncertain: () => this.markUncertain(handle), + bindExecution: (execution) => this.bindExecution(handle, execution), + completeExecution: (execution) => this.completeExecution(handle, execution), } }) } @@ -134,12 +140,87 @@ export class ExecutionAdmissionsRepo { return { uncertain: result.changes > 0 } } + async _lockAdmission({ admissionId, ownerId }) { + return this.db + .prepare('SELECT * FROM execution_admissions WHERE id = $1 AND owner_id = $2 FOR UPDATE') + .get(admissionId, ownerId) + } + + async bindExecution(handle, execution) { + return withTransaction(this.db, async () => { + const admission = await this._lockAdmission(handle) + if (!admission || admission.released_at || admission.uncertain_at) { + throw jobError('execution_admission_inactive') + } + if (typeof execution?.requestId !== 'string' || !execution.requestId.trim()) { + throw jobError('invalid_execution_job') + } + const pending = await this.db + .prepare('SELECT request_id FROM execution_jobs WHERE admission_id = $1 AND ended_at IS NULL LIMIT 1') + .get(handle.admissionId) + if (pending) throw jobError('execution_job_pending') + const row = await this.db + .prepare(`INSERT INTO execution_jobs (request_id, admission_id) VALUES ($1, $2) + ON CONFLICT (request_id) DO NOTHING RETURNING request_id`) + .get(execution.requestId, handle.admissionId) + if (!row) throw jobError('execution_job_exists') + return { bound: true, admissionId: handle.admissionId, requestId: execution.requestId } + }) + } + + async completeExecution(handle, execution) { + const result = await withTransaction(this.db, async () => { + const admission = await this._lockAdmission(handle) + if (!admission) throw jobError('execution_admission_inactive') + const valid = + typeof execution?.requestId === 'string' && + execution.requestId.trim() && + typeof execution.started === 'boolean' && + typeof execution.ended === 'boolean' && + typeof execution.reason === 'string' && + execution.reason.trim() + if (!valid) { + await this.markUncertain(handle) + // Commit the uncertainty before reporting the invalid metadata. + return { error: 'invalid_execution_job' } + } + const job = await this.db + .prepare('SELECT * FROM execution_jobs WHERE request_id = $1 AND admission_id = $2') + .get(execution.requestId, handle.admissionId) + if (!job) { + await this.markUncertain(handle) + return { error: 'execution_job_mismatch' } + } + if (job.ended_at) return { ended: true, uncertain: !!admission.uncertain_at, requestId: execution.requestId } + await this.db + .prepare(`UPDATE execution_jobs SET + ended_at = CASE WHEN $1 THEN clock_timestamp() ELSE ended_at END, + started = COALESCE(started, false) OR $2, reason = $3 + WHERE request_id = $4 AND admission_id = $5 AND ended_at IS NULL`) + .run(execution.ended, execution.started, execution.reason, execution.requestId, handle.admissionId) + if (!execution.ended) await this.markUncertain(handle) + return { + ended: execution.ended, + uncertain: !execution.ended || !!admission.uncertain_at, + requestId: execution.requestId, + } + }) + if (result.error) throw jobError(result.error) + return result + } + async release(handle, { uncertain = false } = {}) { if (uncertain) return this.markUncertain(handle) - const result = await this.db - .prepare(`UPDATE execution_admissions SET released_at = clock_timestamp() - WHERE id = $1 AND owner_id = $2 AND released_at IS NULL AND uncertain_at IS NULL`) - .run(handle.admissionId, handle.ownerId) - return { released: result.changes > 0 } + return withTransaction(this.db, async () => { + // Lock before the pending-job query so a concurrent bind commits before + // this statement takes its snapshot. An UPDATE alone can use an older one. + if (!(await this._lockAdmission(handle))) return { released: false } + const result = await this.db + .prepare(`UPDATE execution_admissions SET released_at = clock_timestamp() + WHERE id = $1 AND owner_id = $2 AND released_at IS NULL AND uncertain_at IS NULL + AND NOT EXISTS (SELECT 1 FROM execution_jobs j WHERE j.admission_id = execution_admissions.id AND j.ended_at IS NULL)`) + .run(handle.admissionId, handle.ownerId) + return { released: result.changes > 0 } + }) } } From fb90450d06fd3aaaee8a0a4b52aca13d2f675431 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 18:53:31 +0800 Subject: [PATCH 095/473] test(k8s): verify execution job migrations and backup recovery --- test/unit/backup-service.test.mjs | 7 ++++--- test/unit/database.test.mjs | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index f20ef95c2..920a09711 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,7 +79,7 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", + "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() @@ -96,16 +96,17 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'gateway_rate_buckets', 'execution_admissions', 'execution_seats', + 'execution_jobs', ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 6) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 7) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", + "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index ed1f78e06..6f91fa5c6 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 35) + assert.equal(rows.length, 36) for (const name of [ 'settings', 'users', @@ -50,6 +50,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'gateway_rate_buckets', 'execution_admissions', 'execution_seats', + 'execution_jobs', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -110,7 +111,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 6) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 7) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -126,7 +127,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 6) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 7) fs.rmSync(dir, { recursive: true, force: true }) }) From d5beeba981a822914fb39fded754c611adf8638f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:03:44 +0800 Subject: [PATCH 096/473] docs(k8s): record shared execution verification and remaining scale-out work --- .../2026-10-21-kubernetes-control-plane.md | 16 +- ...2026-10-21-kubernetes-execution-results.md | 53 +++++ .../baseline-unit-recheck.json | 80 +++++++ .../execution-state/verification.json | 206 ++++++++++++++++++ 4 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 docs/plans/2026-10-21-kubernetes-execution-results.md create mode 100644 docs/plans/evidence/kubernetes/execution-state/baseline-unit-recheck.json create mode 100644 docs/plans/evidence/kubernetes/execution-state/verification.json diff --git a/docs/plans/2026-10-21-kubernetes-control-plane.md b/docs/plans/2026-10-21-kubernetes-control-plane.md index 1d998261e..52a018c09 100644 --- a/docs/plans/2026-10-21-kubernetes-control-plane.md +++ b/docs/plans/2026-10-21-kubernetes-control-plane.md @@ -3,7 +3,13 @@ 日期：2026-10-21。分支：`codex/kubernetes-control-plane`。 依赖：`codex/postgresql-migration@cf9462d5`（#647）。 -## 当前范围和完成条件 +## 目标和验收条件 + +目标为 #648 的 Kubernetes 部署与多活动控制面水平扩容。至少两个副本须部署在不同节点，同时承载重叠请求；认证、配置、业务数据和全局限额保持一致；后台任务不重复，异常退出后能够安全接管；HTTP、SSE、WebSocket、扩缩容和滚动更新均需运行验证。全部推理转交给单个 leader 不能代替此验收。 + +当前 Draft PR #651 和以下第一阶段结果尚未满足全部条件。后续增量继续在同一分支实现，不关闭 #648。 + +## 第一阶段范围和完成条件（历史） 本次先提交可运行、可评审的 Kubernetes 部署基础 Draft PR。默认只允许一个活动控制面。控制面多副本同时处理推理尚未具备安全条件，本 PR 不关闭 #648，也不将单副本部署称为水平扩容。 @@ -57,3 +63,11 @@ 4. 全部条件具备后再允许两个活动副本，补充 HPA、PodDisruptionBudget 和多节点部署，并验证并发、滚动更新、缩容与故障接管。 本轮运行证据及未验证项将在增量验证报告中记录。第一阶段的镜像 digest 与 k3s 结果只覆盖当时源码，不能用于证明本轮新代码已部署。 + +## 共享执行准入增量（进行中） + +`fb90450d` 已加入账号/VM/Fable/RPM 的共享准入、设备座位与 generation、发送前实际 kernel request ID 绑定，以及未知执行保留和停止重试。设备预算按账号跨 VM 计算。standalone 保留原同步调度路径。详细结果见 [共享执行验证](2026-10-21-kubernetes-execution-results.md)。全量 unit 为 3493 pass、7 fail、10 skip；全量 E2E 为 103 pass、17 fail、1 skip。失败差异和基线依据均在报告中列出。 + +当前队列仍为进程内队列。同会话顺序和 Sticky 的跨副本条件更新未完成。HTTP/SSE EOF 终态还依赖正常 kernel 协议，尚无执行端 fencing；控制面死亡后的占位没有自动回收。独占锁和单副本限制继续保留。 + +下一阶段需优先拆除整行快照覆盖：AccountQuota 的计数改为 PG 原子增量，响应头/probe 改为有数据库观测顺序的字段补丁，身份与并发/RPM 配置使用管理补丁。`panel-api` 的显示检查、Fable probe、手工清理以及 `panel-routes` 的身份修改也会调用整行 `save`，需要同时覆盖。保留已有 bigint 精度和 `countRequest:false` 行为。ProxyPool 的 probe 当前仍调用 `replaceAll` 删除重建代理表，需要拆成目标行配置、健康、绑定及删除操作，并校验 probe 对应的配置版本。 diff --git a/docs/plans/2026-10-21-kubernetes-execution-results.md b/docs/plans/2026-10-21-kubernetes-execution-results.md new file mode 100644 index 000000000..f7b3f10ab --- /dev/null +++ b/docs/plans/2026-10-21-kubernetes-execution-results.md @@ -0,0 +1,53 @@ +# Kubernetes 共享执行准入增量（#648，进行中） + +本轮继续实现账号/VM 准入、设备座位与执行状态绑定。两个活动控制面尚未启用。`replicas: 1`、Recreate 和独占控制面锁仍保留，本轮结果不满足 #648 的多节点验收条件。 + +## 变更范围 + +- `execution_admissions` 原子协调账号并发、VM 并发、账号 RPM 和 Fable 并发。账号和 VM 上限分别计算。释放后保留 RPM 窗口记录；活动记录没有 TTL，不因时间经过或控制面断开而重新分配。 +- `execution_seats` 与请求准入在同一事务中分配。设备的 VM、slot index 和 generation 从 PG 读取。有效座位数量按 VM 检查容量，按账号检查预算；同一账号跨 VM 共享预算。未知执行继续持有座位；最后一个已确认请求释放后才开始 grace。 +- `execution_jobs` 在发送前将实际 kernel request ID 绑定到持久准入。同一准入不能同时绑定两个未结束作业。释放在锁定准入行后重新检查 pending 作业；迟到的旧作业结果不能释放后来绑定的作业。 +- Kubernetes 调度器等待异步准入，进程内 inflight 不覆盖 PG 的容量判断。保留 standalone 同步路径。异步排队处理拒绝、错误、取消后的迟到准入及释放；退出时等待调度器任务。等待队列目前仍属于各进程，未实现全局 FIFO。 +- 工作客户端返回实际 kernel request ID 与 `execution.started/ended/reason`。`ended=true` 由完整 HTTP 响应或 SSE EOF 推断，依赖正常 kernel 协议在结束响应前停止对应执行的约定，尚不是独立的执行端终止证明。超时、连接中断和取消请求的成功返回不能作为执行结束证明。此元数据仅供控制面内部使用。 +- 共享准入下，kernel router 等待作业结果（包括结束未确认）写入 PG。未确认结束时禁止隐式重试。FailoverRunner 同时停止跨账号重试，并保留账号/VM 和 managed key 并发。用量结算与执行结束分别判断。 +- 作业绑定或完成记录失败时不继续发送新执行。mock 和预先取消而未提交的请求不完成未注册的作业。 + +## 修复前的实际差异 + +先提交失败测试，再提交对应修复。除新增行为规格外，本轮独立审查还复现了以下问题： + +- 迁移被目标 VM 拒绝后，原 grace 座位实际为 `null`，预期保留原 VM 和 generation。 +- 同一账号跨两台 VM，各新座位预留 0.02，账号 headroom 0.03。第二个座位实际获准，预期 `seat_budget` 拒绝。 +- 第一次 PG 释放失败后，后续 release 重复返回同一个拒绝的 Promise，没有重试 SQL。 +- 未知状态标记尚在持久化时，普通 release 曾先释放容量，PG inflight 实际为 0，预期保留 1。 +- 截断传输曾触发第二次 kernel 请求，实际作业数 2，预期 1。 +- mock 和预先取消未提交的请求曾调用完成回调，实际调用 1 次，预期 0 次。 + +## 验证结果 + +验证提交为 `fb90450d06fd3aaaee8a0a4b52aca13d2f675431`。测试使用本任务 PostgreSQL 17.11 的随机 schema、Node.js 22.23.2 及 Unix socket 合成 kernel；没有调用真实模型，也没有修改现有业务环境。 + +| 检查 | 结果 | +| --- | --- | +| 本轮变更的 13 个测试文件 | 181/181 通过，包含迁移/备份 31 项和 Unix socket 共享执行生命周期 5 项 | +| 全量 unit | 301 文件，3510 项：3493 通过、7 失败、10 跳过 | +| 全量 E2E | 28 文件，121 项：103 通过、17 失败、1 跳过 | +| Node 语法、Biome | 本轮 22 个 MJS 文件通过 | +| `git diff --check 919c4e5e..fb90450d` | 通过 | + +全量回归仍有失败。7 项 unit 失败已在原始 `eee8f6ecb6ca04455e6edef0c8e498fb3be21f21` 上重新执行；测试名称、断言位置及实际/预期差异逐项相同。例如实际 TTL 为 `1h`，预期 `undefined`。归档中 698 个源码、测试、脚本及 package 文件与该提交的 git blob 核对一致。 + +17 项 E2E 失败的断言文本与上一轮已核实基线记录相同，本轮未重新运行原始提交的整套 E2E。例如 Responses 实际返回 400 `protocol_not_allowed`，预期 200；classifier 的 thinking 实际多出 `display: summarized`。未将这些失败改成通过或跳过。 + +结构化结果与日志摘要见 [验证元数据](evidence/kubernetes/execution-state/verification.json) 和 [原始 unit 复核](evidence/kubernetes/execution-state/baseline-unit-recheck.json)。上一轮原始 E2E 对照见 [既有基线记录](evidence/kubernetes/shared-state/baseline-e2e.json)。本轮没有重新部署容器，验证 SHA 仅标识被测试的源码。 + +## 尚未完成 + +- 同会话跨副本顺序、全局等待队列、Sticky generation 的数据库条件更新。 +- AccountQuota、ProxyPool、SchedulingRuntime 的字段级原子更新及配置同步优化。 +- Codex/Search 全部执行和会话状态的共享协调。 +- 执行端 epoch/fencing、未知作业的终止证明、控制面死亡后占位的安全接管。当前保留未知占位，尚未提供自动恢复。 +- routing、VM/凭据文件共享归属、后台任务唯一执行、全局维护及备份恢复。 +- 两个不同 Kubernetes 节点上的活动副本、HPA、PodDisruptionBudget、滚动更新、缩容和故障接管。 + +第一阶段镜像与 k3s 证据不覆盖本轮新增源码。本轮没有重新部署，不能用旧镜像 digest 证明当前代码生效。Draft PR #651 的远端分支更新仍等待用户确认 push。 diff --git a/docs/plans/evidence/kubernetes/execution-state/baseline-unit-recheck.json b/docs/plans/evidence/kubernetes/execution-state/baseline-unit-recheck.json new file mode 100644 index 000000000..2d7e10113 --- /dev/null +++ b/docs/plans/evidence/kubernetes/execution-state/baseline-unit-recheck.json @@ -0,0 +1,80 @@ +{ + "baselineCommit": "eee8f6ecb6ca04455e6edef0c8e498fb3be21f21", + "baselineTreeCheck": { + "checkedGitBlobs": 698, + "missing": 0, + "different": 0 + }, + "currentTestCommit": "fb90450d06fd3aaaee8a0a4b52aca13d2f675431", + "nodeVersion": "v22.23.2", + "originalRunnerContract": "node --test ; no pg-setup import", + "tests": 29, + "passed": 22, + "failed": 7, + "allDiagnosticsEqual": true, + "results": [ + { + "name": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "assertion": "test/unit/cli-hop-body.test.mjs:34", + "diagnostic": " error: |-\n Expected values to be strictly equal:\n \n '1h' !== undefined\n \n code: 'ERR_ASSERTION'\n name: 'AssertionError'\n actual: '1h'\n operator: 'strictEqual'", + "assertionSha256": "a43e28c8c8a102e389ec8f57d05595f4304e1cbe8f10e5afd7e4f06f218c0e00", + "sameDiagnostic": true, + "sameAssertion": true, + "currentAssertionSha256": "a43e28c8c8a102e389ec8f57d05595f4304e1cbe8f10e5afd7e4f06f218c0e00" + }, + { + "name": "every cli-hop system becomes the identity plus the caller body", + "assertion": "test/unit/cli-hop-body.test.mjs:59", + "diagnostic": " error: |-\n Expected values to be strictly deep-equal:\n + actual - expected\n \n {\n + ttl: '1h',\n type: 'ephemeral'\n }\n \n code: 'ERR_ASSERTION'\n name: 'AssertionError'\n expected:\n type: 'ephemeral'\n actual:\n type: 'ephemeral'\n ttl: '1h'\n operator: 'deepStrictEqual'", + "assertionSha256": "53ea5bcdad72e73478dab67c05fd993567253e722ff47e6681bf5011bd12540f", + "sameDiagnostic": true, + "sameAssertion": true, + "currentAssertionSha256": "53ea5bcdad72e73478dab67c05fd993567253e722ff47e6681bf5011bd12540f" + }, + { + "name": "cli-hop normalizes mixed caller markers to the official template", + "assertion": "test/unit/cli-hop-body.test.mjs:81", + "diagnostic": " error: |-\n Expected values to be strictly deep-equal:\n + actual - expected\n \n {\n + ttl: '1h',\n type: 'ephemeral'\n }\n \n code: 'ERR_ASSERTION'\n name: 'AssertionError'\n expected:\n type: 'ephemeral'\n actual:\n type: 'ephemeral'\n ttl: '1h'\n operator: 'deepStrictEqual'", + "assertionSha256": "53ea5bcdad72e73478dab67c05fd993567253e722ff47e6681bf5011bd12540f", + "sameDiagnostic": true, + "sameAssertion": true, + "currentAssertionSha256": "53ea5bcdad72e73478dab67c05fd993567253e722ff47e6681bf5011bd12540f" + }, + { + "name": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "assertion": "test/unit/cli-hop-body.test.mjs:377", + "diagnostic": " error: |-\n Expected values to be strictly deep-equal:\n + actual - expected\n \n {\n + ttl: '5m',\n type: 'ephemeral'\n }\n \n code: 'ERR_ASSERTION'\n name: 'AssertionError'\n expected:\n type: 'ephemeral'\n actual:\n type: 'ephemeral'\n ttl: '5m'\n operator: 'deepStrictEqual'", + "assertionSha256": "f7e05b9b943131bd7a5937a9eef9f7479132d9ab6b1615630be8f52f6e96c479", + "sameDiagnostic": true, + "sameAssertion": true, + "currentAssertionSha256": "f7e05b9b943131bd7a5937a9eef9f7479132d9ab6b1615630be8f52f6e96c479" + }, + { + "name": "cli-hop short history only marks the last message", + "assertion": "test/unit/cli-hop-body.test.mjs:395", + "diagnostic": " error: |-\n Expected values to be strictly deep-equal:\n + actual - expected\n \n [\n {\n blockIndex: 0,\n index: 2,\n + ttl: '1h',\n - ttl: undefined,\n type: 'text'\n }\n ]\n \n code: 'ERR_ASSERTION'\n name: 'AssertionError'\n expected:\n 0:\n index: 2\n blockIndex: 0\n type: 'text'\n actual:\n 0:\n index: 2\n blockIndex: 0\n type: 'text'\n ttl: '1h'\n operator: 'deepStrictEqual'", + "assertionSha256": "90bd2262aa99eacd53591ddeb9b25f30e5aefcd3d591fd8eb768a3c290c140f4", + "sameDiagnostic": true, + "sameAssertion": true, + "currentAssertionSha256": "90bd2262aa99eacd53591ddeb9b25f30e5aefcd3d591fd8eb768a3c290c140f4" + }, + { + "name": "cli-hop uses the inbound session TTL on both message anchors", + "assertion": "test/unit/cli-hop-body.test.mjs:435", + "diagnostic": " error: |-\n Expected values to be strictly deep-equal:\n + actual - expected\n \n [\n + '1h'\n - undefined\n ]\n \n code: 'ERR_ASSERTION'\n name: 'AssertionError'\n expected:\n actual:\n 0: '1h'\n operator: 'deepStrictEqual'", + "assertionSha256": "c34d08b21f1ac688087e5aa9bbd2c65e9c0439d14bae7db90f2ee9e42525bcd8", + "sameDiagnostic": true, + "sameAssertion": true, + "currentAssertionSha256": "c34d08b21f1ac688087e5aa9bbd2c65e9c0439d14bae7db90f2ee9e42525bcd8" + }, + { + "name": "openai.chat multi-turn cli-hop marks last and penultimate user", + "assertion": "test/unit/convert-cache-shape.test.mjs:155", + "diagnostic": " error: |-\n Expected values to be strictly deep-equal:\n + actual - expected\n \n {\n + ttl: '1h',\n type: 'ephemeral'\n }\n \n code: 'ERR_ASSERTION'\n name: 'AssertionError'\n expected:\n type: 'ephemeral'\n actual:\n type: 'ephemeral'\n ttl: '1h'\n operator: 'deepStrictEqual'", + "assertionSha256": "53ea5bcdad72e73478dab67c05fd993567253e722ff47e6681bf5011bd12540f", + "sameDiagnostic": true, + "sameAssertion": true, + "currentAssertionSha256": "53ea5bcdad72e73478dab67c05fd993567253e722ff47e6681bf5011bd12540f" + } + ] +} diff --git a/docs/plans/evidence/kubernetes/execution-state/verification.json b/docs/plans/evidence/kubernetes/execution-state/verification.json new file mode 100644 index 000000000..e8df48f26 --- /dev/null +++ b/docs/plans/evidence/kubernetes/execution-state/verification.json @@ -0,0 +1,206 @@ +{ + "source_commit": "fb90450d06fd3aaaee8a0a4b52aca13d2f675431", + "scope": "shared account/VM/seat admission and persisted kernel job identity; active-active control plane remains disabled", + "node": "22.23.2", + "postgresql": "17.11", + "checks": { + "unit": { + "files": 301, + "totals": { + "tests": 3510, + "pass": 3493, + "fail": 7, + "cancelled": 0, + "skipped": 10 + }, + "failures": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "error_sha256": "1afab608e2476a88783807d547c9b239ad4ac831679731a3ed4d44812b0c31d2", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "error_sha256": "7862d1144e6748d63aa6e98dabf5a9fa7d8d1582f7fbfe851d978eb15f6d5c92", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "error_sha256": "81b2ee3a4bb8b1ab4f351bede4398032dc9637c683b82b146630fd6857e94c2e", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "error_sha256": "01732f4307f58f479e9b3d8329d92a93d9c748c58028276022257d3645e77e87", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + } + ], + "log_sha256": "920d788dd2e5313edb0e672e6b7332153ef4e749c279e4780c4707113048a4b6", + "all_failures_match_previous_verified_error": true, + "verified_commit": "fb90450d06fd3aaaee8a0a4b52aca13d2f675431", + "exit_code": 1 + }, + "e2e": { + "files": 28, + "totals": { + "tests": 121, + "pass": 103, + "fail": 17, + "cancelled": 0, + "skipped": 1 + }, + "failures": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "error_sha256": "a25cb59468e4decc45b1d8b3a66b0a4fbd2195af9f498da39aa83e92169825b5", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "error_sha256": "84e06146cf59485c3b46e05070f6a9a184ecb010a57ef8ab1b62a3ebae90c36a", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "error_sha256": "dfd84d60931072a864eb94f48ce13d4223e963ae919898236f1859bedc8d917e", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "error_sha256": "6ea819807eaee98e6a4c38817c9f011fb426e6a5ff499fa441d8f7bb07371461", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "error_sha256": "c23279127cad70a092d0cc7bb6ac400e05aff03e4614d1eaaa40252878c221c2", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "error_sha256": "97ae4156650fa0de2f613aa7753666781545cb2fe7818350a6df904ab696cdbc", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "error_sha256": "a2c6319267f1b752b63d96a41e15b1a5133952e88f0c145bc96dda53fd31cbe5", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "error_sha256": "ba0ac5e90ea008839723c328e8b172228820837ab895e425d2190c1aa5caa708", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "error_sha256": "8353154a58e84007d91c2d217c516dc771e359ddb43d527c7db6734ab5cff6ca", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "error_sha256": "085c13ab1e54fb324bce5520b12006fbf7f84b4a69a1c5cf9e09727b5b33da99", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "error_sha256": "0be25feaedaa697c09835d15c8cfdae694de2710330cadfadcbdfa146625a099", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_previous_verified_error": true + } + ], + "log_sha256": "95b796682d1a6c267aee6cb9a5da4f92c6218ebc902ccd0c53ce724b9f086ffb", + "all_failures_match_previous_verified_error": true, + "verified_commit": "fb90450d06fd3aaaee8a0a4b52aca13d2f675431", + "exit_code": 1 + } + }, + "changed_test_files": { + "files": 13, + "totals": { + "tests": 181, + "pass": 181, + "fail": 0, + "cancelled": 0, + "skipped": 0 + } + }, + "syntax_mjs_files": 22, + "biome_mjs_files": 22, + "diff_check": true, + "runtime_deployment_performed": false, + "limitations": [ + "same-session order and global FIFO remain local", + "Codex/Search execution and session coordination incomplete", + "executor fencing and safe orphan recovery not implemented", + "account/proxy/runtime full-snapshot conflicts remain", + "background tasks, shared files and global maintenance uncoordinated", + "no multi-node active replicas, HPA, rollout or failure takeover verification" + ] +} From 83af95ac7c5e88d72758b982680afe4f25b0f5bf Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:06:30 +0800 Subject: [PATCH 097/473] test: preserve peer proxy creations during stale replica probes --- test/unit/proxy-pool-shared.test.mjs | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 test/unit/proxy-pool-shared.test.mjs diff --git a/test/unit/proxy-pool-shared.test.mjs b/test/unit/proxy-pool-shared.test.mjs new file mode 100644 index 000000000..ea67efbdb --- /dev/null +++ b/test/unit/proxy-pool-shared.test.mjs @@ -0,0 +1,37 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' + +async function replicas(t) { + const db = getDb() + const other = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: db.schema }) + const first = await new ProxyPool({ db }).init() + const second = await new ProxyPool({ db: other }).init() + for (const pool of [first, second]) { + pool.stopScheduler() + pool.probeOne = async () => ({ ok: true, scope: 'socks', latency_ms: 1 }) + } + t.after(() => { + first.stopScheduler() + second.stopScheduler() + }) + return [first, second] +} + +test('a stale replica probe and flush preserve a proxy created by another connection', async (t) => { + const [first, second] = await replicas(t) + const existing = await first.importLines('192.0.2.1:1080') + await first.flush() + await second.reload() + const created = await first.importLines('192.0.2.2:1080') + await first.flush() + await second.probeAll() + await second.flush() + const ids = (await first.repo.loadAll()).map((proxy) => proxy.id) + assert.ok(ids.includes(existing.items[0].id)) + assert.ok( + ids.includes(created.items[0].id), + 'a probe must not replace the whole proxy table with its stale working set', + ) +}) From 3a0183caaee70fac90dcd1fcfd165889d3838d00 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:07:40 +0800 Subject: [PATCH 098/473] test: reproduce cross-replica account state overwrites --- test/unit/shared-account-quota.test.mjs | 124 ++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 test/unit/shared-account-quota.test.mjs diff --git a/test/unit/shared-account-quota.test.mjs b/test/unit/shared-account-quota.test.mjs new file mode 100644 index 000000000..b3ce2d79f --- /dev/null +++ b/test/unit/shared-account-quota.test.mjs @@ -0,0 +1,124 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' + +async function pair(t) { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + const left = await new AccountQuota({ db, config: {} }).init() + left.ensure({ account_id: 'shared-account', vm_id: 'vm-a' }) + await left.flush() + const right = await new AccountQuota({ db: other, config: {} }).init() + return { left, right } +} + +test('independent account caches persist additive exact usage without losing other replica increments', async (t) => { + const { left, right } = await pair(t) + left.ingestHeaders( + 'shared-account', + {}, + { + input_tokens: '9007199254740993', + output_tokens: 3, + cache_read_input_tokens: 4, + cache_creation_input_tokens: 5, + }, + ) + await left.flush() + right.ingestHeaders( + 'shared-account', + {}, + { + input_tokens: 7, + output_tokens: 11, + cache_read_input_tokens: 13, + cache_creation_input_tokens: 17, + }, + ) + right.ingestHeaders('shared-account', {}, { input_tokens: 100 }, { countRequest: false }) + await right.flush() + const saved = await left.persistence.get('shared-account') + assert.equal(saved.requests, 2) + assert.equal(saved.tokens_in, '9007199254741000') + assert.equal(saved.tokens_out, 14) + assert.equal(saved.cache_read_tokens, 17) + assert.equal(saved.cache_creation_tokens, 22) +}) + +test('a stale probe cache preserves another replica management fields and usage', async (t) => { + const { left, right } = await pair(t) + left.setMaxConcurrency('shared-account', 9, { override: true }) + left.setMaxRpm('shared-account', 123, { override: true }) + left.rebindToVm('shared-account', 'vm-b', { email: 'new@example.test' }) + left.ingestHeaders('shared-account', {}, { input_tokens: 7 }) + await left.flush() + right.ingestOAuthUsage('shared-account', { + ok: true, + limits_present: true, + five_hour: { utilization: 42 }, + }) + await right.flush() + const saved = await left.persistence.get('shared-account') + assert.equal(saved.max_concurrency, 9) + assert.equal(saved.concurrency_override, 1) + assert.equal(saved.max_rpm, 123) + assert.equal(saved.rpm_override, 1) + assert.equal(saved.vm_id, 'vm-b') + assert.equal(saved.email, 'new@example.test') + assert.equal(saved.requests, 1) + assert.equal(saved.tokens_in, 7) +}) + +test('concurrent ensure on independent connections inserts one account without replacing it', async (t) => { + const { left, right } = await pair(t) + let arrived = 0 + let ready + const gate = new Promise((resolve) => { + ready = resolve + }) + for (const quota of [left, right]) { + const get = quota.persistence.get.bind(quota.persistence) + quota.persistence.get = async (id) => { + const row = await get(id) + if (id === 'concurrent-account' && !row) { + if (++arrived === 2) ready() + await gate + } + return row + } + quota.ensure({ account_id: 'concurrent-account', vm_id: 'vm-a' }) + } + await Promise.all([left.flush(), right.flush()]) + assert.equal((await left.persistence.list()).filter((row) => row.account_id === 'concurrent-account').length, 1) +}) + +test('an older probe finishing after a newer observation cannot overwrite the newer usage window', async (t) => { + const { left, right } = await pair(t) + const oldObservation = await left.beginObservation() + const newObservation = await right.beginObservation() + right.ingestOAuthUsage( + 'shared-account', + { + ok: true, + limits_present: true, + five_hour: { utilization: 81 }, + }, + { observation: newObservation }, + ) + await right.flush() + left.ingestOAuthUsage( + 'shared-account', + { + ok: true, + limits_present: true, + five_hour: { utilization: 12 }, + }, + { observation: oldObservation }, + ) + await left.flush() + const saved = await left.persistence.get('shared-account') + assert.equal(saved.unified['5h'].utilization, 0.81) + assert.equal(left.repo.get('shared-account').unified['5h'].utilization, 0.81) +}) From 26b3076f0f7eb5e5ba422e4cb7ca525df0e528c6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:08:17 +0800 Subject: [PATCH 099/473] test(k8s): require proxy writes to settle before panel responses --- test/unit/proxy-panel-persistence.test.mjs | 67 ++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test/unit/proxy-panel-persistence.test.mjs diff --git a/test/unit/proxy-panel-persistence.test.mjs b/test/unit/proxy-panel-persistence.test.mjs new file mode 100644 index 000000000..333026802 --- /dev/null +++ b/test/unit/proxy-panel-persistence.test.mjs @@ -0,0 +1,67 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setImmediate } from 'node:timers/promises' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' +import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' +import { getDb } from '../../src/lib/db/database.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +for (const kind of ['update', 'disable']) { + test(`proxy ${kind} waits for persistence before responding`, async () => { + const pool = await new ProxyPool({ db: getDb() }).init() + const added = await pool.importLines('127.0.0.1:1080') + const id = added.items[0].id + await pool.flush() + const entered = deferred() + const persist = deferred() + const name = kind === 'update' ? 'update' : 'setEnabled' + const original = pool[name].bind(pool) + let operation + pool[name] = (...args) => { + operation = (async () => { + entered.resolve() + await persist.promise + return await original(...args) + })() + return operation + } + let response + const handler = createPanelHandler({ + cfg: { paths: { project: '/tmp/vm2api-648-proxy-panel-no-vms' } }, + proxyPool: pool, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + readBody: async () => ({ label: 'persisted label' }), + json(_res, status, body) { + response = { status, body } + return true + }, + }) + const pathname = `/api/panel/proxies/${id}${kind === 'disable' ? '/disable' : ''}` + const request = handler({ method: kind === 'update' ? 'PUT' : 'POST', headers: {} }, {}, { pathname }) + try { + await entered.promise + await setImmediate() + assert.equal(response, undefined, 'an unfinished database write cannot produce a success or false failure response') + } finally { + persist.resolve() + await operation + await request + await pool.flush() + } + assert.equal(response.status, 200) + const persisted = await getDb().prepare('SELECT enabled, label FROM proxies WHERE id = $1').get(id) + if (kind === 'disable') assert.equal(persisted.enabled, 0) + else assert.equal(persisted.label, 'persisted label') + }) +} From e8191dc74297929c61758f5ef9d7bd183304d941 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:09:05 +0800 Subject: [PATCH 100/473] test(k8s): reproduce cross-controller session turn overlap --- test/unit/shared-session-turns.test.mjs | 303 ++++++++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 test/unit/shared-session-turns.test.mjs diff --git a/test/unit/shared-session-turns.test.mjs b/test/unit/shared-session-turns.test.mjs new file mode 100644 index 000000000..af5d50bea --- /dev/null +++ b/test/unit/shared-session-turns.test.mjs @@ -0,0 +1,303 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +const success = () => ({ + ok: true, + status: 200, + terminalState: 'verified', + execution: { requestId: 'synthetic-hop', started: true, ended: true, reason: 'eof' }, + body: { type: 'message', role: 'assistant', content: [{ type: 'text', text: 'done' }], stop_reason: 'end_turn' }, +}) + +function args(requestId, callAttempt, extra = {}) { + return { + requestId, + stickyKey: 'shared-conversation', + model: 'claude-test', + canonicalBody: { model: 'claude-test', messages: [] }, + callAttempt, + ...extra, + } +} + +async function until(check) { + const deadline = Date.now() + 2000 + while (!(await check())) { + if (Date.now() >= deadline) assert.fail('condition did not become true') + await delay(5) + } +} + +async function fixture(t) { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(async () => { + await other.close() + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) + return { + db, + other, + rows: () => db.prepare('SELECT * FROM session_turns ORDER BY sequence').all(), + runner(connection, { release = async () => {}, timeout = 2000, sessionTurns } = {}) { + return new FailoverRunner({ + scheduler: { + accountQuota: { db: connection }, + async selectAndReserve() { + return { + ok: true, + accountId: 'synthetic-account', + vmId: 'synthetic-vm', + bindExecution: async () => {}, + markUncertain: async () => {}, + release, + } + }, + markSuccess() {}, + markCooldown() {}, + }, + sessionTurns, + config: { session_queue_timeout_ms: timeout, max_total_attempts: 1 }, + }) + }, + } +} + +test('independent runners serialize one session in database FIFO order and await reservation release', async (t) => { + const fx = await fixture(t) + const entered = deferred() + const finish = deferred() + const releaseEntered = deferred() + const released = deferred() + let releaseCalls = 0 + const first = fx.runner(fx.db, { + async release() { + if (++releaseCalls !== 1) return + releaseEntered.resolve() + await released.promise + }, + }) + const second = fx.runner(fx.other) + const events = [] + const holder = first.run( + args('first', async () => { + events.push('first') + entered.resolve() + await finish.promise + return success() + }), + ) + const pending = [holder] + try { + await entered.promise + const follower = second.run( + args('second', async () => { + events.push('second') + return success() + }), + ) + pending.push(follower) + await delay(30) + assert.deepEqual(events, ['first'], 'a separate runner must wait for the active conversation turn') + await until(async () => (await fx.rows()).some((row) => row.request_id === 'second')) + const third = first.run( + args('third', async () => { + events.push('third') + return success() + }), + ) + pending.push(third) + await until(async () => (await fx.rows()).some((row) => row.request_id === 'third')) + const queue = await fx.rows() + assert.deepEqual( + queue.map((row) => [row.request_id, row.state]), + [ + ['first', 'active'], + ['second', 'waiting'], + ['third', 'waiting'], + ], + ) + const unrelated = await second.run( + args( + 'unrelated', + async () => { + events.push('unrelated') + return success() + }, + { stickyKey: 'another-conversation' }, + ), + ) + assert.equal(unrelated.ok, true, 'a different conversation can execute while this one is held') + finish.resolve() + await releaseEntered.promise + assert.deepEqual(events, ['first', 'unrelated'], 'the next turn waits for the selected lease release') + released.resolve() + await Promise.all(pending) + assert.deepEqual(events, ['first', 'unrelated', 'second', 'third']) + assert.deepEqual(await fx.rows(), []) + } finally { + finish.resolve() + released.resolve() + await Promise.allSettled(pending) + } +}) + +test('waiting cancellation and timeout remove only their own records without overlapping the holder', async (t) => { + const fx = await fixture(t) + const entered = deferred() + const finish = deferred() + const first = fx.runner(fx.db) + const second = fx.runner(fx.other, { timeout: 50 }) + const holder = first.run( + args('holder', async () => { + entered.resolve() + await finish.promise + return success() + }), + ) + const controller = new AbortController() + let cancelled + try { + await entered.promise + cancelled = second.run( + args('cancelled', () => assert.fail('a cancelled waiter cannot execute'), { signal: controller.signal }), + ) + await until(async () => (await fx.rows()).some((row) => row.request_id === 'cancelled')) + controller.abort() + assert.equal((await cancelled).clientCancelled, true) + const timedOut = await second.run(args('timed-out', () => assert.fail('a timed out waiter cannot execute'))) + assert.equal(timedOut.status, 529) + assert.equal(timedOut.body.error.details.reason, 'session_queue_timeout') + assert.deepEqual( + (await fx.rows()).map((row) => [row.request_id, row.state]), + [['holder', 'active']], + ) + } finally { + controller.abort() + finish.resolve() + await Promise.allSettled([holder, cancelled]) + } + assert.deepEqual(await fx.rows(), []) +}) + +test('an unknown execution retains its active session turn after the request returns', async (t) => { + const fx = await fixture(t) + const first = fx.runner(fx.db) + const second = fx.runner(fx.other, { timeout: 40 }) + const result = await first.run( + args('unknown', async () => ({ + ok: false, + status: 0, + transportError: true, + execution: { requestId: 'unknown-hop', started: true, ended: false, reason: 'transport_error' }, + })), + ) + assert.equal(result.executionUncertain, true) + const rows = await fx.rows() + assert.equal(rows.length, 1) + assert.equal(rows[0].state, 'active') + assert.ok(rows[0].uncertain_at) + const blocked = await second.run(args('follower', () => assert.fail('the previous execution is not proven ended'))) + assert.equal(blocked.body.error.details.reason, 'session_queue_timeout') + assert.equal((await fx.rows())[0].request_id, 'unknown') +}) + +test('request preparation rejection safely releases the shared session turn', async (t) => { + const fx = await fixture(t) + const first = fx.runner(fx.db) + const rejected = await first.run( + args('invalid', () => assert.fail('preparation precedes execution'), { + applyAttempt() { + throw Object.assign(new Error('synthetic invalid cache control'), { + status: 400, + body: { error: { code: 'invalid_cache_control' } }, + }) + }, + }), + ) + assert.equal(rejected.status, 400) + assert.deepEqual(await fx.rows(), []) + assert.equal((await fx.runner(fx.other).run(args('next', async () => success()))).ok, true) +}) + +test('an exception during selected reservation release conservatively retains the shared session turn', async (t) => { + const fx = await fixture(t) + const failure = new Error('synthetic reservation release failure') + const first = fx.runner(fx.db, { + release: async () => { + throw failure + }, + }) + await assert.rejects(first.run(args('release-failed', async () => success())), (error) => error === failure) + const rows = await fx.rows() + assert.equal(rows[0].state, 'active') + assert.ok(rows[0].uncertain_at) + const blocked = await fx.runner(fx.other, { timeout: 40 }).run(args('next', () => assert.fail('release failed'))) + assert.equal(blocked.status, 529) +}) + +test('terminating the owner process leaves its persistent active turn blocking another runner', { + timeout: 5000, +}, async (t) => { + const fx = await fixture(t) + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { createDatabase } from './src/lib/db/database.mjs'; + import { FailoverRunner } from './src/lib/pool/failover-runner.mjs'; + const db = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: process.env.KIN_DATABASE_SCHEMA }); + const runner = new FailoverRunner({ scheduler: { + accountQuota: { db }, + selectAndReserve: async () => ({ ok: true, accountId: 'account', vmId: 'vm', release: async () => {} }), + }}); + setInterval(() => {}, 1000); + await runner.run({ requestId: 'killed-owner', stickyKey: 'shared-conversation', model: 'claude-test', canonicalBody: {}, + callAttempt: async () => { process.stdout.write('active\\n'); await new Promise(() => {}); } + }); + `, + ], + { cwd: process.cwd(), env: process.env, stdio: ['ignore', 'pipe', 'pipe'] }, + ) + t.after(() => { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + }) + let stderr = '' + child.stderr.on('data', (chunk) => { + stderr += chunk + }) + await Promise.race([ + once(child.stdout, 'data').then(([chunk]) => assert.match(String(chunk), /active/)), + once(child, 'exit').then(() => assert.fail(`owner exited before entering execution: ${stderr}`)), + ]) + const exited = once(child, 'exit') + child.kill('SIGKILL') + await exited + const rows = await fx.rows() + assert.equal(rows.length, 1) + assert.equal(rows[0].state, 'active') + assert.equal(rows[0].request_id, 'killed-owner') + const blocked = await fx + .runner(fx.other, { timeout: 40 }) + .run(args('next', () => assert.fail('owner loss is not termination proof'))) + assert.equal(blocked.status, 529) + assert.equal((await fx.rows())[0].id, rows[0].id) +}) From 10edf4595bf30e37fb31a44c5a20391d9cd139f6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:11:23 +0800 Subject: [PATCH 101/473] test(pool): reproduce unawaited proxy failure callback --- test/unit/failover-release.test.mjs | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/unit/failover-release.test.mjs b/test/unit/failover-release.test.mjs index 40d0ec0f3..1e18bd5e8 100644 --- a/test/unit/failover-release.test.mjs +++ b/test/unit/failover-release.test.mjs @@ -174,3 +174,47 @@ test('same-session requests wait until asynchronous capacity release completes', assert.equal(releases, 2) assert.equal(calls, 2) }) + +test('asynchronous proxy failure reporting finishes before releasing the selected reservation', async () => { + const entered = deferred() + const finish = deferred() + let releases = 0 + let selections = 0 + const scheduler = schedulerWithRelease(async () => releases++) + const runner = new FailoverRunner({ + scheduler: { + ...scheduler, + selectAndReserve: () => + ++selections === 1 ? scheduler.selectAndReserve() : { ok: false, reason: 'no_eligible_accounts' }, + markCooldown() {}, + }, + config: { max_total_attempts: 1 }, + onProxyFailure: async () => { + entered.resolve() + await finish.promise + }, + }) + const pending = runner.run( + request('async-proxy-release', { + callAttempt: () => ({ + ok: false, + status: 0, + transportError: true, + terminalState: 'transport_error', + body: { error: { code: 'worker_transport_error', message: 'SOCKS proxy dial failed' } }, + }), + }), + ) + try { + await Promise.race([ + entered.promise, + pending.then((result) => assert.fail(`proxy callback was not invoked: ${JSON.stringify(result.policy)}`)), + ]) + await setImmediate() + assert.equal(releases, 0, 'asynchronous proxy state writes are part of the attempt lifecycle') + } finally { + finish.resolve() + await pending + } + assert.equal(releases, 1) +}) From a7deeb81a5af9ff7ffc7280dca9c0e88a7717fa5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:11:58 +0800 Subject: [PATCH 102/473] refactor(proxy): await durable mutations in panel and VM operations --- src/lib/admin/panel-routes.mjs | 34 +++++++++++----------- src/lib/admin/routing-runtime.mjs | 4 +-- src/lib/vm/proxy-timezone.mjs | 1 + src/lib/vm/slot-recovery.mjs | 2 +- src/lib/vm/vm-package.mjs | 15 +++++----- src/server.mjs | 11 +++---- test/unit/proxy-panel-persistence.test.mjs | 6 +++- 7 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 501f60bfe..c16a94514 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -3357,7 +3357,7 @@ export function createPanelHandler(ctx) { } catch {} // detach this VM only — do not unbind other slots sharing the SOCKS5 try { - proxyPool.unbindVm(id) + await proxyPool.unbindVm(id) } catch {} // remove cli-home try { @@ -3408,7 +3408,7 @@ export function createPanelHandler(ctx) { } catch {} if (!hasBoundExit(vm.proxy)) { try { - const allocated = proxyPool.allocateForVm(id) + const allocated = await proxyPool.allocateForVm(id) if (allocated) { bindVmProxy(cfg.paths.project, id, proxyPool.getProxyForVm(id)) adoptVmProxyBinding(vm, getVm(cfg.paths.project, id)) @@ -4046,7 +4046,7 @@ export function createPanelHandler(ctx) { let pickedProxyId = typeof body.proxy_id === 'string' ? body.proxy_id.trim() : '' if (pickedProxyId && isLocalEgressId(pickedProxyId)) { // Local egress is per VPS: a generic pick resolves to the target machine's own row. - const resolved = proxyPool.resolveLocalTarget(pickedProxyId, nodeId || null) + const resolved = await proxyPool.resolveLocalTarget(pickedProxyId, nodeId || null) if (!resolved.ok) { return json(res, 409, { ok: false, @@ -4228,12 +4228,12 @@ export function createPanelHandler(ctx) { const hasExit = (v) => !!(v?.proxy?.url || isLocalEgressProxy(v?.proxy)) if (wantProxy && !hasExit(vm)) { if (pickedProxyId) { - const bound = proxyPool.bind(pickedProxyId, id) + const bound = await proxyPool.bind(pickedProxyId, id) if (bound.ok) allocated = bound.proxy else proxyError = bound.error || 'proxy_bind_failed' } else { try { - allocated = proxyPool.allocateForVm(id, { + allocated = await proxyPool.allocateForVm(id, { ownerUserId: vm.owner_user_id || null, role: ident.role, }) @@ -5515,10 +5515,10 @@ export function createPanelHandler(ctx) { return json(res, 200, panel.ok(snap)) } if (req.method === 'POST' && p === '/api/panel/proxies/local') { - const result = proxyPool.ensureLocal() + const result = await proxyPool.ensureLocal() if (result.created) await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(result.proxy.id)) // Enabling local egress on a cluster that already has nodes: each VPS gets its row now. - proxyPool.syncLocalNodes(clusterNodesOf(ctx).map((node) => node.id)) + await proxyPool.syncLocalNodes(clusterNodesOf(ctx).map((node) => node.id)) return json(res, 200, panel.ok(result)) } if (req.method === 'POST' && p === '/api/panel/proxies/import') { @@ -5529,7 +5529,7 @@ export function createPanelHandler(ctx) { if (bindVmId && userMissesVmId(req, cfg.paths.project, bindVmId)) { return json(res, 404, { ok: false, error: { message: 'vm not found' } }) } - const result = proxyPool.importLines(text, { + const result = await proxyPool.importLines(text, { fields: body.proxies || body.entries || null, host: body.host, port: body.port, @@ -5541,7 +5541,7 @@ export function createPanelHandler(ctx) { let worker = null let timezone = null if (bindVmId && result.items?.[0]?.id) { - const bind = proxyPool.bind(result.items[0].id, bindVmId) + const bind = await proxyPool.bind(result.items[0].id, bindVmId) if (bind.ok) { bindVmProxy(cfg.paths.project, bindVmId, proxyPool.getProxyForVm(bindVmId)) bound = bind.proxy @@ -5581,7 +5581,7 @@ export function createPanelHandler(ctx) { if (req.method === 'PUT' && p === '/api/panel/proxies/config') { const body = await readBody(req, 64 * 1024) const previousConfig = proxyPool.snapshot().config - const result = proxyPool.updateConfig(body) + const result = await proxyPool.updateConfig(body) if (!result.ok) return json(res, 400, { ok: false, @@ -5624,7 +5624,7 @@ export function createPanelHandler(ctx) { for (const key of ['host', 'port', 'username', 'password', 'label', 'domain_forward']) { if (Object.prototype.hasOwnProperty.call(body, key)) patch[key] = body[key] } - const result = proxyPool.update(id, patch) + const result = await proxyPool.update(id, patch) if (!result.ok) { const status = result.error === 'proxy_not_found' ? 404 : 400 const type = status === 404 ? 'not_found_error' : 'invalid_request_error' @@ -5749,7 +5749,7 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/proxies\/[^/]+\/enable$/.test(p)) { const id = p.split('/')[4] - const result = proxyPool.setEnabled(id, true) + const result = await proxyPool.setEnabled(id, true) if (!result.ok) return json(res, 404, { ok: false, @@ -5768,7 +5768,7 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/proxies\/[^/]+\/disable$/.test(p)) { const id = p.split('/')[4] - const result = proxyPool.setEnabled(id, false) + const result = await proxyPool.setEnabled(id, false) if (!result.ok) return json(res, 404, { ok: false, @@ -5785,7 +5785,7 @@ export function createPanelHandler(ctx) { ok: false, error: { type: 'invalid_request_error', code: 'missing_field', message: 'vm_id required', param: 'vm_id' }, }) - const result = proxyPool.bind(id, vmId) + const result = await proxyPool.bind(id, vmId) if (!result.ok) { const message = result.error === 'proxy_bind_limit' @@ -5830,7 +5830,7 @@ export function createPanelHandler(ctx) { : snap?.bound_vm_id ? [snap.bound_vm_id] : [] - const result = proxyPool.unbind(id, vmId || null) + const result = await proxyPool.unbind(id, vmId || null) if (!result.ok) return json(res, 404, { ok: false, @@ -5849,7 +5849,7 @@ export function createPanelHandler(ctx) { const id = p.split('/').pop() const snap = proxyPool.snapshot().proxies.find((proxy) => proxy.id === id) const targets = snap?.bound_vm_ids?.length ? snap.bound_vm_ids : snap?.bound_vm_id ? [snap.bound_vm_id] : [] - const result = proxyPool.remove(id) + const result = await proxyPool.remove(id) if (!result.ok) return json(res, 404, { ok: false, @@ -5865,7 +5865,7 @@ export function createPanelHandler(ctx) { // Auto-allocate proxy for a VM if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/allocate-proxy$/.test(p)) { const vmId = p.split('/')[4] - const allocated = proxyPool.allocateForVm(vmId) + const allocated = await proxyPool.allocateForVm(vmId) if (!allocated) return json(res, 409, { ok: false, diff --git a/src/lib/admin/routing-runtime.mjs b/src/lib/admin/routing-runtime.mjs index 88f1cd5a5..f5422ad1a 100644 --- a/src/lib/admin/routing-runtime.mjs +++ b/src/lib/admin/routing-runtime.mjs @@ -723,8 +723,8 @@ export function createRoutingRuntime(ctx) { attemptsRepo, rateLimitService, config: routingConfig.failover || {}, - onProxyFailure: (vmId, reason) => { - ctx.proxyPool.reportRuntimeFailure(vmId, reason) + onProxyFailure: async (vmId, reason) => { + await ctx.proxyPool.reportRuntimeFailure(vmId, reason) }, onFablePlanDenied: ({ selected }) => { const accountId = selected?.accountId diff --git a/src/lib/vm/proxy-timezone.mjs b/src/lib/vm/proxy-timezone.mjs index 5b771546b..6e7db35d1 100644 --- a/src/lib/vm/proxy-timezone.mjs +++ b/src/lib/vm/proxy-timezone.mjs @@ -25,6 +25,7 @@ export async function syncVmTimezoneFromProxy( ) { const id = String(vmId || '').trim() if (!id || !proxyPool) return skip('vm_id_required') + await proxyPool.refresh?.() if (!force && !proxyPool.followProxyTimezoneEnabled()) return skip('follow_proxy_timezone_disabled') const vm = getVm(projectRoot, id) if (!vm) return skip('vm_not_found') diff --git a/src/lib/vm/slot-recovery.mjs b/src/lib/vm/slot-recovery.mjs index e9bad0433..096d1e288 100644 --- a/src/lib/vm/slot-recovery.mjs +++ b/src/lib/vm/slot-recovery.mjs @@ -370,7 +370,7 @@ export async function powerOnSlot({ projectRoot, id, proxyPool, routing = {}, fi } catch (error) { return { ok: false, status: 400, code: error.code, error: error.message } } - const bound = proxyPool.ensureBoundToVm(id, vm.proxy?.id || null) + const bound = await proxyPool.ensureBoundToVm(id, vm.proxy?.id || null) if (!bound) { return { ok: false, status: 409, code: 'proxy_required', error: 'No SOCKS5 is bound or free for this VM' } } diff --git a/src/lib/vm/vm-package.mjs b/src/lib/vm/vm-package.mjs index cf1a7523e..f88c57729 100644 --- a/src/lib/vm/vm-package.mjs +++ b/src/lib/vm/vm-package.mjs @@ -515,8 +515,9 @@ function mirrorCodex(vm, account) { * SOCKS rows, never px-local or a platform/other-tenant row with equal credentials. * Runs before any file is written so a refused package leaves nothing behind. */ -function proxyBindAllowed(spec, vm, role, proxyPool) { +async function proxyBindAllowed(spec, vm, role, proxyPool) { if (!spec.proxy) return true + await proxyPool.refresh?.() let owner = null if (spec.proxy.kind === 'local') { if (role === 'user') return false @@ -529,23 +530,23 @@ function proxyBindAllowed(spec, vm, role, proxyPool) { async function attachProxy(projectRoot, id, spec, proxyPool, ownerUserId) { if (!spec.proxy) { - proxyPool.unbindVm?.(id) + await proxyPool.unbindVm?.(id) bindVmProxy(projectRoot, id, null) return { ok: true } } if (spec.proxy.kind === 'local') { - const ensured = proxyPool.ensureLocal() + const ensured = await proxyPool.ensureLocal() const proxyId = ensured?.proxy?.id if (!proxyId) return fail('proxy_import_failed', '本机直连出口不可用') - const bound = proxyPool.bind(proxyId, id) + const bound = await proxyPool.bind(proxyId, id) if (!bound.ok) return fail(bound.error || 'proxy_bind_failed', '绑定直连出口失败') // bind() lands a node slot on its own VPS's local row, not the control plane's. bindVmProxy(projectRoot, id, proxyPool.getProxyByIdWithAuth(bound.proxy?.id || proxyId)) return { ok: true } } - const ensured = proxyPool.ensureSocks(spec.proxy, { ownerUserId, label: spec.proxy.label }) + const ensured = await proxyPool.ensureSocks(spec.proxy, { ownerUserId, label: spec.proxy.label }) if (!ensured.ok) return fail(ensured.error || 'invalid_proxy', 'SOCKS5 无法写入') - const bound = proxyPool.bind(ensured.proxy.id, id) + const bound = await proxyPool.bind(ensured.proxy.id, id) if (!bound.ok) { const message = bound.error === 'proxy_bind_limit' ? 'SOCKS5 绑定数量已满' : '绑定 SOCKS5 失败' return fail(bound.error || 'proxy_bind_failed', message) @@ -669,7 +670,7 @@ export async function commitVmPackage({ } status = vm.status || 'stopped' } - if (!proxyBindAllowed(spec, vm, owner.role, proxyPool)) { + if (!(await proxyBindAllowed(spec, vm, owner.role, proxyPool))) { return { ok: false, status: 403, error: { code: 'proxy_forbidden', message: '无权绑定这个出口' } } } applyTraits(vm, spec) diff --git a/src/server.mjs b/src/server.mjs index aa2d06255..d4f2c0c4c 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -437,7 +437,7 @@ proxyPool.startScheduler() // 首次安装的出口池是空的，槽没有可绑出口就起不来。本机出口永远成立，先补上。 try { if (!proxyPool.snapshot().proxies?.length) { - const seeded = proxyPool.ensureLocal() + const seeded = await proxyPool.ensureLocal() if (seeded.created) { void (await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(seeded.proxy.id))) console.log('[bootstrap] seeded local egress px-local') @@ -817,6 +817,7 @@ async function requireAuth(req, res) { refreshModelPolicy(snapshot.model_policy) refreshGptModelPolicy(snapshot.gpt_model_policy) invalidatePriceCatalog() + await proxyPool.refresh?.() } // Panel session const session = await verifyPanelSession(token) @@ -1006,10 +1007,10 @@ const responsesWs = createResponsesWsHandler({ * another machine's row (the shared px-local before per-node rows) move onto * their own node's row, with the slot's recorded proxy rewritten to match. */ -function syncNodeLocalEgress() { +async function syncNodeLocalEgress() { try { - proxyPool.syncLocalNodes(clusterManager.list().map((node) => node.id)) - for (const move of proxyPool.adoptNodeLocalBindings()) { + await proxyPool.syncLocalNodes(clusterManager.list().map((node) => node.id)) + for (const move of await proxyPool.adoptNodeLocalBindings()) { bindVmProxy(cfg.paths.project, move.vm_id, proxyPool.getProxyByIdWithAuth(move.to)) console.log(`[local-egress] ${move.vm_id}: ${move.from} -> ${move.to}`) } @@ -1050,7 +1051,7 @@ bindPlacement({ manager: clusterManager, projectRoot: cfg.paths.project }) await startNodeEgressSocks({ clientFor: (nodeId) => clusterManager.client(nodeId) }) await clusterManager.start() clusterManager.restoreSlotRelays() -syncNodeLocalEgress() +await syncNodeLocalEgress() if (proxyPool.snapshot().config.ipv6_enabled !== true) { const exits = await syncIpv6ProxyEgress(cfg.paths.project, proxyPool) for (const exit of exits.filter((item) => !item.ok)) { diff --git a/test/unit/proxy-panel-persistence.test.mjs b/test/unit/proxy-panel-persistence.test.mjs index 333026802..ee53eb5db 100644 --- a/test/unit/proxy-panel-persistence.test.mjs +++ b/test/unit/proxy-panel-persistence.test.mjs @@ -52,7 +52,11 @@ for (const kind of ['update', 'disable']) { try { await entered.promise await setImmediate() - assert.equal(response, undefined, 'an unfinished database write cannot produce a success or false failure response') + assert.equal( + response, + undefined, + 'an unfinished database write cannot produce a success or false failure response', + ) } finally { persist.resolve() await operation From 27c51620722ceeec9700f84f91c962c06f0226d4 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:13:18 +0800 Subject: [PATCH 103/473] test(pool): retain uncertainty across special attempt errors --- test/unit/shared-session-turns.test.mjs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/unit/shared-session-turns.test.mjs b/test/unit/shared-session-turns.test.mjs index af5d50bea..41cb707c7 100644 --- a/test/unit/shared-session-turns.test.mjs +++ b/test/unit/shared-session-turns.test.mjs @@ -236,6 +236,28 @@ test('request preparation rejection safely releases the shared session turn', as assert.equal((await fx.runner(fx.other).run(args('next', async () => success()))).ok, true) }) +for (const failure of [ + Object.assign(new Error('synthetic response already sent'), { code: 'response_sent' }), + Object.assign(new Error('synthetic rejection after entering execution'), { + status: 400, + body: { error: { code: 'invalid_cache_control' } }, + }), +]) { + test(`a ${failure.code || failure.body.error.code} exception after entering execution keeps the shared turn`, async (t) => { + const fx = await fixture(t) + const result = await fx.runner(fx.db).run( + args('unknown-rejection', () => { + throw failure + }), + ) + assert.equal(result.executionUncertain, true) + const rows = await fx.rows() + assert.equal(rows.length, 1) + assert.equal(rows[0].state, 'active') + assert.ok(rows[0].uncertain_at) + }) +} + test('an exception during selected reservation release conservatively retains the shared session turn', async (t) => { const fx = await fixture(t) const failure = new Error('synthetic reservation release failure') From 3dbb627ce3d554e2ecadb1b5020403eacba20927 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:15:17 +0800 Subject: [PATCH 104/473] feat(k8s): persist FIFO session turns across control planes --- .../postgres-migrations/010_session_turns.sql | 16 +++ src/lib/pool/failover-runner.mjs | 52 ++++++- src/lib/pool/shared-session-turns.mjs | 135 ++++++++++++++++++ test/unit/failover-release.test.mjs | 90 ++++++------ test/unit/shared-session-turns.test.mjs | 23 ++- 5 files changed, 268 insertions(+), 48 deletions(-) create mode 100644 src/lib/db/postgres-migrations/010_session_turns.sql create mode 100644 src/lib/pool/shared-session-turns.mjs diff --git a/src/lib/db/postgres-migrations/010_session_turns.sql b/src/lib/db/postgres-migrations/010_session_turns.sql new file mode 100644 index 000000000..5ade41fe2 --- /dev/null +++ b/src/lib/db/postgres-migrations/010_session_turns.sql @@ -0,0 +1,16 @@ +CREATE TABLE session_turns ( + sequence bigserial PRIMARY KEY, + id text COLLATE "C" NOT NULL UNIQUE, + session_key text COLLATE "C" NOT NULL, + owner_id text COLLATE "C" NOT NULL, + request_id text COLLATE "C", + state text COLLATE "C" NOT NULL CHECK (state IN ('waiting', 'active')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + activated_at timestamptz, + uncertain_at timestamptz +); + +CREATE INDEX idx_session_turns_queue ON session_turns (session_key, sequence); +CREATE UNIQUE INDEX idx_session_turns_active ON session_turns (session_key) WHERE state = 'active'; + +-- Active turns have no expiry: losing an owner connection does not end its execution. diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index f48731e60..713185734 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -21,6 +21,8 @@ import { ensureWorkerCredential } from '../transport/go-worker-client.mjs' import { resolveOfficialCcInference } from '../vm/slot-engine.mjs' import { AttemptCoordinator } from './unit-decision.mjs' import { SOFT_COOLDOWN_REASONS } from './pool-scheduler.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { SharedSessionTurns } from './shared-session-turns.mjs' // Seat wait for the one-shot repair hop when the rejected hop already spent the deadline. const REPAIR_HOP_WAIT_MS = 30_000 @@ -235,11 +237,11 @@ function classifyAttempt( }) } -function notifyProxyFailure(onProxyFailure, selected, policy) { +async function notifyProxyFailure(onProxyFailure, selected, policy) { if (policy?.reason !== 'proxy_transport_failure') return if (typeof onProxyFailure !== 'function') return try { - onProxyFailure(selected?.vmId, policy.reason) + await onProxyFailure(selected?.vmId, policy.reason) } catch { /* control-plane disconnect is best-effort */ } @@ -444,6 +446,7 @@ export class FailoverRunner { stickyRouter = null, attemptsRepo = null, rateLimitService = null, + sessionTurns = null, config = {}, onProxyFailure = null, onCredentialFailure = null, @@ -460,6 +463,8 @@ export class FailoverRunner { this.onFablePlanDenied = onFablePlanDenied this.onFableSuccess = onFableSuccess this.sessionTails = new Map() + this.sessionTurns = sessionTurns + this.sharedSessions = isKubernetesMode() this.searchRepairs = new Map() } @@ -538,6 +543,7 @@ export class FailoverRunner { async run(args = {}) { const sessionKey = String(args.stickyKey || '') if (!sessionKey) return await this.runOnce(args) + if (this.sharedSessions || this.sessionTurns) return this.runSharedSession(sessionKey, args) const previous = this.sessionTails.get(sessionKey) || Promise.resolve() let releaseTurn const turn = new Promise((resolve) => { @@ -570,6 +576,42 @@ export class FailoverRunner { } } + async runSharedSession(sessionKey, args) { + this.sessionTurns ||= new SharedSessionTurns( + this.scheduler?.executionAdmissions?.db || this.scheduler?.accountQuota?.db, + ) + let turn + try { + turn = await this.sessionTurns.acquire(sessionKey, { + requestId: args.requestId, + signal: args.signal, + timeoutMs: this.config.session_queue_timeout_ms, + }) + } catch (error) { + if (error?.code === 'request_cancelled') return clientCancelledResult() + if (error?.code === 'session_queue_timeout') { + const retryMs = Number(this.config.session_queue_timeout_ms) + return poolOverloadedError({ + reason: 'session_queue_timeout', + retry_after_ms: Number.isFinite(retryMs) && retryMs > 0 ? retryMs : 1000, + }) + } + throw error + } + try { + const result = await this.runOnce(args) + // runOnce awaits selected.release in its finally before resolving. + if (result?.executionUncertain) await turn.markUncertain() + else await turn.release() + return result + } catch (error) { + // A thrown attempt or release can leave an execution active. Connection + // loss and HTTP cancellation do not provide proof that it ended. + await turn.markUncertain() + throw error + } + } + async runOnce({ requestId, canonicalBody, @@ -912,7 +954,7 @@ export class FailoverRunner { lastResult = result lastPolicy = policy this.noteUnitHealth(selected, policy, result) - notifyProxyFailure(this.onProxyFailure, selected, policy) + await notifyProxyFailure(this.onProxyFailure, selected, policy) if (policy.reason === 'fable_plan_denied' && typeof this.onFablePlanDenied === 'function') { try { this.onFablePlanDenied({ selected, policy }) @@ -1018,6 +1060,7 @@ export class FailoverRunner { }) return { ok: false, + ...(unknownExecution ? { executionUncertain: true } : {}), responseSent: true, status: 403, via: 'account-scope', @@ -1038,6 +1081,7 @@ export class FailoverRunner { }) return { ok: false, + ...(unknownExecution ? { executionUncertain: true } : {}), status: 400, body: error.body, terminalState: 'invalid_request', @@ -1091,7 +1135,7 @@ export class FailoverRunner { lastResult = result lastPolicy = policy this.noteUnitHealth(selected, policy, result) - notifyProxyFailure(this.onProxyFailure, selected, policy) + await notifyProxyFailure(this.onProxyFailure, selected, policy) await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, errorScope: policy.scope, diff --git a/src/lib/pool/shared-session-turns.mjs b/src/lib/pool/shared-session-turns.mjs new file mode 100644 index 000000000..cf78196eb --- /dev/null +++ b/src/lib/pool/shared-session-turns.mjs @@ -0,0 +1,135 @@ +import { randomUUID } from 'node:crypto' +import { getDb, withTransaction } from '../db/database.mjs' + +function cancelled() { + return Object.assign(new Error('Request was cancelled'), { code: 'request_cancelled' }) +} + +function timedOut() { + return Object.assign(new Error('Session queue timed out'), { code: 'session_queue_timeout' }) +} + +function wait(ms, signal) { + if (signal?.aborted) return Promise.reject(cancelled()) + return new Promise((resolve, reject) => { + const finish = (error) => { + clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) + if (error) reject(error) + else resolve() + } + const onAbort = () => finish(cancelled()) + const timer = setTimeout(() => finish(), ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + +/** + * Persistent per-session FIFO. Transaction locks protect state changes, never execution lifetimes. + * Owner loss does not delete active or waiting rows; orphan recovery requires separate execution evidence. + */ +export class SharedSessionTurns { + constructor(db = getDb(), { ownerId = 'controller_' + randomUUID() } = {}) { + this.db = db + this.ownerId = ownerId + } + + async lock(sessionKey) { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ + `vm2api:session-turn:${this.db.schema}`, + sessionKey, + ]) + } + + async enqueue(sessionKey, requestId) { + const id = randomUUID() + return withTransaction(this.db, async () => { + // Allocate sequence under the same lock as activation, so an uncommitted + // earlier insert cannot be invisible when a later waiter claims the head. + await this.lock(sessionKey) + await this.db + .prepare(`INSERT INTO session_turns (id, session_key, owner_id, request_id, state) + VALUES ($1,$2,$3,$4,'waiting')`) + .run(id, sessionKey, this.ownerId, requestId ?? null) + return { id, sessionKey, ownerId: this.ownerId } + }) + } + + async tryActivate(handle) { + return withTransaction(this.db, async () => { + await this.lock(handle.sessionKey) + const head = await this.db + .prepare('SELECT * FROM session_turns WHERE session_key = $1 ORDER BY sequence LIMIT 1 FOR UPDATE') + .get(handle.sessionKey) + if (head?.id !== handle.id || head.owner_id !== handle.ownerId || head.state !== 'waiting') return false + await this.db + .prepare("UPDATE session_turns SET state = 'active', activated_at = clock_timestamp() WHERE id = $1") + .run(handle.id) + return true + }) + } + + async removeWaiting(handle) { + await this.db + .prepare("DELETE FROM session_turns WHERE id = $1 AND owner_id = $2 AND state = 'waiting'") + .run(handle.id, handle.ownerId) + } + + async release(handle) { + return withTransaction(this.db, async () => { + await this.lock(handle.sessionKey) + const result = await this.db + .prepare( + "DELETE FROM session_turns WHERE id = $1 AND owner_id = $2 AND state = 'active' AND uncertain_at IS NULL", + ) + .run(handle.id, handle.ownerId) + return { released: result.changes > 0 } + }) + } + + async markUncertain(handle) { + return withTransaction(this.db, async () => { + await this.lock(handle.sessionKey) + const result = await this.db + .prepare(`UPDATE session_turns SET uncertain_at = COALESCE(uncertain_at, clock_timestamp()) + WHERE id = $1 AND owner_id = $2 AND state = 'active'`) + .run(handle.id, handle.ownerId) + return { uncertain: result.changes > 0 } + }) + } + + async acquire(sessionKey, { requestId, signal, timeoutMs } = {}) { + if (typeof sessionKey !== 'string' || !sessionKey.trim()) { + throw Object.assign(new Error('Session key is required'), { code: 'invalid_session_turn' }) + } + if (signal?.aborted) throw cancelled() + const timeout = Number(timeoutMs) + const deadline = Number.isFinite(timeout) && timeout > 0 ? Date.now() + timeout : Infinity + const handle = await this.enqueue(sessionKey, requestId) + try { + for (;;) { + if (signal?.aborted) throw cancelled() + if (Date.now() >= deadline) throw timedOut() + if (await this.tryActivate(handle)) { + // Cancellation while activation committed is still proven not sent: + // the caller has not received this turn or entered runOnce yet. + if (signal?.aborted || Date.now() >= deadline) { + await this.release(handle) + if (signal?.aborted) throw cancelled() + throw timedOut() + } + return { + turnId: handle.id, + release: () => this.release(handle), + markUncertain: () => this.markUncertain(handle), + } + } + await wait(Math.min(250, Math.max(1, deadline - Date.now())), signal) + } + } catch (error) { + // Never delete an active turn if a transaction's outcome is unknown. + await this.removeWaiting(handle) + throw error + } + } +} diff --git a/test/unit/failover-release.test.mjs b/test/unit/failover-release.test.mjs index 1e18bd5e8..b7c6efe32 100644 --- a/test/unit/failover-release.test.mjs +++ b/test/unit/failover-release.test.mjs @@ -175,46 +175,52 @@ test('same-session requests wait until asynchronous capacity release completes', assert.equal(calls, 2) }) -test('asynchronous proxy failure reporting finishes before releasing the selected reservation', async () => { - const entered = deferred() - const finish = deferred() - let releases = 0 - let selections = 0 - const scheduler = schedulerWithRelease(async () => releases++) - const runner = new FailoverRunner({ - scheduler: { - ...scheduler, - selectAndReserve: () => - ++selections === 1 ? scheduler.selectAndReserve() : { ok: false, reason: 'no_eligible_accounts' }, - markCooldown() {}, - }, - config: { max_total_attempts: 1 }, - onProxyFailure: async () => { - entered.resolve() - await finish.promise - }, - }) - const pending = runner.run( - request('async-proxy-release', { - callAttempt: () => ({ - ok: false, - status: 0, - transportError: true, - terminalState: 'transport_error', - body: { error: { code: 'worker_transport_error', message: 'SOCKS proxy dial failed' } }, +for (const mode of ['result', 'exception']) { + test(`asynchronous proxy failure reporting finishes before release after a transport ${mode}`, async () => { + const entered = deferred() + const finish = deferred() + let releases = 0 + let selections = 0 + const scheduler = schedulerWithRelease(async () => releases++) + const runner = new FailoverRunner({ + scheduler: { + ...scheduler, + selectAndReserve: () => + ++selections === 1 ? scheduler.selectAndReserve() : { ok: false, reason: 'no_eligible_accounts' }, + markCooldown() {}, + }, + config: { max_total_attempts: 1 }, + onProxyFailure: async () => { + entered.resolve() + await finish.promise + throw new Error('synthetic proxy report failed') + }, + }) + const pending = runner.run( + request('async-proxy-release', { + callAttempt: () => { + if (mode === 'exception') throw new Error('SOCKS proxy dial failed') + return { + ok: false, + status: 0, + transportError: true, + terminalState: 'transport_error', + body: { error: { code: 'worker_transport_error', message: 'SOCKS proxy dial failed' } }, + } + }, }), - }), - ) - try { - await Promise.race([ - entered.promise, - pending.then((result) => assert.fail(`proxy callback was not invoked: ${JSON.stringify(result.policy)}`)), - ]) - await setImmediate() - assert.equal(releases, 0, 'asynchronous proxy state writes are part of the attempt lifecycle') - } finally { - finish.resolve() - await pending - } - assert.equal(releases, 1) -}) + ) + try { + await Promise.race([ + entered.promise, + pending.then((result) => assert.fail(`proxy callback was not invoked: ${JSON.stringify(result.policy)}`)), + ]) + await setImmediate() + assert.equal(releases, 0, 'asynchronous proxy state writes are part of the attempt lifecycle') + } finally { + finish.resolve() + await pending + } + assert.equal(releases, 1) + }) +} diff --git a/test/unit/shared-session-turns.test.mjs b/test/unit/shared-session-turns.test.mjs index 41cb707c7..10934cf2d 100644 --- a/test/unit/shared-session-turns.test.mjs +++ b/test/unit/shared-session-turns.test.mjs @@ -79,6 +79,23 @@ async function fixture(t) { } } +test('a runner can use an explicitly injected shared turn service', async (t) => { + const fx = await fixture(t) + const { SharedSessionTurns } = await import('../../src/lib/pool/shared-session-turns.mjs') + const service = new SharedSessionTurns(fx.other, { ownerId: 'injected-controller' }) + const runner = fx.runner(fx.db, { sessionTurns: service }) + const result = await runner.run( + args('injected', async () => { + const rows = await fx.rows() + assert.equal(rows[0].owner_id, 'injected-controller') + assert.equal(rows[0].state, 'active') + return success() + }), + ) + assert.equal(result.ok, true) + assert.deepEqual(await fx.rows(), []) +}) + test('independent runners serialize one session in database FIFO order and await reservation release', async (t) => { const fx = await fixture(t) const entered = deferred() @@ -163,7 +180,7 @@ test('waiting cancellation and timeout remove only their own records without ove const entered = deferred() const finish = deferred() const first = fx.runner(fx.db) - const second = fx.runner(fx.other, { timeout: 50 }) + const second = fx.runner(fx.other) const holder = first.run( args('holder', async () => { entered.resolve() @@ -181,7 +198,9 @@ test('waiting cancellation and timeout remove only their own records without ove await until(async () => (await fx.rows()).some((row) => row.request_id === 'cancelled')) controller.abort() assert.equal((await cancelled).clientCancelled, true) - const timedOut = await second.run(args('timed-out', () => assert.fail('a timed out waiter cannot execute'))) + const timedOut = await fx + .runner(fx.other, { timeout: 100 }) + .run(args('timed-out', () => assert.fail('a timed out waiter cannot execute'))) assert.equal(timedOut.status, 529) assert.equal(timedOut.body.error.details.reason, 'session_queue_timeout') assert.deepEqual( From 782608a962016d187f5cea6972a2088499655c15 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:17:23 +0800 Subject: [PATCH 105/473] test(k8s): cover observation and session schema backup recovery --- test/unit/backup-service.test.mjs | 15 ++++++++++++--- test/unit/database.test.mjs | 7 ++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 920a09711..9bf4b55eb 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,7 +79,11 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns; + ALTER TABLE accounts DROP COLUMN observation_versions; + ALTER TABLE proxies DROP COLUMN config_revision; + DROP SEQUENCE IF EXISTS account_observation_seq; + DELETE FROM schema_migrations WHERE version <> '001'`, ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() @@ -97,16 +101,21 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'execution_admissions', 'execution_seats', 'execution_jobs', + 'session_turns', ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 7) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 10) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - "DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats; DELETE FROM schema_migrations WHERE version <> '001'", + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns; + ALTER TABLE accounts DROP COLUMN observation_versions; + ALTER TABLE proxies DROP COLUMN config_revision; + DROP SEQUENCE IF EXISTS account_observation_seq; + DELETE FROM schema_migrations WHERE version <> '001'`, ) await new SettingsRepo(db).set('older-snapshot', 'before') const rec = await svc.createBackup() diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index 6f91fa5c6..c758d9bdb 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 36) + assert.equal(rows.length, 37) for (const name of [ 'settings', 'users', @@ -51,6 +51,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'execution_admissions', 'execution_seats', 'execution_jobs', + 'session_turns', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -111,7 +112,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 7) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 10) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -127,7 +128,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 7) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 10) fs.rmSync(dir, { recursive: true, force: true }) }) From ac4b58f985f1635d241b554e6c1157335708ec65 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:19:18 +0800 Subject: [PATCH 106/473] test(k8s): retain attempt observation order for 429 headers --- test/unit/rate-limit-service.test.mjs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/rate-limit-service.test.mjs b/test/unit/rate-limit-service.test.mjs index f9a2b9992..e0539e891 100755 --- a/test/unit/rate-limit-service.test.mjs +++ b/test/unit/rate-limit-service.test.mjs @@ -238,3 +238,23 @@ test('anthropic429Reset prefers an exhausted 7d window over 5h', () => { assert.equal(out.window, '7d') assert.equal(out.resetAt, now + 86_400_000) }) + +test('429 headers retain the ticket captured before the attempt began', async () => { + const { service, accountQuota, close } = await setup() + let options + accountQuota.ingestHeaders = (_accountId, _headers, _usage, value) => { + options = value + } + try { + service.handleUpstreamError({ + accountId: 'acc-1', + vmId: 'vm-01', + result: { ...limitResult('rate limited'), accountObservation: 41 }, + policy: quotaPolicy, + }) + assert.equal(options.observation, 41) + assert.equal(options.countRequest, false) + } finally { + await close() + } +}) From 648a224b69cec83c46f2f5c57d1c5b426734bf8e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:20:16 +0800 Subject: [PATCH 107/473] test(proxy): await durable mutations in VM and sticky fixtures --- test/unit/panel-vm-create.test.mjs | 10 +++++----- test/unit/panel-vm-exit-probe.test.mjs | 8 ++++---- test/unit/sticky-router.test.mjs | 26 ++++++++++++++------------ test/unit/vm-package.test.mjs | 7 +++++-- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/test/unit/panel-vm-create.test.mjs b/test/unit/panel-vm-create.test.mjs index 0f2865893..c3c9acda6 100644 --- a/test/unit/panel-vm-create.test.mjs +++ b/test/unit/panel-vm-create.test.mjs @@ -251,8 +251,8 @@ test('create binds an explicitly chosen exit instead of auto-allocating', async const pool = await new ProxyPool({ dataDir: root, geoLookup: async () => ({ ok: false, error: 'offline' }) }).init() pool.stopScheduler() try { - const socks = pool.importLines('1.2.3.4:1080').items[0].id - const local = pool.ensureLocal().proxy.id + const socks = (await pool.importLines('1.2.3.4:1080')).items[0].id + const local = (await pool.ensureLocal()).proxy.id const { handlePanel, response } = makeCreateHandler(root, { start: false, proxy_id: local }, pool) await handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/create')) assert.equal(response.status, 200, response.body?.error?.message || JSON.stringify(response.body)) @@ -302,10 +302,10 @@ test('create refuses an unknown or full exit before writing the slot', async () await missing.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/create')) assert.equal(missing.response.status, 404) - const local = pool.ensureLocal().proxy.id - pool.updateConfig({ bind_limit: 1 }) + const local = (await pool.ensureLocal()).proxy.id + await pool.updateConfig({ bind_limit: 1 }) pool.stopScheduler() - pool.bind(local, 'vm-other') + await pool.bind(local, 'vm-other') const full = makeCreateHandler(root, { id: 'vm-x', start: false, proxy_id: local }, pool) await full.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/create')) assert.equal(full.response.status, 409) diff --git a/test/unit/panel-vm-exit-probe.test.mjs b/test/unit/panel-vm-exit-probe.test.mjs index c6f66e643..b7f2ddb6d 100644 --- a/test/unit/panel-vm-exit-probe.test.mjs +++ b/test/unit/panel-vm-exit-probe.test.mjs @@ -63,8 +63,8 @@ test('an unbound slot has no exit to probe', async (t) => { test('a node slot on local egress probes its own node row and refreshes it', async (t) => { const { root, pool, probe, lookups } = await setup(t) seed(root, { id: 'vm-tokyo', node_id: 'node-a', proxy: { id: 'px-local', host: 'local', scheme: 'local' } }) - pool.ensureLocal() - assert.equal(pool.bind('px-local', 'vm-tokyo').proxy.id, 'px-local-node-a') + await pool.ensureLocal() + assert.equal((await pool.bind('px-local', 'vm-tokyo')).proxy.id, 'px-local-node-a') const res = await probe('vm-tokyo') assert.equal(res.status, 200, JSON.stringify(res.body)) assert.equal(res.body.data.via, 'node') @@ -89,8 +89,8 @@ test('a control-plane Codex slot is probed through the deployment proxy', async family: 'codex', proxy: { id: 'px-local', host: 'local', scheme: 'local' }, }) - pool.ensureLocal() - pool.bind('px-local', 'vm-gpt') + await pool.ensureLocal() + await pool.bind('px-local', 'vm-gpt') const res = await probe('vm-gpt') assert.equal(res.status, 200, JSON.stringify(res.body)) assert.equal(res.body.data.via, 'deploy_proxy') diff --git a/test/unit/sticky-router.test.mjs b/test/unit/sticky-router.test.mjs index 0da51bd73..d9f39fa88 100644 --- a/test/unit/sticky-router.test.mjs +++ b/test/unit/sticky-router.test.mjs @@ -742,11 +742,13 @@ test('unbind and unbindByAccount drop dead bindings', async () => { test('proxy pool import/bind/config persist across re-open', async () => { const dir = tmpDir('kin-proxy-') const pool = await new ProxyPool({ dataDir: dir }).init() - const res = pool.importLines('socks5://user:pass@10.0.0.1:1080\n10.0.0.2:1080\nbadline:xx\n10.0.0.1:1080:user:pass') + const res = await pool.importLines( + 'socks5://user:pass@10.0.0.1:1080\n10.0.0.2:1080\nbadline:xx\n10.0.0.1:1080:user:pass', + ) assert.equal(res.added, 2) const id = pool.snapshot().proxies[0].id - assert.equal(pool.bind(id, 'vm-1').ok, true) - pool.updateConfig({ probe_interval_min: 30, max_failures: 3 }) + assert.equal((await pool.bind(id, 'vm-1')).ok, true) + await pool.updateConfig({ probe_interval_min: 30, max_failures: 3 }) pool.stopScheduler() const pool2 = await new ProxyPool({ dataDir: dir }).init() @@ -763,11 +765,11 @@ test('proxy pool import/bind/config persist across re-open', async () => { test('proxy remove + unbindVm persist', async () => { const dir = tmpDir('kin-proxy-') const pool = await new ProxyPool({ dataDir: dir }).init() - pool.importLines('10.1.1.1:1080\n10.1.1.2:1080') + await pool.importLines('10.1.1.1:1080\n10.1.1.2:1080') const [a, b] = pool.snapshot().proxies.map((p) => p.id) - pool.bind(a, 'vm-z') - pool.unbindVm('vm-z') - pool.remove(b) + await pool.bind(a, 'vm-z') + await pool.unbindVm('vm-z') + await pool.remove(b) pool.stopScheduler() const pool2 = await new ProxyPool({ dataDir: dir }).init() @@ -786,15 +788,15 @@ test('disconnect_on_error config persists and runtime failure disables slot', as onDisableVm: (vmId, reason, proxyId) => disabled.push({ vmId, reason, proxyId }), onDisconnectVm: (vmId, reason, proxyId) => disconnected.push({ vmId, reason, proxyId }), }).init() - pool.importLines('10.2.2.2:1080') + await pool.importLines('10.2.2.2:1080') const id = pool.snapshot().proxies[0].id - pool.bind(id, 'vm-err') + await pool.bind(id, 'vm-err') assert.equal(pool.snapshot().config.disconnect_on_error, false) - const skipped = pool.reportRuntimeFailure('vm-err', 'proxy_transport_failure') + const skipped = await pool.reportRuntimeFailure('vm-err', 'proxy_transport_failure') assert.equal(skipped.skipped, true) assert.equal(disconnected.length, 0) - const updated = pool.updateConfig({ enabled: false, disconnect_on_error: true, max_failures: 1 }) + const updated = await pool.updateConfig({ enabled: false, disconnect_on_error: true, max_failures: 1 }) assert.equal(updated.ok, true) assert.equal(updated.config.disconnect_on_error, true) pool.stopScheduler() @@ -805,7 +807,7 @@ test('disconnect_on_error config persists and runtime failure disables slot', as onDisconnectVm: (vmId, reason, proxyId) => disconnected.push({ vmId, reason, proxyId }), }).init() assert.equal(pool2.snapshot().config.disconnect_on_error, true) - const reported = pool2.reportRuntimeFailure('vm-err', 'proxy_transport_failure') + const reported = await pool2.reportRuntimeFailure('vm-err', 'proxy_transport_failure') assert.equal(reported.ok, true) assert.equal(reported.skipped, false) assert.equal(reported.proxy.status, 'dead') diff --git a/test/unit/vm-package.test.mjs b/test/unit/vm-package.test.mjs index 3217b9a32..8554341f5 100644 --- a/test/unit/vm-package.test.mjs +++ b/test/unit/vm-package.test.mjs @@ -155,7 +155,10 @@ test('import creates the slot and rolls a colliding id forward', async (t) => { assert.equal(saved.claude.access_token, undefined) const cred = readWorkerCredentialFile(path.join(root, 'vms', 'vm-pkg', 'cli-home')) assert.equal(cred.access_token, 'sk-ant-oat01-PACKAGE') - assert.equal(pool.ensureSocks({ host: '10.2.2.2', port: 1080, username: 'alice', password: 's3cret' }).created, false) + assert.equal( + (await pool.ensureSocks({ host: '10.2.2.2', port: 1080, username: 'alice', password: 's3cret' })).created, + false, + ) const again = makeHandler(root, doc, { pool }) await again.handlePanel({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/package')) @@ -291,7 +294,7 @@ test('tenant import cannot bind px-local or another owner SOCKS and drops seed_p assert.equal(fs.existsSync(path.join(root, 'vms', 'vm-t-local.json')), false) // The platform already owns this SOCKS row; equal credentials must not hand it to the tenant. - pool.ensureSocks({ host: '10.2.2.2', port: 1080, username: 'alice', password: 's3cret' }) + await pool.ensureSocks({ host: '10.2.2.2', port: 1080, username: 'alice', password: 's3cret' }) const taken = await importAs(claudePackage('vm-t-taken')) assert.equal(taken.status, 403) assert.equal(taken.body.error.code, 'proxy_forbidden') From 91b537d917349053f13f7b29e98f74c4be2c26f2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:20:35 +0800 Subject: [PATCH 108/473] fix: persist account usage and observations across replicas --- src/lib/admin/panel-api.mjs | 32 ++- src/lib/admin/vm-test-chat.mjs | 18 +- .../008_account_observations.sql | 4 + src/lib/db/repos/accounts-repo.mjs | 209 +++++++++++++++--- src/lib/oauth/import-commit.mjs | 6 +- src/lib/pool/account-quota.mjs | 143 ++++++++---- src/lib/protocol/user-count-tokens.mjs | 5 +- test/unit/shared-account-quota.test.mjs | 193 ++++++++++++++++ 8 files changed, 522 insertions(+), 88 deletions(-) create mode 100644 src/lib/db/postgres-migrations/008_account_observations.sql diff --git a/src/lib/admin/panel-api.mjs b/src/lib/admin/panel-api.mjs index 5e88752ce..151ef11cb 100644 --- a/src/lib/admin/panel-api.mjs +++ b/src/lib/admin/panel-api.mjs @@ -1009,7 +1009,7 @@ export async function buildClaudeResetRedeem({ cfg, id, idempotencyKey, transpor }) } -function applyFableProbeResult(accountQuota, accountId, found) { +function applyFableProbeResult(accountQuota, accountId, found, observation) { if (found?.tier !== 'pro' && found?.tier !== 'max') return null const saved = accountQuota.repo.get(accountId) if (!saved || !found.fable) return found.tier @@ -1028,7 +1028,7 @@ function applyFableProbeResult(accountQuota, accountId, found) { reset: found.fable.reset_at || prev.reset || null, probed_at: new Date().toISOString(), } - accountQuota.repo.save(saved) + accountQuota.repo.observe(saved, { observation, paths: [['unified', 'fable']] }) return found.tier } function fatalUsageCredentialFailure(result = {}) { @@ -1137,6 +1137,7 @@ export async function buildProbeOne({ if (force) cache.clear(accountId) const skipHop = !shouldHopOfficialUsage(acc?.unified, { hop, force }) const headerProbe = skipHop ? probeFromPassiveHeaders(acc?.unified || {}) : null + const observation = await accountQuota.beginObservation() const result = skipHop ? { ok: false, @@ -1157,12 +1158,19 @@ export async function buildProbeOne({ }) } catch {} } - if (!skipHop) accountQuota.ingestOAuthUsage(accountId, result) + if (!skipHop) accountQuota.ingestOAuthUsage(accountId, result, { observation }) if (isSetupTokenMode(credentialModeOfVm(vm)) && (force || includeFable) && !result.usage_scope_missing) { try { - await applyFableProbeResult(accountQuota, accountId, await fableProbe({ exec, timeoutMs: 20000 })) + const fableObservation = await accountQuota.beginObservation() + await applyFableProbeResult( + accountQuota, + accountId, + await fableProbe({ exec, timeoutMs: 20000 }), + fableObservation, + ) } catch {} } + await accountQuota.flush() const after = accountQuota.repo.get(accountId) const qAfter = quotaFromAccount(after) const rateLimited = isOfficialUsageRateLimited(result) @@ -1186,11 +1194,17 @@ export async function buildProbeOne({ ).key const completeUsage = result.ok === true && result.limits_present === true if (completeUsage && (result.account_tier === 'pro' || result.account_tier === 'max')) { - accountQuota.setAccountTier(accountId, result.account_tier, { source: 'usage' }) - persistAccountTier(cfg.paths.project, id, result.account_tier, { source: 'usage' }) + accountQuota.setAccountTier(accountId, result.account_tier, { source: 'usage', observation }) + await accountQuota.flush() + if (accountQuota.repo.get(accountId)?.unified?.account_tier === result.account_tier) { + persistAccountTier(cfg.paths.project, id, result.account_tier, { source: 'usage' }) + } } else if (completeUsage && (tier === 'pro' || tier === 'max')) { - accountQuota.setAccountTier(accountId, tier, { source: 'usage' }) - persistAccountTier(cfg.paths.project, id, tier, { source: 'usage' }) + accountQuota.setAccountTier(accountId, tier, { source: 'usage', observation }) + await accountQuota.flush() + if (accountQuota.repo.get(accountId)?.unified?.account_tier === tier) { + persistAccountTier(cfg.paths.project, id, tier, { source: 'usage' }) + } } const availability = evaluateAccount({ vm, @@ -1287,7 +1301,7 @@ export async function buildProbeOne({ error: data.error, data_at: data.via === 'passive-headers' ? checked.unified.headers?.sampled_at || null : data.probed_at, } - accountQuota.repo.save(checked) + accountQuota.repo.observe(checked, { observation, paths: [['unified', 'last_probe_check']] }) return ok(data) } diff --git a/src/lib/admin/vm-test-chat.mjs b/src/lib/admin/vm-test-chat.mjs index 089f4aa37..402965398 100644 --- a/src/lib/admin/vm-test-chat.mjs +++ b/src/lib/admin/vm-test-chat.mjs @@ -1099,7 +1099,9 @@ export async function runVmTestChat(opts = {}) { const timeoutMs = Math.min(Math.max(Number(opts.timeoutMs) || 90000, 10000), 180000) let result + let observation try { + if (!codex) observation = await opts.accountQuota?.beginObservation?.() result = await (codex ? postV1Responses : postV1Messages)({ baseUrl, apiKey, @@ -1197,12 +1199,16 @@ export async function runVmTestChat(opts = {}) { if (errObj.request_id) push('info', `request_id=${errObj.request_id}`) if (result?.status === 401 && /revoked/i.test(String(errObj.message || errObj.upstream_message || ''))) { try { - opts.accountQuota?.recordLastProbe?.(vm.claude?.account_uuid || vm.account_uuid || vmId, { - ok: false, - source: 'test-chat', - error: errObj.message, - status: 401, - }) + opts.accountQuota?.recordLastProbe?.( + vm.claude?.account_uuid || vm.account_uuid || vmId, + { + ok: false, + source: 'test-chat', + error: errObj.message, + status: 401, + }, + { observation }, + ) } catch {} } } diff --git a/src/lib/db/postgres-migrations/008_account_observations.sql b/src/lib/db/postgres-migrations/008_account_observations.sql new file mode 100644 index 000000000..a99167c20 --- /dev/null +++ b/src/lib/db/postgres-migrations/008_account_observations.sql @@ -0,0 +1,4 @@ +-- Tickets are allocated before an asynchronous observation is persisted. +-- Per-field revisions reject older completions without replacing other observations. +ALTER TABLE accounts ADD COLUMN observation_versions jsonb NOT NULL DEFAULT '{}'; +CREATE SEQUENCE account_observation_seq OWNED BY accounts.observation_versions; diff --git a/src/lib/db/repos/accounts-repo.mjs b/src/lib/db/repos/accounts-repo.mjs index fd707673a..acd2a0858 100644 --- a/src/lib/db/repos/accounts-repo.mjs +++ b/src/lib/db/repos/accounts-repo.mjs @@ -9,9 +9,91 @@ */ import { getDb, withTransaction } from '../database.mjs' -import { integerValue } from '../integers.mjs' +import { addInteger, integerValue } from '../integers.mjs' +import { isDeepStrictEqual } from 'node:util' const MAX_ALLOCATIONS_PER_ACCOUNT = 50 +const COUNTERS = ['requests', 'tokens_in', 'tokens_out', 'cache_read_tokens', 'cache_creation_tokens'] +const FIELDS = { + vm_id: 'vm_id', + email: 'email', + name: 'name', + platform: 'platform', + type: 'type', + status: 'status', + priority: 'priority', + rate_multiplier: 'rate_multiplier', + schedulable: 'schedulable', + max_concurrency: 'concurrency', + concurrency_override: 'concurrency_override', + max_rpm: 'max_rpm', + rpm_override: 'rpm_override', +} + +function observationChanges(before, after, path, out) { + if (isDeepStrictEqual(before, after)) return + if (after && typeof after === 'object' && !Array.isArray(after)) { + const previous = before && typeof before === 'object' && !Array.isArray(before) ? before : {} + for (const key of new Set([...Object.keys(previous), ...Object.keys(after)])) { + observationChanges(previous[key], after[key], [...path, key], out) + } + } else { + out.push({ path, value: after, before }) + } +} + +function touchObservation(before, after, path, out) { + if (after && typeof after === 'object' && !Array.isArray(after) && Object.keys(after).length) { + for (const key of Object.keys(after)) touchObservation(before?.[key], after[key], [...path, key], out) + return + } + if (!out.some((change) => isDeepStrictEqual(change.path, path))) out.push({ path, value: after, before }) +} + +/** Describe the caller's mutations, never persist the unchanged parts of its cached account. */ +export function accountChanges(before, after, { paths = [] } = {}) { + const fields = {} + const deltas = {} + const observations = [] + for (const field of Object.keys(FIELDS)) { + if (!isDeepStrictEqual(before[field], after[field])) fields[field] = after[field] ?? null + } + for (const field of COUNTERS) { + const delta = addInteger(after[field], (-BigInt(integerValue(before[field]))).toString()) + if (delta !== 0) deltas[field] = delta + } + for (const field of ['unified', 'last_blocked', 'last_cli_rate_limit']) { + observationChanges(before[field], after[field], [field], observations) + } + for (const path of paths) touchObservation(valueAt(before, path), valueAt(after, path), path, observations) + return { fields, deltas, observations } +} + +function valueAt(object, path) { + return path.reduce((value, key) => value?.[key], object) +} + +function setAt(object, path, value) { + let target = object + for (const key of path.slice(0, -1)) { + if (!Object.hasOwn(target, key) || !target[key] || typeof target[key] !== 'object' || Array.isArray(target[key])) { + Object.defineProperty(target, key, { value: {}, writable: true, enumerable: true, configurable: true }) + } + target = target[key] + } + if (value === undefined) delete target[path.at(-1)] + else + Object.defineProperty(target, path.at(-1), { + value: structuredClone(value), + writable: true, + enumerable: true, + configurable: true, + }) +} + +function overlapping(left, right) { + return left.slice(0, Math.min(left.length, right.length)).every((key, index) => key === right[index]) +} function parse(json, fallback = null) { if (json == null) return fallback @@ -71,6 +153,7 @@ function buildExtra(acc) { export class AccountsRepo { constructor(db = getDb()) { this.db = db + this.snapshots = new WeakMap() this._get = db.prepare('SELECT * FROM accounts WHERE id = $1') this._list = db.prepare('SELECT * FROM accounts ORDER BY id') this._insert = db.prepare(` @@ -81,15 +164,13 @@ export class AccountsRepo { cache_read_tokens, cache_creation_tokens, extra, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) + ON CONFLICT (id) DO NOTHING `) - this._update = db.prepare(` - UPDATE accounts SET vm_id = $1, email = $2, name = $3, type = $4, - concurrency = $5, concurrency_override = $6, - max_rpm = $7, rpm_override = $8, - requests = $9, tokens_in = $10, tokens_out = $11, - cache_read_tokens = $12, cache_creation_tokens = $13, - extra = $14, updated_at = $15 - WHERE id = $16 + this._nextObservation = db.prepare("SELECT nextval('account_observation_seq') AS observation") + this._lock = db.prepare('SELECT * FROM accounts WHERE id = $1 FOR UPDATE') + this._observe = db.prepare(` + UPDATE accounts SET extra = $1, observation_versions = $2::jsonb, updated_at = $3 + WHERE id = $4 `) this._insertAlloc = db.prepare(` INSERT INTO account_allocations (account_id, at, source, util_5h, util_7d, status_5h, status_7d, claim, tokens_in, tokens_out) @@ -114,11 +195,17 @@ export class AccountsRepo { } async get(accountId) { - return rowToAccount(await this._get.get(accountId)) + const account = rowToAccount(await this._get.get(accountId)) + if (account) this.snapshots.set(account, structuredClone(account)) + return account } async list() { - return (await this._list.all()).map(rowToAccount) + return (await this._list.all()).map((row) => { + const account = rowToAccount(row) + this.snapshots.set(account, structuredClone(account)) + return account + }) } /** Insert if missing, returns stored account. New accounts join the coding group (id=1). */ @@ -151,27 +238,89 @@ export class AccountsRepo { }) } - /** Full-row save of a mutated account object. */ + /** Compatibility for callers mutating a snapshot returned by this repository. */ async save(acc) { - await this._update.run( - acc.vm_id ?? null, - acc.email ?? null, - acc.name ?? acc.email ?? acc.account_id, - acc.type ?? 'oauth', - acc.max_concurrency ?? 2, - acc.concurrency_override ? 1 : 0, - acc.max_rpm ?? 0, - acc.rpm_override ? 1 : 0, - integerValue(acc.requests), - integerValue(acc.tokens_in), - integerValue(acc.tokens_out), - integerValue(acc.cache_read_tokens), - integerValue(acc.cache_creation_tokens), - buildExtra(acc), - new Date().toISOString(), - acc.account_id, + for (const field of COUNTERS) integerValue(acc[field]) + const before = this.snapshots.get(acc) + if (!before) throw new Error('account save requires a repository snapshot; use an explicit patch') + const stored = await this.applyChanges(acc.account_id, accountChanges(before, acc)) + this.snapshots.set(acc, structuredClone(acc)) + return stored + } + + async beginObservation() { + return (await this._nextObservation.get()).observation + } + + async patchFields(accountId, fields) { + const entries = Object.entries(fields) + for (const [field] of entries) { + if (!Object.hasOwn(FIELDS, field)) throw new Error(`unsupported account field: ${field}`) + } + if (!entries.length) return this.get(accountId) + const values = entries.map(([field, value]) => + ['schedulable', 'concurrency_override', 'rpm_override'].includes(field) ? (value ? 1 : 0) : value, + ) + const assignments = entries.map(([field], index) => `${FIELDS[field]} = $${index + 1}`) + values.push(new Date().toISOString(), accountId) + const row = await this.db + .prepare(`UPDATE accounts SET ${assignments.join(', ')}, + updated_at = $${values.length - 1} WHERE id = $${values.length} RETURNING *`) + .get(...values) + return rowToAccount(row) + } + + async incrementUsage(accountId, deltas) { + const values = COUNTERS.map((field) => integerValue(deltas[field])) + values.push(new Date().toISOString(), accountId) + return rowToAccount( + await this.db + .prepare(`UPDATE accounts SET + ${COUNTERS.map((field, index) => `${field} = COALESCE(${field}, 0) + $${index + 1}::bigint`).join(', ')}, + updated_at = $6 WHERE id = $7 RETURNING *`) + .get(...values), ) - return await this.get(acc.account_id) + } + + async patchObservation(accountId, changes, observation, { conditional = false, probeBase } = {}) { + const sequence = BigInt(observation) + return withTransaction(this.db, async () => { + const row = await this._lock.get(accountId) + if (!row) return null + const extra = parse(row.extra, {}) || {} + const versions = row.observation_versions || {} + const probeChanged = probeBase !== undefined && !isDeepStrictEqual(extra.unified?.last_probe ?? null, probeBase) + let changed = false + for (const change of changes) { + if (!['unified', 'last_blocked', 'last_cli_rate_limit'].includes(change.path[0])) { + throw new Error('unsupported account observation') + } + if ( + Object.entries(versions).some( + ([path, version]) => overlapping(JSON.parse(path), change.path) && BigInt(version) > sequence, + ) + ) + continue + if (probeChanged && change.path[0] === 'unified' && change.path[1] === 'last_probe') continue + if (conditional && !isDeepStrictEqual(valueAt(extra, change.path), change.before)) continue + setAt(extra, change.path, change.value) + versions[JSON.stringify(change.path)] = sequence.toString() + changed = true + } + if (changed) + await this._observe.run(JSON.stringify(extra), JSON.stringify(versions), new Date().toISOString(), accountId) + return this.get(accountId) + }) + } + + async applyChanges(accountId, { fields = {}, deltas = {}, observations = [] }, options = {}) { + const observation = observations.length ? await (options.observation ?? this.beginObservation()) : null + return withTransaction(this.db, async () => { + if (Object.keys(fields).length) await this.patchFields(accountId, fields) + if (Object.keys(deltas).length) await this.incrementUsage(accountId, deltas) + if (observations.length) await this.patchObservation(accountId, observations, observation, options) + return this.get(accountId) + }) } async remove(accountId) { diff --git a/src/lib/oauth/import-commit.mjs b/src/lib/oauth/import-commit.mjs index c0c3d1d39..c5f13f275 100644 --- a/src/lib/oauth/import-commit.mjs +++ b/src/lib/oauth/import-commit.mjs @@ -111,17 +111,19 @@ export function createImportCommit(ctx) { } function officialCcStatsHandler(vmId, accountUuid = null) { + const observation = ctx.accountQuota?.beginObservation?.() + observation?.catch(() => {}) return (stats) => { const accountId = accountUuid || getVm(ctx.cfg.paths.project, vmId)?.claude?.account_uuid || vmId if (stats?.five_hour || stats?.seven_day || stats?.seven_day_oi || stats?.extra_usage) { try { - ctx.accountQuota.ingestOAuthUsage(accountId, { usage_status: 200, ok: true, ...stats }) + ctx.accountQuota.ingestOAuthUsage(accountId, { usage_status: 200, ok: true, ...stats }, { observation }) } catch {} } if ((stats?.account_tier === 'pro' || stats?.account_tier === 'max') && stats?.limits_present === true) { const source = 'usage' try { - ctx.accountQuota.setAccountTier(accountId, stats.account_tier, { source }) + ctx.accountQuota.setAccountTier(accountId, stats.account_tier, { source, observation }) } catch {} try { persistAccountTier(ctx.cfg.paths.project, vmId, stats.account_tier, { source }) diff --git a/src/lib/pool/account-quota.mjs b/src/lib/pool/account-quota.mjs index e685e5e31..58db854b7 100644 --- a/src/lib/pool/account-quota.mjs +++ b/src/lib/pool/account-quota.mjs @@ -11,7 +11,7 @@ import { resolveStoreDb } from '../db/database.mjs' import { addInteger } from '../db/integers.mjs' -import { AccountsRepo } from '../db/repos/accounts-repo.mjs' +import { accountChanges, AccountsRepo } from '../db/repos/accounts-repo.mjs' import { WriteQueue } from './write-queue.mjs' import { computeWeeklySplit, weeklySplitConfig } from './weekly-split.mjs' import { @@ -65,6 +65,7 @@ export class AccountQuota { this.persistence = new AccountsRepo(this.db) this.writes = this.db.schedulingWriteQueue ||= new WriteQueue(this.db) this.rows = new Map() + this.revisions = new Map() this.allocations = new Map() this.seedAccounts = accounts || [] this.repo = this._workingSet() @@ -111,21 +112,44 @@ export class AccountQuota { } _workingSet() { - const save = (account, insert = false) => { + const publish = (account, write) => { const next = structuredClone(account) + const revision = (this.revisions.get(next.account_id) || 0) + 1 + this.revisions.set(next.account_id, revision) this.rows.set(next.account_id, next) this.writes.enqueue(async () => { - const existing = insert ? await this.persistence.get(next.account_id) : true - return this.persistence[existing ? 'save' : 'insert'](next) + const stored = await write() + // A completed write may carry remote changes, but must not erase a newer local mutation. + if (stored && this.revisions.get(next.account_id) === revision) this.rows.set(next.account_id, stored) + return stored }) return structuredClone(next) } + const save = (account, { observation, conditional = true, paths, probeBase } = {}) => { + const changes = accountChanges(this.rows.get(account.account_id) || {}, account, { paths }) + const ticket = changes.observations.length ? (observation ?? this.beginObservation()) : null + // The ordered queue awaits the ticket; attach its rejection handler immediately. + if (ticket?.catch) ticket.catch(() => {}) + return publish(account, () => + this.persistence.applyChanges(account.account_id, changes, { + observation: ticket, + conditional, + probeBase, + }), + ) + } return { get: (id) => structuredClone(this.rows.get(id) || null), list: () => [...this.rows.values()].map((row) => structuredClone(row)), - insert: (row) => save(row, true), + insert: (row) => publish(row, () => this.persistence.insert(row)), save: (row) => save(row), + observe: (row, options) => save(row, { ...options, conditional: false }), + patch: (id, fields) => + publish({ ...this.rows.get(id), ...fields, account_id: id }, () => + this.persistence.applyChanges(id, { fields }), + ), remove: (id) => { + this.revisions.set(id, (this.revisions.get(id) || 0) + 1) const removed = this.rows.delete(id) this.allocations.delete(id) this.writes.enqueue(() => this.persistence.remove(id)) @@ -141,6 +165,11 @@ export class AccountQuota { } } + /** Reserve before remote I/O when the producer can identify when an observation starts. */ + beginObservation() { + return this.persistence.beginObservation() + } + /** Re-bind to a fresh DB connection (after backup restore). */ async rebind(db) { await this.flush() @@ -211,9 +240,10 @@ export class AccountQuota { last_error: null, }) } else if ((account.vm_id && acc.vm_id !== account.vm_id) || (account.type && acc.type !== account.type)) { - if (account.vm_id) acc.vm_id = account.vm_id - if (account.type) acc.type = account.type - acc = this.repo.save(acc) + acc = this.repo.patch(id, { + ...(account.vm_id ? { vm_id: account.vm_id } : {}), + ...(account.type ? { type: account.type } : {}), + }) } return acc } @@ -233,6 +263,16 @@ export class AccountQuota { const writeExtra = countRequest || exhausted const u5 = num(h['anthropic-ratelimit-unified-5h-utilization']) const u7 = num(h['anthropic-ratelimit-unified-7d-utilization']) + const paths = [] + for (const key of ['5h', '7d']) { + if (!writeExtra) continue + const prefix = `anthropic-ratelimit-unified-${key}-` + if (h[`${prefix}utilization`] != null) { + paths.push(['unified', 'headers', key, 'utilization'], ['unified', 'headers', key, 'status']) + } + if (h[`${prefix}status`]) paths.push(['unified', 'headers', key, 'status']) + if (h[`${prefix}reset`]) paths.push(['unified', 'headers', key, 'reset']) + } let sampled = false if (exhausted) { dropElapsedReset(acc, '5h', h['anthropic-ratelimit-unified-5h-reset']) @@ -320,7 +360,7 @@ export class AccountQuota { }) } - const saved = this.repo.save(acc) + const saved = this.repo.observe(acc, { observation: extra.observation, paths }) this._writeSessionWindow(saved) if (sampled) { const observed5 = headerWindowOrEmpty(saved.unified, '5h') @@ -334,7 +374,7 @@ export class AccountQuota { return saved } - recordLastProbe(accountId, probe = {}) { + recordLastProbe(accountId, probe = {}, { observation } = {}) { if (!accountId) return null const acc = this.ensure({ account_id: accountId }) const prev = acc.last_probe || acc.unified?.last_probe || null @@ -350,14 +390,18 @@ export class AccountQuota { status: probe.status || null, } acc.unified.last_probe = acc.last_probe - return this.repo.save(acc) + return this.repo.observe(acc, { + observation, + paths: [['unified', 'last_probe']], + ...(isTestProbeSource(probe) ? { probeBase: prev } : {}), + }) } /** * CRS oauth/usage snapshot from the VM UID probe. * Fable weekly limit is stored separately and does not unschedulable the account. */ - ingestOAuthUsage(accountId, probe = {}) { + ingestOAuthUsage(accountId, probe = {}, { observation } = {}) { const acc = this.ensure({ account_id: accountId }) const nowIso = probe.probed_at || new Date().toISOString() if (isOfficialUsageRateLimited(probe)) { @@ -378,12 +422,19 @@ export class AccountQuota { attempted_at: nowIso, } acc.unified.last_probe = acc.last_probe - return this.repo.save(acc) + return this.repo.observe(acc, { observation, probeBase: prev.at ? prev : null }) } if (acc.unified) delete acc.unified.usage_rate_limited_until const official = probe.interpretations?.official || {} const w5 = official.five_hour || probe.five_hour || {} const w7 = official.seven_day || probe.seven_day || {} + const paths = [['unified', 'last_probe']] + for (const [key, window] of [ + ['5h', w5], + ['7d', w7], + ]) { + if (applyOfficialWindow(window)) paths.push(['unified', key], ['unified', 'official', key]) + } writeOfficialWindow(acc, '5h', w5) writeOfficialWindow(acc, '7d', w7) if (probe.seven_day_sonnet) { @@ -440,6 +491,8 @@ export class AccountQuota { else if (probe.usage_has_fable === false) acc.unified.account_tier = 'pro' if (acc.unified.account_tier === 'pro' || acc.unified.account_tier === 'max') acc.unified.account_tier_source = 'usage' + paths.push(['unified', 'account_tier'], ['unified', 'account_tier_source']) + if (probe.usage_has_fable != null) paths.push(['unified', 'usage_has_fable']) } if (probe.fable && !fableTransport) { const planDenied = completeOfficialUsage && !hasFableUsage && isFablePlanDenied(probe.fable) @@ -492,7 +545,7 @@ export class AccountQuota { healOfficialOverwriteExtra(acc, accountId, this.repo) clearHeaderExhaustIfOfficialOpen(acc) } - const saved = this.repo.save(acc) + const saved = this.repo.observe(acc, { observation, paths }) this._writeSessionWindow(saved) if (usageOk) { const observed5 = headerWindowOrEmpty(saved.unified, '5h') @@ -513,9 +566,10 @@ export class AccountQuota { * Official Claude Code stream-json `rate_limit_event`. * Live CLI includes `unifiedWindows.*.utilization`; keep status + reset too. */ - ingestCliRateLimit(accountId, rateLimitInfo, usageBody = null, { countRequest = null } = {}) { + ingestCliRateLimit(accountId, rateLimitInfo, usageBody = null, { countRequest = null, observation } = {}) { const acc = this.ensure({ account_id: accountId }) const infos = Array.isArray(rateLimitInfo) ? rateLimitInfo : rateLimitInfo ? [rateLimitInfo] : [] + const paths = [] for (const info of infos) { if (!info || typeof info !== 'object') continue const typ = String(info.rateLimitType || info.rate_limit_type || '') @@ -524,6 +578,9 @@ export class AccountQuota { if (window) { const slot = window === '7d' ? windows.seven_day || windows['7d'] : windows.five_hour || windows['5h'] const utilization = officialUtilToExtra(slot?.utilization) + if (utilization != null) paths.push(['unified', 'headers', window, 'utilization']) + if (info.status || utilization != null) paths.push(['unified', 'headers', window, 'status']) + if (info.resetsAt != null || info.resets_at != null) paths.push(['unified', 'headers', window, 'reset']) writeHeaderWindow(acc, window, { status: info.status || (utilization != null ? statusFromUtil(utilization) : null), reset: info.resetsAt != null || info.resets_at != null ? epochToIso(info.resetsAt ?? info.resets_at) : null, @@ -561,7 +618,7 @@ export class AccountQuota { tokens_out: usageBody?.output_tokens ?? null, }) - return this.repo.save(acc) + return this.repo.observe(acc, { observation, paths }) } /** @@ -1142,9 +1199,7 @@ export class AccountQuota { const acc = this.ensure({ account_id: accountUuid, vm_id: vmId, email }) if (!acc) return null if (acc.vm_id !== vmId || (email && acc.email !== email)) { - acc.vm_id = vmId - if (email) acc.email = email - return this.repo.save(acc) + return this.repo.patch(accountUuid, { vm_id: vmId, ...(email ? { email } : {}) }) } return acc } @@ -1182,18 +1237,31 @@ export class AccountQuota { return this.repo.save(acc) } - setAccountTier(accountId, tier, { source = null } = {}) { + setAccountTier(accountId, tier, { source = null, observation } = {}) { const key = String(tier || '').toLowerCase() if (!accountId || (key !== 'pro' && key !== 'max')) return null const acc = this.repo.get(accountId) if (!acc) return null if (acc.unified?.account_tier_source === 'profile' && source !== 'profile' && source !== 'usage') return acc - if (acc.unified?.account_tier === key && (acc.unified?.account_tier_source || null) === source) return acc + if ( + observation == null && + acc.unified?.account_tier === key && + (acc.unified?.account_tier_source || null) === source + ) + return acc acc.unified = acc.unified || {} acc.unified.account_tier = key acc.unified.account_tier_source = source acc.unified.updated_at = new Date().toISOString() - return this.repo.save(acc) + return observation == null + ? this.repo.save(acc) + : this.repo.observe(acc, { + observation, + paths: [ + ['unified', 'account_tier'], + ['unified', 'account_tier_source'], + ], + }) } markModelUnsupported(accountId, model, durationMs = 60 * 60_000) { @@ -1225,9 +1293,10 @@ export class AccountQuota { setMaxConcurrency(accountId, n, { override = false } = {}) { const acc = this.ensure({ account_id: accountId }) if (!acc) return null - acc.max_concurrency = Math.max(0, Math.min(256, Number(n) || 0)) - acc.concurrency_override = override ? 1 : 0 - return this.repo.save(acc) + return this.repo.patch(accountId, { + max_concurrency: Math.max(0, Math.min(256, Number(n) || 0)), + concurrency_override: override ? 1 : 0, + }) } setMaxConcurrencyForVm(vmId, n) { @@ -1235,8 +1304,7 @@ export class AccountQuota { const out = [] for (const a of this.repo.list()) { if (a.vm_id === vmId || a.account_id === vmId) { - a.max_concurrency = v - out.push(this.repo.save(a)) + out.push(this.repo.patch(a.account_id, { max_concurrency: v })) } } if (!out.length) out.push(this.setMaxConcurrency(vmId, v)) @@ -1249,8 +1317,7 @@ export class AccountQuota { const out = [] for (const a of this.repo.list()) { if (skip.has(a.account_id) || skip.has(a.vm_id)) continue - a.max_concurrency = v - out.push(this.repo.save(a)) + out.push(this.repo.patch(a.account_id, { max_concurrency: v })) } return out } @@ -1258,9 +1325,10 @@ export class AccountQuota { setMaxRpm(accountId, n, { override = false } = {}) { const acc = this.ensure({ account_id: accountId }) if (!acc) return null - acc.max_rpm = Math.max(0, Math.min(1e6, Number(n) || 0)) - acc.rpm_override = override ? 1 : 0 - return this.repo.save(acc) + return this.repo.patch(accountId, { + max_rpm: Math.max(0, Math.min(1e6, Number(n) || 0)), + rpm_override: override ? 1 : 0, + }) } setMaxRpmForVm(vmId, n, { override = false } = {}) { @@ -1268,9 +1336,7 @@ export class AccountQuota { const out = [] for (const a of this.repo.list()) { if (a.vm_id === vmId || a.account_id === vmId) { - a.max_rpm = v - if (override) a.rpm_override = 1 - out.push(this.repo.save(a)) + out.push(this.repo.patch(a.account_id, { max_rpm: v, ...(override ? { rpm_override: 1 } : {}) })) } } if (!out.length) out.push(this.setMaxRpm(vmId, v, { override })) @@ -1286,8 +1352,7 @@ export class AccountQuota { const key = accountTierKey(a) const v = Math.max(0, Math.min(1e6, Number(policies[key]?.max_rpm) || 0)) if (Number(a.max_rpm) === v) continue - a.max_rpm = v - out.push(this.repo.save(a)) + out.push(this.repo.patch(a.account_id, { max_rpm: v })) } return out } @@ -1301,9 +1366,7 @@ export class AccountQuota { const key = accountTierKey(a) const v = Math.max(0, Math.min(256, Number(policies[key]?.max_concurrency) || 2)) if (Number(a.max_concurrency) === v) continue - a.max_concurrency = v - a.concurrency_override = 0 - out.push(this.repo.save(a)) + out.push(this.repo.patch(a.account_id, { max_concurrency: v, concurrency_override: 0 })) } return out } diff --git a/src/lib/protocol/user-count-tokens.mjs b/src/lib/protocol/user-count-tokens.mjs index 516d946f2..774299df8 100644 --- a/src/lib/protocol/user-count-tokens.mjs +++ b/src/lib/protocol/user-count-tokens.mjs @@ -355,14 +355,17 @@ export async function resolveUsageWindows({ if (!shouldHopOfficialUsage(acc?.unified, { hop: true })) return { listed, source } const cache = usageCache || getUsageCache() let result + let observation try { + observation = await accountQuota?.beginObservation?.() result = await cache.load(accountId, () => probe({ exec, vm, includeFable: false })) } catch { return { listed, source } } if (!result || isOfficialUsageRateLimited(result)) return { listed, source } try { - accountQuota?.ingestOAuthUsage?.(accountId, result) + accountQuota?.ingestOAuthUsage?.(accountId, result, { observation }) + await accountQuota?.flush?.() } catch {} const next = accountQuota?.repo?.get?.(accountId) || acc listed = listQuotaFromHeaders(next?.unified || {}) diff --git a/test/unit/shared-account-quota.test.mjs b/test/unit/shared-account-quota.test.mjs index b3ce2d79f..c8fcc0be8 100644 --- a/test/unit/shared-account-quota.test.mjs +++ b/test/unit/shared-account-quota.test.mjs @@ -1,7 +1,11 @@ import test from 'node:test' import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' +import { buildProbeOne } from '../../src/lib/admin/panel-api.mjs' async function pair(t) { const db = getDb() @@ -122,3 +126,192 @@ test('an older probe finishing after a newer observation cannot overwrite the ne assert.equal(saved.unified['5h'].utilization, 0.81) assert.equal(left.repo.get('shared-account').unified['5h'].utilization, 0.81) }) + +test('a fresh zero observation and an unchanged observation still supersede older tickets', async (t) => { + const { left, right } = await pair(t) + const older = await left.beginObservation() + const fresh = await right.beginObservation() + right.ingestOAuthUsage('shared-account', { ok: true, five_hour: { utilization: 0 } }, { observation: fresh }) + await right.flush() + left.ingestOAuthUsage('shared-account', { ok: true, five_hour: { utilization: 50 } }, { observation: older }) + await left.flush() + assert.equal((await left.persistence.get('shared-account')).unified['5h'].utilization, 0) +}) + +test('late header results keep additive usage but cannot replace a newer header observation', async (t) => { + const { left, right } = await pair(t) + const observation = await left.beginObservation() + right.ingestHeaders('shared-account', { 'anthropic-ratelimit-unified-5h-utilization': '0.81' }) + await right.flush() + left.ingestHeaders( + 'shared-account', + { 'anthropic-ratelimit-unified-5h-utilization': '0.12' }, + { input_tokens: 7 }, + { observation }, + ) + await left.flush() + const saved = await left.persistence.get('shared-account') + assert.equal(saved.requests, 2) + assert.equal(saved.tokens_in, 7) + assert.equal(saved.unified.headers['5h'].utilization, 0.81) +}) + +test('panel probes take their tickets before asynchronous I/O and preserve the newer quota and tier', async (t) => { + const { left, right } = await pair(t) + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'shared-account-probe-')) + t.after(() => fs.rmSync(project, { recursive: true, force: true })) + fs.mkdirSync(path.join(project, 'vms')) + fs.writeFileSync( + path.join(project, 'vms/vm-a.json'), + JSON.stringify({ + id: 'vm-a', + status: 'stopped', + claude: { mode: 'oauth', has_access: true, account_uuid: 'shared-account' }, + }), + ) + let entered + let resume + const started = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + resume = resolve + }) + const args = { cfg: { paths: { project }, rewrite: {} }, id: 'vm-a', force: true, hop: true } + const oldResult = buildProbeOne({ + ...args, + accountQuota: left, + usageCache: { + clear() {}, + async load() { + entered() + await gate + return { + ok: true, + limits_present: true, + usage_has_fable: false, + account_tier: 'pro', + five_hour: { utilization: 12 }, + } + }, + }, + }) + await started + await buildProbeOne({ + ...args, + accountQuota: right, + usageCache: { + clear() {}, + async load() { + return { + ok: true, + limits_present: true, + usage_has_fable: true, + account_tier: 'max', + five_hour: { utilization: 81 }, + } + }, + }, + }) + resume() + await oldResult + await Promise.all([left.flush(), right.flush()]) + const saved = await left.persistence.get('shared-account') + assert.equal(saved.unified['5h'].utilization, 0.81) + assert.equal(saved.unified.account_tier, 'max') + assert.equal(saved.unified.usage_has_fable, true) + assert.equal(JSON.parse(fs.readFileSync(path.join(project, 'vms/vm-a.json'))).claude.account_tier, 'max') +}) + +test('header observations preserve unrelated official and model fields from another replica', async (t) => { + const { left, right } = await pair(t) + right.ingestOAuthUsage('shared-account', { ok: true, seven_day: { utilization: 61 } }) + right.markModelUnsupported('shared-account', 'claude-fable-5') + await right.flush() + left.ingestHeaders('shared-account', { 'anthropic-ratelimit-unified-5h-utilization': '0.2' }) + await left.flush() + const saved = await left.persistence.get('shared-account') + assert.equal(saved.unified['7d'].utilization, 0.61) + assert.equal(saved.unified.headers['5h'].utilization, 0.2) + assert.ok(saved.unified.model_denied_until['claude-fable-5'] > Date.now()) +}) + +test('a stale health check cannot replace remote header windows with an empty initialized object', async (t) => { + const { left, right } = await pair(t) + left.ingestHeaders('shared-account', { 'anthropic-ratelimit-unified-5h-utilization': '0.81' }) + await left.flush() + right.ingestHeaders('shared-account', {}, null, { countRequest: false }) + await right.flush() + assert.equal((await left.persistence.get('shared-account')).unified.headers['5h'].utilization, 0.81) +}) + +test('a stale test-chat or rate-limited probe preserves the successful usage probe from another replica', async (t) => { + const { left, right } = await pair(t) + left.recordLastProbe('shared-account', { ok: true, source: 'vm-oauth-usage' }) + await left.flush() + right.recordLastProbe('shared-account', { ok: false, source: 'test-chat', error: 'expired', status: 401 }) + await right.flush() + let saved = await left.persistence.get('shared-account') + assert.equal(saved.unified.last_probe.ok, true) + assert.equal(saved.unified.last_probe.source, 'vm-oauth-usage') + // A third, independently loaded stale cache must not synthesize a failed last_probe on 429. + const third = await new AccountQuota({ db: right.db, config: {} }).init() + left.recordLastProbe('shared-account', { ok: true, source: 'official-cc-usage' }) + await left.flush() + third.ingestOAuthUsage('shared-account', { usage_status: 429, rate_limited: true }) + await third.flush() + saved = await left.persistence.get('shared-account') + assert.equal(saved.unified.last_probe.ok, true) + assert.equal(saved.unified.last_probe.source, 'official-cc-usage') + assert.ok(saved.unified.usage_rate_limited_until) +}) + +test('an in-flight database result does not publish over a later synchronous local mutation', async (t) => { + const { left } = await pair(t) + const apply = left.persistence.applyChanges.bind(left.persistence) + let entered + let resume + let nextEntered + let nextResume + const started = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + resume = resolve + }) + const nextStarted = new Promise((resolve) => { + nextEntered = resolve + }) + const nextGate = new Promise((resolve) => { + nextResume = resolve + }) + let first = true + left.persistence.applyChanges = async (...args) => { + if (!first) { + nextEntered() + await nextGate + } + const saved = await apply(...args) + if (first) { + first = false + entered() + await gate + } + return saved + } + left.ingestHeaders('shared-account', {}, { input_tokens: 3 }) + await started + left.setMaxConcurrency('shared-account', 17, { override: true }) + left.ingestHeaders('shared-account', {}, { input_tokens: 7 }) + resume() + await nextStarted + assert.equal(left.repo.get('shared-account').max_concurrency, 17) + assert.equal(left.repo.get('shared-account').tokens_in, 10) + nextResume() + await left.flush() + const saved = await left.persistence.get('shared-account') + assert.equal(saved.max_concurrency, 17) + assert.equal(saved.tokens_in, 10) + assert.equal(left.repo.get('shared-account').max_concurrency, 17) + assert.equal(left.repo.get('shared-account').tokens_in, 10) +}) From 3542a38840dd1945856f26bf3091cb2177ef69c0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:22:46 +0800 Subject: [PATCH 109/473] test: specify cross-replica account cache refresh --- test/unit/shared-account-quota.test.mjs | 79 +++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/test/unit/shared-account-quota.test.mjs b/test/unit/shared-account-quota.test.mjs index c8fcc0be8..188387671 100644 --- a/test/unit/shared-account-quota.test.mjs +++ b/test/unit/shared-account-quota.test.mjs @@ -315,3 +315,82 @@ test('an in-flight database result does not publish over a later synchronous loc assert.equal(left.repo.get('shared-account').max_concurrency, 17) assert.equal(left.repo.get('shared-account').tokens_in, 10) }) + +test('refresh reads remote account management changes without replacing transient state or allocations', async (t) => { + const { left, right } = await pair(t) + left.inflight.set('shared-account', 2) + const sessions = left.sessions + const allocations = left.allocations + const history = [{ source: 'synthetic-existing-history' }] + allocations.set('shared-account', history) + right.setMaxConcurrency('shared-account', 19, { override: true }) + right.setMaxRpm('shared-account', 321, { override: true }) + await right.flush() + assert.notEqual(left.repo.get('shared-account').max_concurrency, 19) + await left.refresh() + assert.equal(left.repo.get('shared-account').max_concurrency, 19) + assert.equal(left.repo.get('shared-account').max_rpm, 321) + assert.equal(left.inflight.get('shared-account'), 2) + assert.equal(left.sessions, sessions) + assert.equal(left.allocations, allocations) + assert.equal(left.allocations.get('shared-account'), history) +}) + +test('a delayed refresh cannot overwrite a local write started after the read', async (t) => { + const { left } = await pair(t) + const list = left.persistence.list.bind(left.persistence) + let entered + let resume + const started = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + resume = resolve + }) + left.persistence.list = async () => { + const rows = await list() + entered() + await gate + return rows + } + const refreshing = left.refresh() + await started + left.setMaxConcurrency('shared-account', 23, { override: true }) + left.ingestHeaders('shared-account', {}, { input_tokens: 7 }) + await left.flush() + resume() + await refreshing + assert.equal(left.repo.get('shared-account').max_concurrency, 23) + assert.equal(left.repo.get('shared-account').tokens_in, 7) +}) + +test('an older refresh finishing late cannot replace a newer committed remote snapshot', async (t) => { + const { left, right } = await pair(t) + const list = left.persistence.list.bind(left.persistence) + let entered + let resume + const started = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + resume = resolve + }) + let first = true + left.persistence.list = async () => { + const rows = await list() + if (first) { + first = false + entered() + await gate + } + return rows + } + const older = left.refresh() + await started + right.setMaxConcurrency('shared-account', 29, { override: true }) + await right.flush() + await left.refresh() + resume() + await older + assert.equal(left.repo.get('shared-account').max_concurrency, 29) +}) From 623d489273f6e5a0e47b85cfc356f90ada58abf0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:23:08 +0800 Subject: [PATCH 110/473] fix: refresh shared account configuration without stale publication --- src/lib/pool/account-quota.mjs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/lib/pool/account-quota.mjs b/src/lib/pool/account-quota.mjs index 58db854b7..4b696ef91 100644 --- a/src/lib/pool/account-quota.mjs +++ b/src/lib/pool/account-quota.mjs @@ -66,6 +66,8 @@ export class AccountQuota { this.writes = this.db.schedulingWriteQueue ||= new WriteQueue(this.db) this.rows = new Map() this.revisions = new Map() + this.refreshStarted = 0 + this.refreshPublished = 0 this.allocations = new Map() this.seedAccounts = accounts || [] this.repo = this._workingSet() @@ -107,6 +109,23 @@ export class AccountQuota { return await this.init({ seed: false }) } + /** Refresh committed account rows without replacing transient state or allocation history. */ + async refresh() { + await this.flush() + const refresh = ++this.refreshStarted + const revisions = new Map(this.revisions) + const rows = await this.persistence.list() + if (refresh < this.refreshPublished) return + this.refreshPublished = refresh + const present = new Set(rows.map((row) => row.account_id)) + for (const row of rows) { + if (this.revisions.get(row.account_id) === revisions.get(row.account_id)) this.rows.set(row.account_id, row) + } + for (const id of this.rows.keys()) { + if (!present.has(id) && this.revisions.get(id) === revisions.get(id)) this.rows.delete(id) + } + } + flush() { return this.writes.flush() } From 8904775b95696b2e2c457f5b5fd042b077846326 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:23:38 +0800 Subject: [PATCH 111/473] fix: persist proxy mutations atomically across replicas --- .../009_proxy_revisions.sql | 1 + src/lib/db/repos/proxies-repo.mjs | 97 +++- src/lib/vm/proxy-pool.mjs | 413 ++++++++++++++---- test/unit/local-egress-per-node.test.mjs | 87 ++-- test/unit/panel-api-proxy.test.mjs | 24 +- test/unit/proxy-geo.test.mjs | 38 +- test/unit/proxy-ipv6.test.mjs | 37 +- test/unit/proxy-pool-bind.test.mjs | 63 +-- test/unit/proxy-pool-import.test.mjs | 8 +- test/unit/proxy-pool-shared.test.mjs | 256 +++++++++++ test/unit/proxy-pool-update.test.mjs | 76 ++-- test/unit/proxy-resolve.test.mjs | 10 +- 12 files changed, 850 insertions(+), 260 deletions(-) create mode 100644 src/lib/db/postgres-migrations/009_proxy_revisions.sql diff --git a/src/lib/db/postgres-migrations/009_proxy_revisions.sql b/src/lib/db/postgres-migrations/009_proxy_revisions.sql new file mode 100644 index 000000000..21c26fce5 --- /dev/null +++ b/src/lib/db/postgres-migrations/009_proxy_revisions.sql @@ -0,0 +1 @@ +ALTER TABLE proxies ADD COLUMN config_revision bigint NOT NULL DEFAULT 0; diff --git a/src/lib/db/repos/proxies-repo.mjs b/src/lib/db/repos/proxies-repo.mjs index 1af1bb305..794fa0dce 100644 --- a/src/lib/db/repos/proxies-repo.mjs +++ b/src/lib/db/repos/proxies-repo.mjs @@ -1,7 +1,7 @@ /** * proxies repository — SOCKS5 pool rows + pool config in settings. - * The ProxyPool keeps its working set in memory (probe loop mutates it); - * this repo is the persistence surface: loadAll / replaceAll. + * Runtime mutations update only the intended rows and columns. replaceAll is + * retained for the offline legacy importer, never for pool health persistence. */ import { getDb, withTransaction } from '../database.mjs' @@ -50,6 +50,22 @@ const COLUMNS = [ 'domain_forward', ] +export const PROXY_COLUMNS = COLUMNS +export const PROXY_HEALTH_COLUMNS = COLUMNS.filter( + (column) => + ['status', 'enabled', 'consecutive_failures', 'latency_ms', 'last_probe_at', 'last_error'].includes(column) || + column.startsWith('geo_'), +) + +function valueFor(proxy, column) { + if (column === 'enabled') return proxy.enabled === false ? 0 : 1 + if (column === 'domain_forward') return proxy.domain_forward ? 1 : 0 + if (column === 'protocol') return proxy.protocol ?? proxy.scheme ?? 'socks5' + if (column === 'name') + return proxy.name ?? (proxy.host && proxy.port != null ? `${proxy.host}:${proxy.port}` : proxy.id) + return proxy[column] ?? null +} + function rowToProxy(row) { if (!row) return null // ProxyPool still speaks `scheme`; the column is sub2api's `protocol`. @@ -72,6 +88,71 @@ export class ProxiesRepo { return (await this._all.all()).map(rowToProxy) } + /** One statement provides a consistent committed config/rows snapshot. */ + async snapshot() { + const row = await this.db + .prepare(`SELECT + (SELECT value FROM settings WHERE key = $1) AS config, + COALESCE((SELECT json_agg(p ORDER BY created_at, id) FROM proxies p), '[]'::json) AS proxies`) + .get(CONFIG_KEY) + let config = {} + try { + config = row.config ? JSON.parse(row.config) : {} + } catch {} + return { config, proxies: row.proxies.map(rowToProxy) } + } + + async lockMutations() { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ + this.db.schema, + 'proxy-pool:mutations', + ]) + } + + async getById(id, { forUpdate = false } = {}) { + return rowToProxy( + await this.db.prepare(`SELECT * FROM proxies WHERE id = $1${forUpdate ? ' FOR UPDATE' : ''}`).get(id), + ) + } + + async insert(proxy) { + await this._insert.run(...COLUMNS.map((column) => valueFor(proxy, column))) + return this.getById(proxy.id) + } + + async patchConfig(id, patch) { + return this._patch(id, patch, { bumpRevision: true }) + } + + async patchHealth(id, patch, expectedRevision) { + const allowed = Object.fromEntries( + Object.entries(patch).filter(([column]) => PROXY_HEALTH_COLUMNS.includes(column)), + ) + return this._patch(id, allowed, { expectedRevision }) + } + + async _patch(id, patch, { bumpRevision = false, expectedRevision } = {}) { + const columns = Object.keys(patch).filter((column) => column !== 'id' && COLUMNS.includes(column)) + if (!columns.length) return this.getById(id) + const assignments = columns.map((column, i) => `${column} = $${i + 1}`) + if (bumpRevision) assignments.push('config_revision = config_revision + 1') + const params = [...columns.map((column) => valueFor(patch, column)), id] + let condition = `id = $${params.length}` + if (expectedRevision !== undefined) { + params.push(expectedRevision) + condition += ` AND config_revision = $${params.length}` + } + return rowToProxy( + await this.db + .prepare(`UPDATE proxies SET ${assignments.join(', ')} WHERE ${condition} RETURNING *`) + .get(...params), + ) + } + + async remove(id) { + return (await this.db.prepare('DELETE FROM proxies WHERE id = $1').run(id)).changes > 0 + } + async replaceAll(proxies = []) { await withTransaction(this.db, async () => { await this._clear.run() @@ -96,4 +177,16 @@ export class ProxiesRepo { async setConfig(config) { return await this.settings.set(CONFIG_KEY, config) } + + async patchPoolConfig(patch) { + const row = await this.db + .prepare(`INSERT INTO settings (key, value, updated_at) VALUES ($1, $2, $3) + ON CONFLICT (key) DO UPDATE SET + value = (settings.value::jsonb || excluded.value::jsonb)::text, + updated_at = excluded.updated_at RETURNING value`) + .get(CONFIG_KEY, JSON.stringify(patch), new Date().toISOString()) + const config = JSON.parse(row.value) + this.db.afterCommit(() => this.db.settingsState?.set(CONFIG_KEY, config)) + return config + } } diff --git a/src/lib/vm/proxy-pool.mjs b/src/lib/vm/proxy-pool.mjs index a3c465113..429ad9453 100644 --- a/src/lib/vm/proxy-pool.mjs +++ b/src/lib/vm/proxy-pool.mjs @@ -1,5 +1,5 @@ /** - * SOCKS5 Proxy Pool (SQLite-backed) + * SOCKS5 Proxy Pool (PostgreSQL-backed) * - Bulk import * - 1 proxy ↔ up to N VMs (`config.bind_limit`, default 5) * - Auto-assign a proxy with remaining capacity on VM create @@ -7,14 +7,13 @@ * - On failure: disable proxy + disable bound VM scheduling * * Persistence: `proxies` table + `settings.proxy_pool_config`. - * Working set stays in memory (probe loop mutates it); save() writes through. + * PostgreSQL is authoritative; synchronous accessors expose an explicit runtime cache. */ import { EventEmitter } from 'node:events' import net from 'node:net' import crypto from 'node:crypto' -import { resolveStoreDb } from '../db/database.mjs' -import { ProxiesRepo } from '../db/repos/proxies-repo.mjs' -import { WriteQueue } from '../pool/write-queue.mjs' +import { resolveStoreDb, withTransaction } from '../db/database.mjs' +import { ProxiesRepo, PROXY_COLUMNS, PROXY_HEALTH_COLUMNS } from '../db/repos/proxies-repo.mjs' import { canBindProxyToVm, normalizeOwnerId, proxyOwnerId, vmOwnerId } from '../admin/resource-owner.mjs' import { validTimezone } from '../core/timezone.mjs' import { lookupProxyGeo, lookupProxyGeoV6 } from './proxy-geo.mjs' @@ -342,54 +341,219 @@ export class ProxyPool extends EventEmitter { this._probeSockets = new Map() // Node exits are not pool rows: px-local's geo is the control plane's, a node slot leaves elsewhere. this._exitGeo = new Map() - this.writes = new WriteQueue(this.db) + this._operations = new Set() + this._cacheVersion = 0 + this._refreshIssued = 0 + this._refreshPublished = 0 + this.db.pendingWriteQueues ||= new Set() this.db.pendingWriteQueues.add(this) } - /** Re-read working set from DB (startup + post-restore). */ + _emitChange() { + if (!this._mutation) this.emit('change') + } + + _publish(snapshot) { + const previous = new Map(this.state.proxies.map((proxy) => [proxy.id, proxy])) + const proxies = snapshot.proxies.map((record) => { + const proxy = hydrateProxy(record) + const existing = previous.get(proxy.id) + return existing ? Object.assign(existing, proxy) : proxy + }) + this.state = { config: { ...DEFAULT_CONFIG, ...snapshot.config }, proxies } + this._savedState = structuredClone(this.state) + } + + /** Refresh committed PostgreSQL state without waiting for network probes. */ + async refresh() { + const issued = ++this._refreshIssued + const version = this._cacheVersion + const snapshot = await this.repo.snapshot() + if (issued > this._refreshPublished && version === this._cacheVersion) { + this._refreshPublished = issued + this._publish(snapshot) + } + return this + } + async load() { - for (const queue of this.db.pendingWriteQueues || []) await queue.flush() await this.flush() - this.state.config = { ...DEFAULT_CONFIG, ...((await this.repo.getConfig({})) || {}) } - this.state.proxies = (await this.repo.loadAll()).map(hydrateProxy) - return this + return this.refresh() } async init() { - return await this.load() + return this.load() } - async reload() { - return await this.load() + return this.load() } async rebind(db) { await this.flush() + this.db.pendingWriteQueues?.delete(this) this.db = db this.repo = new ProxiesRepo(db) - return await this.load() + db.pendingWriteQueues ||= new Set() + db.pendingWriteQueues.add(this) + return this.load() } async flush() { while (this._probing) await new Promise((resolve) => setTimeout(resolve, 10)) - await this.writes.flush() + while (this._operations.size) await Promise.all([...this._operations]) } + _track(operation) { + this._operations.add(operation) + operation.finally(() => this._operations.delete(operation)).catch(() => {}) + return operation + } + + _stored(proxy) { + return { ...proxy, protocol: proxy.protocol ?? proxy.scheme, bound_vm_id: encodeBoundVmIds(boundVmIdsOf(proxy)) } + } + + _changes(before, after, columns = PROXY_COLUMNS) { + return Object.fromEntries( + columns + .filter( + (column) => + column !== 'id' && JSON.stringify(before[column] ?? null) !== JSON.stringify(after[column] ?? null), + ) + .map((column) => [column, after[column] ?? null]), + ) + } + + /** Each control mutation starts from PG and writes only its changed rows/fields. */ + _mutate(change) { + const operation = withTransaction(this.db, async () => { + await this.repo.lockMutations() + const snapshot = await this.repo.snapshot() + const working = { config: { ...DEFAULT_CONFIG, ...snapshot.config }, proxies: snapshot.proxies.map(hydrateProxy) } + const before = structuredClone(working) + const cached = this.state + const context = { callbacks: [], restartScheduler: false } + let result + this.state = working + this._mutation = context + try { + result = change() + } finally { + this.state = cached + this._mutation = null + } + if (result?.ok === false) return result + const previous = new Map(before.proxies.map((proxy) => [proxy.id, proxy])) + for (const proxy of working.proxies) { + const old = previous.get(proxy.id) + let row + if (!old) row = await this.repo.insert(this._stored(proxy)) + else { + const patch = this._changes(this._stored(old), this._stored(proxy)) + if (Object.keys(patch).length) row = await this.repo.patchConfig(proxy.id, patch) + } + if (row) Object.assign(proxy, hydrateProxy(row)) + previous.delete(proxy.id) + } + for (const id of previous.keys()) await this.repo.remove(id) + const configPatch = this._changes(before.config, working.config, Object.keys(DEFAULT_CONFIG)) + if (Object.keys(configPatch).length) { + working.config = { ...DEFAULT_CONFIG, ...(await this.repo.patchPoolConfig(configPatch)) } + } + this.db.afterCommit(() => { + this._cacheVersion++ + this._publish(working) + for (const callback of context.callbacks) callback() + this._emitChange() + if (context.restartScheduler) this.restartScheduler() + }) + return result + }) + return this._track(operation) + } + + /** Compatibility for explicit state edits; production operations use the methods below. */ save() { - if (this._saveDeferred) { - this._savePending = true - return - } - const config = structuredClone(this.state.config) - const proxies = this.state.proxies.map((p) => ({ - ...structuredClone(p), - bound_vm_id: encodeBoundVmIds(boundVmIdsOf(p)), - })) - this.writes.enqueue(async () => { - await this.repo.setConfig(config) - await this.repo.replaceAll(proxies) + if (this._mutation) return + const desired = structuredClone(this.state) + const baseline = structuredClone(this._savedState || { config: DEFAULT_CONFIG, proxies: [] }) + return this._mutate(() => { + Object.assign(this.state.config, this._changes(baseline.config, desired.config, Object.keys(DEFAULT_CONFIG))) + const old = new Map(baseline.proxies.map((proxy) => [proxy.id, proxy])) + const desiredIds = new Set(desired.proxies.map((proxy) => proxy.id)) + for (const proxy of desired.proxies) { + const previous = old.get(proxy.id) + const current = this.state.proxies.find((row) => row.id === proxy.id) + if (!previous) { + if (!current) this.state.proxies.push(proxy) + continue + } + if (!current) continue + const patch = this._changes(this._stored(previous), this._stored(proxy)) + if (Object.keys(patch).length && previous.config_revision !== current.config_revision) { + throw Object.assign(new Error('proxy changed'), { code: 'proxy_changed' }) + } + Object.assign(current, patch) + if (Object.hasOwn(patch, 'bound_vm_id')) setBoundVmIds(current, patch.bound_vm_id) + } + this.state.proxies = this.state.proxies.filter((proxy) => !old.has(proxy.id) || desiredIds.has(proxy.id)) + return { ok: true } }) - this.emit('change') + } + + async ensureLocal(...args) { + return this._mutate(() => this._ensureLocal(...args)) + } + async ensureLocalForNode(...args) { + return this._mutate(() => this._ensureLocalForNode(...args)) + } + async syncLocalNodes(...args) { + return this._mutate(() => this._syncLocalNodes(...args)) + } + async resolveLocalTarget(...args) { + return this._mutate(() => this._resolveLocalTarget(...args)) + } + async adoptNodeLocalBindings(...args) { + return this._mutate(() => this._collectNodeLocalBindings(...args)) + } + async ensureSocks(...args) { + return this._mutate(() => this._ensureSocks(...args)) + } + async importLines(...args) { + return this._mutate(() => this._importLines(...args)) + } + async importParsed(...args) { + return this._mutate(() => this._importParsed(...args)) + } + async ensureBoundToVm(...args) { + return this._mutate(() => this._ensureBoundToVm(...args)) + } + async allocateForVm(...args) { + return this._mutate(() => this._allocateForVm(...args)) + } + async bind(...args) { + return this._mutate(() => this._bind(...args)) + } + async unbind(...args) { + return this._mutate(() => this._unbind(...args)) + } + async unbindVm(...args) { + return this._mutate(() => this._unbindVm(...args)) + } + async setEnabled(...args) { + return this._mutate(() => this._setEnabled(...args)) + } + async remove(...args) { + return this._mutate(() => this._remove(...args)) + } + async update(...args) { + return this._mutate(() => this._update(...args)) + } + async updateConfig(...args) { + return this._mutate(() => this._updateConfig(...args)) + } + async reportRuntimeFailure(...args) { + return this._mutate(() => this._reportRuntimeFailure(...args)) } /** @@ -507,12 +671,12 @@ export class ProxyPool extends EventEmitter { } /** The control plane's local row (`px-local`). */ - ensureLocal() { - return this.ensureLocalForNode(null) + _ensureLocal() { + return this._ensureLocalForNode(null) } /** One local row per VPS: `px-local` for the control plane, `px-local-` per node. */ - ensureLocalForNode(nodeId = null) { + _ensureLocalForNode(nodeId = null) { const id = localEgressIdForNode(nodeId) const existing = this.state.proxies.find((p) => p.id === id) if (existing) return { ok: true, created: false, proxy: this.publicProxy(existing) } @@ -527,13 +691,13 @@ export class ProxyPool extends EventEmitter { * Keep node local rows in step with the cluster: a row per node once local * egress is in use, and drop rows of removed nodes that hold no slot. */ - syncLocalNodes(nodeIds = []) { + _syncLocalNodes(nodeIds = []) { if (!this.state.proxies.some((p) => p.id === LOCAL_EGRESS_ID)) return { created: [], removed: [] } const wanted = new Set(nodeIds.map((id) => localEgressIdForNode(id)).filter((id) => id !== LOCAL_EGRESS_ID)) const created = [] for (const id of wanted) { if (this.state.proxies.some((p) => p.id === id)) continue - this.ensureLocalForNode(localEgressNodeId(id)) + this._ensureLocalForNode(localEgressNodeId(id)) created.push(id) } const removed = this.state.proxies @@ -543,7 +707,7 @@ export class ProxyPool extends EventEmitter { this.state.proxies = this.state.proxies.filter((p) => !removed.includes(p.id)) this.save() } - if (created.length || removed.length) this.emit('change') + if (created.length || removed.length) this._emitChange() return { created, removed } } @@ -557,12 +721,13 @@ export class ProxyPool extends EventEmitter { /** Re-probe one node's local row (its SSH link just changed state). */ async probeLocalNode(nodeId) { + await this.refresh() const p = this.state.proxies.find((x) => x.id === localEgressIdForNode(nodeId) && localEgressNodeId(x.id)) // An operator-disabled row stays out of automatic probes, as in probeAll. if (!p || !p.enabled) return null - const result = await this.probeOne(p) - this._applyProbeResult(p, result) - this.save() + const observed = structuredClone(p) + const result = await this.probeOne(observed) + await this._persistObservation(observed, (current) => this._applyProbeResultInner(current, result)) return result } @@ -576,13 +741,13 @@ export class ProxyPool extends EventEmitter { * `nodeId` (null = control plane). The generic px-local resolves to the * slot's own VPS; another machine's local row is refused. */ - resolveLocalTarget(proxyId, nodeId) { + _resolveLocalTarget(proxyId, nodeId) { const target = localEgressIdForNode(nodeId || null) if (proxyId !== target && proxyId !== LOCAL_EGRESS_ID) { return { ok: false, error: 'local_proxy_node_mismatch', expected: target } } const targetNode = localEgressNodeId(target) - if (targetNode) this.ensureLocalForNode(targetNode) + if (targetNode) this._ensureLocalForNode(targetNode) return { ok: true, id: target } } @@ -591,18 +756,9 @@ export class ProxyPool extends EventEmitter { * px-local before per-node rows). Returns the moves so the caller can * rewrite each slot's recorded proxy. */ - adoptNodeLocalBindings() { + _collectNodeLocalBindings() { const moves = [] - this._saveDeferred = true - try { - this._adoptNodeLocalBindings(moves) - } finally { - this._saveDeferred = false - if (this._savePending) { - this._savePending = false - this.save() - } - } + this._adoptNodeLocalBindings(moves) return moves } @@ -613,7 +769,7 @@ export class ProxyPool extends EventEmitter { if (this.vmNodeOf && this._vmNodeOrUndefined(vmId) === undefined) continue const target = this.localTargetForVm(vmId) if (target === p.id) continue - const bound = this.bind(target, vmId, { force: true }) + const bound = this._bind(target, vmId, { force: true }) if (bound.ok) moves.push({ vm_id: vmId, from: p.id, to: target }) } } @@ -636,12 +792,12 @@ export class ProxyPool extends EventEmitter { * Return the SOCKS row with this host/port/username/password, creating it * when absent. Same credentials must not allocate a second id. */ - ensureSocks(fields = {}, { ownerUserId = null, label = null } = {}) { + _ensureSocks(fields = {}, { ownerUserId = null, label = null } = {}) { const parsed = parseSocks5Fields(fields) if (!parsed) return { ok: false, error: 'invalid_proxy' } const found = this._findSocksRow(parsed) if (found) return { ok: true, created: false, proxy: this._withAuth(found) } - const imported = this.importParsed([parsed], { ownerUserId }) + const imported = this._importParsed([parsed], { ownerUserId }) const id = imported.items[0]?.id const row = this.state.proxies.find((p) => p.id === id) if (!row) return { ok: false, error: 'proxy_import_failed' } @@ -653,7 +809,7 @@ export class ProxyPool extends EventEmitter { return { ok: true, created: true, proxy: this._withAuth(row) } } - importLines(text, extra = {}) { + _importLines(text, extra = {}) { const parsed = [] for (const line of String(text || '').split(/\r?\n/)) { if (!String(line || '').trim()) continue @@ -667,10 +823,10 @@ export class ProxyPool extends EventEmitter { if (extra.host || extra.port) { parsed.push(parseSocks5Fields(extra) || { __invalid: `${extra.host || ''}:${extra.port || ''}` }) } - return this.importParsed(parsed, extra) + return this._importParsed(parsed, extra) } - importParsed(records = [], extra = {}) { + _importParsed(records = [], extra = {}) { const added = [] const skipped = [] // Credentials can select distinct proxies at the same endpoint. Encode a @@ -724,7 +880,7 @@ export class ProxyPool extends EventEmitter { * Restore pool↔VM binding after start/restart. * Prefer an existing bind, then the VM's last proxy id, then a free allocate. */ - ensureBoundToVm(vmId, preferredId = null) { + _ensureBoundToVm(vmId, preferredId = null) { if (!vmId) return null if (this.state.proxies.some((p) => proxyHasVm(p, vmId) && proxyBlockedReason(p, this.state.config.ipv6_enabled))) return null @@ -733,14 +889,14 @@ export class ProxyPool extends EventEmitter { if (preferredId) { const preferred = this.state.proxies.find((p) => p.id === preferredId) if (proxyBlockedReason(preferred, this.state.config.ipv6_enabled)) return null - const bound = this.bind(preferredId, vmId) + const bound = this._bind(preferredId, vmId) if (bound.ok) return this.getProxyForVm(vmId) } - return this.allocateForVm(vmId) + return this._allocateForVm(vmId) } /** Allocate one healthy proxy with remaining capacity and bind to vmId */ - allocateForVm(vmId, { ownerUserId = null, role = 'admin' } = {}) { + _allocateForVm(vmId, { ownerUserId = null, role = 'admin' } = {}) { if (!vmId) return null if (this.state.proxies.some((p) => proxyHasVm(p, vmId) && proxyBlockedReason(p, this.state.config.ipv6_enabled))) return null @@ -753,7 +909,7 @@ export class ProxyPool extends EventEmitter { const owner = normalizeOwnerId(ownerUserId) const localTarget = this.localTargetForVm(vmId) if (localTarget !== LOCAL_EGRESS_ID && this.state.proxies.some((p) => p.id === LOCAL_EGRESS_ID)) { - this.ensureLocalForNode(localEgressNodeId(localTarget)) + this._ensureLocalForNode(localEgressNodeId(localTarget)) } const candidates = this.state.proxies.filter((p) => { if (isLocalEgressProxy(p) && p.id !== localTarget) return false @@ -769,16 +925,16 @@ export class ProxyPool extends EventEmitter { }) const pick = candidates[0] if (!pick) return null - const bound = this.bind(pick.id, vmId, { role }) + const bound = this._bind(pick.id, vmId, { role }) return bound.ok ? bound.proxy : null } - bind(proxyId, vmId, { force = false } = {}) { + _bind(proxyId, vmId, { force = false } = {}) { const vm = String(vmId || '').trim() if (isLocalEgressId(proxyId) && vm) { // Local egress is the slot's own VPS route: a generic px-local request lands // on the slot's node row; another machine's local row is refused. - const resolved = this.resolveLocalTarget(proxyId, this._vmNodeOrUndefined(vm) || null) + const resolved = this._resolveLocalTarget(proxyId, this._vmNodeOrUndefined(vm) || null) if (!resolved.ok) return resolved proxyId = resolved.id } @@ -812,7 +968,7 @@ export class ProxyPool extends EventEmitter { return { ok: true, proxy: this.publicProxy(p) } } - unbind(proxyId, vmId = null) { + _unbind(proxyId, vmId = null) { const p = this.state.proxies.find((x) => x.id === proxyId) if (!p) return { ok: false, error: 'proxy_not_found' } const ids = boundVmIdsOf(p) @@ -823,7 +979,7 @@ export class ProxyPool extends EventEmitter { return { ok: true, proxy: this.publicProxy(p), unbound_vm_ids: removed } } - unbindVm(vmId) { + _unbindVm(vmId) { const vm = String(vmId || '').trim() if (!vm) return for (const p of this.state.proxies) { @@ -837,7 +993,7 @@ export class ProxyPool extends EventEmitter { this.save() } - setEnabled(proxyId, enabled) { + _setEnabled(proxyId, enabled) { const p = this.state.proxies.find((x) => x.id === proxyId) if (!p) return { ok: false, error: 'proxy_not_found' } p.enabled = !!enabled @@ -853,7 +1009,7 @@ export class ProxyPool extends EventEmitter { return { ok: true, proxy: this.publicProxy(p) } } - remove(proxyId) { + _remove(proxyId) { const idx = this.state.proxies.findIndex((x) => x.id === proxyId) if (idx < 0) return { ok: false, error: 'proxy_not_found' } const [p] = this.state.proxies.splice(idx, 1) @@ -871,7 +1027,7 @@ export class ProxyPool extends EventEmitter { * The merged record goes through socks5Record() so this shares the exact * validation and normalization the import path uses. */ - update(proxyId, patch = {}) { + _update(proxyId, patch = {}) { const p = this.state.proxies.find((x) => x.id === proxyId) if (!p) return { ok: false, error: 'proxy_not_found' } const has = (k) => Object.prototype.hasOwnProperty.call(patch, k) @@ -929,7 +1085,7 @@ export class ProxyPool extends EventEmitter { return { ok: true, proxy: this.publicProxy(p), connection_changed: true, domain_forward_changed: domainChanged } } - updateConfig(patch = {}) { + _updateConfig(patch = {}) { const allowed = [5, 10, 30, 60] if (Object.hasOwn(patch, 'dns_disable_svcb_https') && typeof patch.dns_disable_svcb_https !== 'boolean') { return { ok: false, error: 'invalid_dns_disable_svcb_https' } @@ -979,9 +1135,11 @@ export class ProxyPool extends EventEmitter { if (patch.ipv6_enabled != null) this.state.config.ipv6_enabled = patch.ipv6_enabled this.save() if (patch.ipv6_enabled === false) { - for (const [socket, proxy] of this._probeSockets) { - if (socksProxyFamily(proxy) === 6) socket.destroy() - } + this._mutation.callbacks.push(() => { + for (const [socket, proxy] of this._probeSockets) { + if (socksProxyFamily(proxy) === 6) socket.destroy() + } + }) } this.restartScheduler() return { ok: true, config: this.state.config } @@ -995,7 +1153,7 @@ export class ProxyPool extends EventEmitter { * Runtime SOCKS5 failure from a live request. * When disconnect_on_error is off this is a no-op (existing cooldown/failover stays). */ - reportRuntimeFailure(vmId, error = 'runtime_socks_failure') { + _reportRuntimeFailure(vmId, error = 'runtime_socks_failure') { if (!this.disconnectOnErrorEnabled()) { return { ok: true, skipped: true, reason: 'disconnect_on_error_disabled' } } @@ -1019,6 +1177,11 @@ export class ProxyPool extends EventEmitter { } _cascadeDisconnectVm(proxy, reason) { + if (this._mutation) { + const committed = structuredClone(proxy) + this._mutation.callbacks.push(() => this._cascadeDisconnectVm(committed, reason)) + return + } const cb = this.onDisconnectVm || this.onDisableVm if (typeof cb !== 'function') return for (const vmId of boundVmIdsOf(proxy)) { @@ -1031,6 +1194,11 @@ export class ProxyPool extends EventEmitter { } _cascadeDisableVm(proxy, reason) { + if (this._mutation) { + const committed = structuredClone(proxy) + this._mutation.callbacks.push(() => this._cascadeDisableVm(committed, reason)) + return + } if (typeof this.onDisableVm !== 'function') return for (const vmId of boundVmIdsOf(proxy)) { try { @@ -1042,6 +1210,11 @@ export class ProxyPool extends EventEmitter { } _cascadeEnableVm(proxy, reason) { + if (this._mutation) { + const committed = structuredClone(proxy) + this._mutation.callbacks.push(() => this._cascadeEnableVm(committed, reason)) + return + } if (typeof this.onEnableVm !== 'function') return for (const vmId of boundVmIdsOf(proxy)) { try { @@ -1083,6 +1256,10 @@ export class ProxyPool extends EventEmitter { if (typeof this.egressCheck !== 'function') return { ...socks, scope: 'socks' } let eg = this.egressCheck(proxy) if (!eg?.ok && typeof this.repairEgress === 'function') { + const current = await this.repo.getById(proxy.id) + if (!current || current.config_revision !== proxy.config_revision) { + return { ok: false, scope: 'pending', error: 'proxy_changed', latency_ms: socks.latency_ms } + } try { await this.repairEgress(proxy) } catch { @@ -1163,12 +1340,13 @@ export class ProxyPool extends EventEmitter { } async probeById(proxyId) { + await this.refresh() const p = this.state.proxies.find((x) => x.id === proxyId) if (!p) return { ok: false, error: 'proxy_not_found' } - const result = await this.probeOne(p) - this._applyProbeResult(p, result) - this.save() - return { ok: true, proxy: this.publicProxy(p), probe: result } + const observed = structuredClone(p) + const result = await this.probeOne(observed) + const current = await this._persistObservation(observed, (row) => this._applyProbeResultInner(row, result)) + return { ok: true, proxy: this.publicProxy(current || p), probe: result, ...(current ? {} : { stale: true }) } } async probeAll({ onlyEnabled = true } = {}) { @@ -1176,6 +1354,7 @@ export class ProxyPool extends EventEmitter { this._probing = true const results = [] try { + await this.refresh() const list = this.state.proxies.filter((p) => { if (proxyBlockedReason(p, this.state.config.ipv6_enabled)) return false if (!onlyEnabled) return true @@ -1183,11 +1362,17 @@ export class ProxyPool extends EventEmitter { return p.status === 'fail' || (p.status === 'dead' && p.last_error) }) for (const p of list) { - const result = await this.probeOne(p) - this._applyProbeResult(p, result) - results.push({ id: p.id, ...result, status: p.status, enabled: p.enabled }) + const observed = structuredClone(p) + const result = await this.probeOne(observed) + const current = await this._persistObservation(observed, (row) => this._applyProbeResultInner(row, result)) + results.push({ + id: p.id, + ...result, + status: current?.status ?? p.status, + enabled: current?.enabled ?? p.enabled, + ...(current ? {} : { stale: true }), + }) } - this.save() return { ok: true, total: results.length, results } } finally { this._probing = false @@ -1205,8 +1390,10 @@ export class ProxyPool extends EventEmitter { * hit is returned untouched so bind-time detection stays cheap. */ async detectGeo(proxyId, { force = false } = {}) { + await this.refresh() const p = this.state.proxies.find((x) => x.id === proxyId) if (!p) return { ok: false, error: 'proxy_not_found' } + const observed = structuredClone(p) const blocked = proxyBlockedReason(p, this.state.config.ipv6_enabled) if (blocked) return { ok: false, error: blocked, proxy: this.publicProxy(p) } const v4Cached = !force && p.geo_checked_at && p.geo_ip @@ -1235,30 +1422,33 @@ export class ProxyPool extends EventEmitter { } const timeoutMs = this.state.config?.geo_timeout_ms let v4Result = v4Cached ? { ok: true } : await this.geoLookup(url, { timeoutMs }) - if (!v4Cached) this._applyGeoResult(p, v4Result) let v6Result = v6Cached ? { ok: true } : await this.geoV6Lookup(url, { timeoutMs }) - if (!v6Cached) this._applyGeoV6Result(p, v6Result) - this.save() - const pub = this.publicProxy(p) + const current = await this._persistObservation(observed, (row) => { + if (!v4Cached) this._applyGeoResult(row, v4Result) + if (!v6Cached) this._applyGeoV6Result(row, v6Result) + }) + if (!current) return { ok: false, error: 'proxy_changed', stale: true } + const pub = this.publicProxy(current) if (!v4Result?.ok) { return { ok: false, error: v4Result?.error || 'geo_lookup_failed', proxy: pub, - geo: proxyGeoOf(p), - geo_v6: proxyGeoV6Of(p), + geo: proxyGeoOf(current), + geo_v6: proxyGeoV6Of(current), } } return { ok: true, cached: v4Cached && v6Cached, proxy: pub, - geo: proxyGeoOf(p), - geo_v6: proxyGeoV6Of(p), + geo: proxyGeoOf(current), + geo_v6: proxyGeoV6Of(current), } } async detectGeoAll({ onlyEnabled = true, force = false } = {}) { + await this.refresh() const list = this.state.proxies.filter((p) => (onlyEnabled ? p.enabled : true)) const results = [] for (const p of list) { @@ -1384,7 +1574,42 @@ export class ProxyPool extends EventEmitter { _applyProbeResult(p, result, opts = {}) { this._applyProbeResultInner(p, result, opts) - this.emit('change') + this._emitChange() + } + + async _persistObservation(observed, apply) { + const operation = withTransaction(this.db, async () => { + await this.repo.lockMutations() + const row = await this.repo.getById(observed.id, { forUpdate: true }) + if (!row || row.config_revision !== observed.config_revision) return null + const current = hydrateProxy(row) + const before = this._stored(current) + const cached = this.state + const context = { callbacks: [] } + this.state = { config: { ...DEFAULT_CONFIG, ...(await this.repo.getConfig({})) }, proxies: [current] } + this._mutation = context + try { + apply(current) + } finally { + this.state = cached + this._mutation = null + } + const patch = this._changes(before, this._stored(current), PROXY_HEALTH_COLUMNS) + const stored = await this.repo.patchHealth(current.id, patch, observed.config_revision) + if (!stored) return null + const fresh = hydrateProxy(stored) + this.db.afterCommit(() => { + this._cacheVersion++ + const previous = this.state.proxies.find((proxy) => proxy.id === fresh.id) + if (previous) Object.assign(previous, fresh) + else this.state.proxies.push(fresh) + this._savedState = structuredClone(this.state) + for (const callback of context.callbacks) callback() + this._emitChange() + }) + return fresh + }) + return this._track(operation) } startScheduler() { @@ -1407,6 +1632,10 @@ export class ProxyPool extends EventEmitter { } restartScheduler() { + if (this._mutation) { + this._mutation.restartScheduler = true + return + } this.startScheduler() } diff --git a/test/unit/local-egress-per-node.test.mjs b/test/unit/local-egress-per-node.test.mjs index c7a3e44ed..af1ec290e 100644 --- a/test/unit/local-egress-per-node.test.mjs +++ b/test/unit/local-egress-per-node.test.mjs @@ -45,45 +45,45 @@ test('local egress ids name their VPS and share one bridge per host', () => { test('a generic px-local bind lands on the slot’s own VPS row, with its own capacity', async (t) => { const pool = await makePool(t) - pool.ensureLocal() - pool.updateConfig({ bind_limit: 1 }) - assert.equal(pool.bind('px-local', 'vm-ctrl').proxy.id, 'px-local') + await pool.ensureLocal() + await pool.updateConfig({ bind_limit: 1 }) + assert.equal((await pool.bind('px-local', 'vm-ctrl')).proxy.id, 'px-local') // The control plane is full, the node is not: capacity is per machine. - const a1 = pool.bind('px-local', 'vm-a1') + const a1 = await pool.bind('px-local', 'vm-a1') assert.equal(a1.ok, true) assert.equal(a1.proxy.id, 'px-local-node-a') assert.equal(a1.proxy.node_id, 'node-a') - assert.equal(pool.bind('px-local', 'vm-a2').error, 'proxy_bind_limit') + assert.equal((await pool.bind('px-local', 'vm-a2')).error, 'proxy_bind_limit') assert.equal(pool.getProxyForVm('vm-a1').id, 'px-local-node-a') }) test('another machine’s local row is refused', async (t) => { const pool = await makePool(t) - pool.ensureLocal() - pool.ensureLocalForNode('node-a') - const wrong = pool.bind('px-local-node-a', 'vm-b1') + await pool.ensureLocal() + await pool.ensureLocalForNode('node-a') + const wrong = await pool.bind('px-local-node-a', 'vm-b1') assert.equal(wrong.ok, false) assert.equal(wrong.error, 'local_proxy_node_mismatch') assert.equal(wrong.expected, 'px-local-node-b') - assert.equal(pool.bind('px-local-node-a', 'vm-ctrl').error, 'local_proxy_node_mismatch') + assert.equal((await pool.bind('px-local-node-a', 'vm-ctrl')).error, 'local_proxy_node_mismatch') }) test('auto-allocation only offers the slot’s own local row', async (t) => { const pool = await makePool(t) - pool.ensureLocal() - const picked = pool.allocateForVm('vm-b1') + await pool.ensureLocal() + const picked = await pool.allocateForVm('vm-b1') assert.equal(picked.id, 'px-local-node-b') - assert.equal(pool.allocateForVm('vm-ctrl').id, 'px-local') + assert.equal((await pool.allocateForVm('vm-ctrl')).id, 'px-local') }) test('node rows follow the cluster and keep rows that still hold slots', async (t) => { const pool = await makePool(t) // No local egress in use: nothing is created for nodes. - assert.deepEqual(pool.syncLocalNodes(['node-a']), { created: [], removed: [] }) - pool.ensureLocal() - assert.deepEqual(pool.syncLocalNodes(['node-a', 'node-b']).created, ['px-local-node-a', 'px-local-node-b']) - pool.bind('px-local', 'vm-a1') - const synced = pool.syncLocalNodes([]) + assert.deepEqual(await pool.syncLocalNodes(['node-a']), { created: [], removed: [] }) + await pool.ensureLocal() + assert.deepEqual((await pool.syncLocalNodes(['node-a', 'node-b'])).created, ['px-local-node-a', 'px-local-node-b']) + await pool.bind('px-local', 'vm-a1') + const synced = await pool.syncLocalNodes([]) assert.deepEqual(synced.removed, ['px-local-node-b']) assert.ok(pool.state.proxies.some((p) => p.id === 'px-local-node-a')) // Local rows stay grouped at the top of the pool. @@ -92,12 +92,13 @@ test('node rows follow the cluster and keep rows that still hold slots', async ( test('slots left on the shared row move to their node row, past a full limit', async (t) => { const pool = await makePool(t) - pool.ensureLocal() - pool.updateConfig({ bind_limit: 1 }) + await pool.ensureLocal() + await pool.updateConfig({ bind_limit: 1 }) // Pre-split data: every local slot sits on px-local. const row = pool.state.proxies.find((p) => p.id === 'px-local') row.bound_vm_ids = ['vm-ctrl', 'vm-a1', 'vm-a2', 'vm-b1'] - const moves = pool.adoptNodeLocalBindings() + await pool.save() + const moves = await pool.adoptNodeLocalBindings() assert.deepEqual( moves.map((m) => `${m.vm_id}:${m.to}`), ['vm-a1:px-local-node-a', 'vm-a2:px-local-node-a', 'vm-b1:px-local-node-b'], @@ -105,16 +106,17 @@ test('slots left on the shared row move to their node row, past a full limit', a const ids = (id) => pool.state.proxies.find((p) => p.id === id).bound_vm_ids assert.deepEqual(ids('px-local'), ['vm-ctrl']) assert.deepEqual(ids('px-local-node-a'), ['vm-a1', 'vm-a2']) - assert.deepEqual(pool.adoptNodeLocalBindings(), []) + assert.deepEqual(await pool.adoptNodeLocalBindings(), []) }) test('migration leaves ids of deleted slots alone', async (t) => { const pool = await makePool(t, { vmNodeOf: (vmId) => (vmId === 'vm-gone' ? undefined : (NODES[vmId] ?? null)) }) - pool.ensureLocal() + await pool.ensureLocal() await pool.flush() pool.state.proxies.find((p) => p.id === 'px-local').bound_vm_ids = ['vm-gone', 'vm-a1'] + await pool.save() assert.deepEqual( - pool.adoptNodeLocalBindings().map((m) => m.vm_id), + (await pool.adoptNodeLocalBindings()).map((m) => m.vm_id), ['vm-a1'], ) }) @@ -127,8 +129,8 @@ test('a node row probes its node link; a down link marks it failed without parki healthy ? { ok: true, latency_ms: 42 } : { ok: false, error: `node_link_down:${nodeId}` }, onDisableVm: (vmId) => disabled.push(vmId), }) - pool.ensureLocal() - pool.bind('px-local', 'vm-a1') + await pool.ensureLocal() + await pool.bind('px-local', 'vm-a1') const row = pool.state.proxies.find((p) => p.id === 'px-local-node-a') const up = await pool.probeOne(row) assert.deepEqual(up, { ok: true, scope: 'node', mode: 'direct', latency_ms: 42 }) @@ -146,8 +148,8 @@ test('a node row probes its node link; a down link marks it failed without parki test('a node link still dialling leaves the row alone until it settles', async (t) => { let health = { pending: true, error: 'node_link_connecting' } const pool = await makePool(t, { localExitHealth: () => health }) - pool.ensureLocal() - pool.bind('px-local', 'vm-a1') + await pool.ensureLocal() + await pool.bind('px-local', 'vm-a1') const row = pool.state.proxies.find((p) => p.id === 'px-local-node-a') pool._applyProbeResult(row, await pool.probeOne(row)) assert.equal(row.status, 'ok') @@ -173,8 +175,8 @@ test('a node row looks up its geo through that node’s route', async (t) => { }, geoV6Lookup: async () => ({ ok: false, error: 'no_v6' }), }) - pool.ensureLocal() - pool.bind('px-local', 'vm-a1') + await pool.ensureLocal() + await pool.bind('px-local', 'vm-a1') const r = await pool.detectGeo('px-local-node-a') assert.equal(r.ok, true) assert.equal(pool.proxyTimezone('px-local-node-a'), 'Asia/Tokyo') @@ -201,11 +203,12 @@ test('tearing down a node row never touches the control plane bridge', () => { test('an operator-disabled node row stays disabled through link changes', async (t) => { let health = { ok: false, error: 'node_link_backoff' } const pool = await makePool(t, { localExitHealth: () => health }) - pool.ensureLocal() - pool.bind('px-local', 'vm-a1') + await pool.ensureLocal() + await pool.bind('px-local', 'vm-a1') const row = pool.state.proxies.find((p) => p.id === 'px-local-node-a') row.enabled = false row.status = 'dead' + await pool.save() assert.equal(await pool.probeLocalNode('node-a'), null) pool._applyProbeResult(row, await pool.probeOne(row)) health = { ok: true, latency_ms: 10 } @@ -214,27 +217,29 @@ test('an operator-disabled node row stays disabled through link changes', async assert.equal(row.status, 'dead') }) -test('migration persists once for the whole batch', async (t) => { +test('migration commits targeted bindings once without replacing the pool', async (t) => { const pool = await makePool(t) - pool.ensureLocal() + await pool.ensureLocal() await pool.flush() pool.state.proxies.find((p) => p.id === 'px-local').bound_vm_ids = ['vm-a1', 'vm-a2', 'vm-b1'] + await pool.save() let writes = 0 - const replaceAll = pool.repo.replaceAll.bind(pool.repo) - pool.repo.replaceAll = (rows) => { + const lockMutations = pool.repo.lockMutations.bind(pool.repo) + pool.repo.lockMutations = () => { writes += 1 - return replaceAll(rows) + return lockMutations() } - assert.equal(pool.adoptNodeLocalBindings().length, 3) + pool.repo.replaceAll = () => assert.fail('runtime mutations must not replace the proxy table') + assert.equal((await pool.adoptNodeLocalBindings()).length, 3) await pool.flush() assert.equal(writes, 1) }) test('resolveLocalTarget lands a generic pick on the target VPS', async (t) => { const pool = await makePool(t) - pool.ensureLocal() - assert.deepEqual(pool.resolveLocalTarget('px-local', 'node-a'), { ok: true, id: 'px-local-node-a' }) + await pool.ensureLocal() + assert.deepEqual(await pool.resolveLocalTarget('px-local', 'node-a'), { ok: true, id: 'px-local-node-a' }) assert.ok(pool.state.proxies.some((p) => p.id === 'px-local-node-a')) - assert.deepEqual(pool.resolveLocalTarget('px-local', null), { ok: true, id: 'px-local' }) - assert.equal(pool.resolveLocalTarget('px-local-node-a', 'node-b').error, 'local_proxy_node_mismatch') + assert.deepEqual(await pool.resolveLocalTarget('px-local', null), { ok: true, id: 'px-local' }) + assert.equal((await pool.resolveLocalTarget('px-local-node-a', 'node-b')).error, 'local_proxy_node_mismatch') }) diff --git a/test/unit/panel-api-proxy.test.mjs b/test/unit/panel-api-proxy.test.mjs index 594c92187..f02bb2cca 100644 --- a/test/unit/panel-api-proxy.test.mjs +++ b/test/unit/panel-api-proxy.test.mjs @@ -41,8 +41,8 @@ test('summarizeProxyPool includes disconnect_on_error', async () => { }) const dir = tmpDir() const pool = await new ProxyPool({ dataDir: dir }).init() - pool.importLines('10.0.0.9:1080') - pool.updateConfig({ disconnect_on_error: true }) + await pool.importLines('10.0.0.9:1080') + await pool.updateConfig({ disconnect_on_error: true }) pool.stopScheduler() const sum = summarizeProxyPool(pool) assert.equal(sum.total, 1) @@ -64,9 +64,9 @@ test('dashboard and vm detail merge pool health onto bound VM', async () => { policy: { maxConcurrency: 4, weight: 1, priority: 8 }, }) const pool = await new ProxyPool({ dataDir }).init() - pool.importLines('10.8.8.8:1080') + await pool.importLines('10.8.8.8:1080') const px = pool.snapshot().proxies[0] - pool.bind(px.id, 'vm-02') + await pool.bind(px.id, 'vm-02') const bound = pool.state.proxies[0] bound.status = 'ok' bound.latency_ms = 12 @@ -78,7 +78,7 @@ test('dashboard and vm detail merge pool health onto bound VM', async () => { bound.geo_timezone = 'Asia/Tokyo' bound.geo_checked_at = '2026-01-01T00:00:00Z' bound.label = '东京-2' - pool.save() + await pool.save() pool.stopScheduler() const cfg = { paths: { project }, rewrite: { enabled: false }, base_url: 'http://127.0.0.1' } @@ -158,10 +158,10 @@ test('dead bound proxy blocks credential import', async () => { policy: {}, }) const pool = await new ProxyPool({ dataDir }).init() - pool.importLines('10.7.7.7:1080') + await pool.importLines('10.7.7.7:1080') const id = pool.snapshot().proxies[0].id - pool.bind(id, 'vm-03') - pool.setEnabled(id, false) + await pool.bind(id, 'vm-03') + await pool.setEnabled(id, false) pool.stopScheduler() const cfg = { paths: { project }, rewrite: {}, base_url: '' } const detail = await buildVmDetail({ cfg, accountQuota: fakeQuota(), id: 'vm-03', proxyPool: pool }) @@ -363,11 +363,11 @@ test('local proxy node labels are present for admin and absent for tenant', asyn vmNodeOf: (vmId) => vms.find((vm) => vm.id === vmId)?.node_id || null, }).init() t.after(() => pool.stopScheduler()) - pool.ensureLocal() + await pool.ensureLocal() // A generic px-local bind lands each slot on its own VPS's local row. - assert.equal(pool.bind('px-local', 'vm-ctrl').proxy.id, 'px-local') - assert.equal(pool.bind('px-local', 'vm-tokyo').proxy.id, 'px-local-node-a') - assert.equal(pool.bind('px-local', 'vm-tenant').proxy.id, 'px-local-node-a') + assert.equal((await pool.bind('px-local', 'vm-ctrl')).proxy.id, 'px-local') + assert.equal((await pool.bind('px-local', 'vm-tokyo')).proxy.id, 'px-local-node-a') + assert.equal((await pool.bind('px-local', 'vm-tenant')).proxy.id, 'px-local-node-a') const nodes = [{ id: 'node-a', label: '东京-01' }] const adminSnap = pool.panelSnapshot({ role: 'admin', vms, nodes }) diff --git a/test/unit/proxy-geo.test.mjs b/test/unit/proxy-geo.test.mjs index e6fe1353f..0e98620e5 100644 --- a/test/unit/proxy-geo.test.mjs +++ b/test/unit/proxy-geo.test.mjs @@ -19,7 +19,11 @@ function tmpDir(prefix = 'kin-geo-') { } async function makePool(geoLookup) { - const pool = await new ProxyPool({ dataDir: tmpDir(), geoLookup }).init() + const pool = await new ProxyPool({ + dataDir: tmpDir(), + geoLookup, + geoV6Lookup: async () => ({ ok: false, error: 'mock_no_ipv6' }), + }).init() pool.stopScheduler() return pool } @@ -146,7 +150,7 @@ test('detectGeo caches per proxy and publishes location on the snapshot', async ok: true, geo: { ip: '2001:db8::9', country: 'Japan', country_code: 'JP', timezone: 'Asia/Tokyo' }, }) - const imported = pool.importLines('1.2.3.4:1080') + const imported = await pool.importLines('1.2.3.4:1080') const id = imported.items[0].id const first = await pool.detectGeo(id) @@ -175,7 +179,7 @@ test('detectGeo keeps IPv4 and IPv6 results separate and records IPv6 unavailabi geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) pool.geoV6Lookup = async () => ({ ok: false, error: 'geo_transport_error:timeout' }) - const id = pool.importLines('1.2.3.4:1080').items[0].id + const id = (await pool.importLines('1.2.3.4:1080')).items[0].id const result = await pool.detectGeo(id) assert.equal(result.ok, true) assert.equal(result.geo.ip, '203.0.113.9') @@ -189,7 +193,7 @@ test('detectGeo rejects IPv4 masquerading as IPv6 on the v6 path', async () => { geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) pool.geoV6Lookup = async () => ({ ok: false, error: 'geo_ipv6_got_ipv4' }) - const id = pool.importLines('1.2.3.4:1080').items[0].id + const id = (await pool.importLines('1.2.3.4:1080')).items[0].id await pool.detectGeo(id) const snap = pool.snapshot().proxies.find((p) => p.id === id) assert.equal(snap.geo.ip, '203.0.113.9') @@ -204,7 +208,7 @@ test('a failed lookup keeps the last known location and records the error', asyn ? { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' } } : { ok: false, error: 'geo_http_500' }, ) - const id = pool.importLines('1.2.3.4:1080').items[0].id + const id = (await pool.importLines('1.2.3.4:1080')).items[0].id await pool.detectGeo(id) ok = false const failed = await pool.detectGeo(id, { force: true }) @@ -220,8 +224,8 @@ test('binding a proxy moves the slot onto the exit node timezone', async () => { ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: 'Asia/Tokyo' }, })) - const id = pool.importLines('1.2.3.4:1080').items[0].id - pool.bind(id, 'vm-01') + const id = (await pool.importLines('1.2.3.4:1080')).items[0].id + await pool.bind(id, 'vm-01') const root = seedProject({ id: 'vm-01', timezone: 'America/Los_Angeles', @@ -239,8 +243,8 @@ test('binding a proxy moves the slot onto the exit node timezone', async () => { test('a hand-pinned slot timezone survives a proxy bind unless forced', async () => { const pool = await makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) - const id = pool.importLines('1.2.3.4:1080').items[0].id - pool.bind(id, 'vm-02') + const id = (await pool.importLines('1.2.3.4:1080')).items[0].id + await pool.bind(id, 'vm-02') const root = seedProject({ id: 'vm-02', timezone: 'America/New_York', @@ -260,13 +264,13 @@ test('a hand-pinned slot timezone survives a proxy bind unless forced', async () test('follow_proxy_timezone=false stops the bind-time default', async () => { const pool = await makePool(async () => ({ ok: true, geo: { timezone: 'Asia/Tokyo' } })) - pool.updateConfig({ follow_proxy_timezone: false }) + await pool.updateConfig({ follow_proxy_timezone: false }) // updateConfig restarts the probe loop; left running it keeps the test // process alive forever. pool.stopScheduler() assert.equal(pool.followProxyTimezoneEnabled(), false) - const id = pool.importLines('1.2.3.4:1080').items[0].id - pool.bind(id, 'vm-03') + const id = (await pool.importLines('1.2.3.4:1080')).items[0].id + await pool.bind(id, 'vm-03') const root = seedProject({ id: 'vm-03', timezone: 'America/New_York', proxy: { id } }) const skipped = await syncVmTimezoneFromProxy(root, pool, 'vm-03') @@ -276,8 +280,8 @@ test('follow_proxy_timezone=false stops the bind-time default', async () => { test('an unresolvable exit zone leaves the slot timezone alone', async () => { const pool = await makePool(async () => ({ ok: true, geo: { ip: '203.0.113.9', country: 'Japan', timezone: null } })) - const id = pool.importLines('1.2.3.4:1080').items[0].id - pool.bind(id, 'vm-04') + const id = (await pool.importLines('1.2.3.4:1080')).items[0].id + await pool.bind(id, 'vm-04') const root = seedProject({ id: 'vm-04', timezone: 'America/Denver', proxy: { id } }) const skipped = await syncVmTimezoneFromProxy(root, pool, 'vm-04') @@ -292,9 +296,9 @@ test('a node slot on local egress follows its node exit, not the control plane r // '' = the control plane's own route (px-local row); a SOCKS URL = the node forwarder. return { ok: true, geo: { timezone: url ? 'Europe/Berlin' : 'Asia/Tokyo' } } }) - const local = pool.ensureLocal().proxy - pool.bind(local.id, 'vm-05') - pool.bind(local.id, 'vm-06') + const local = (await pool.ensureLocal()).proxy + await pool.bind(local.id, 'vm-05') + await pool.bind(local.id, 'vm-06') await startNodeEgressSocks({ clientFor: () => assert.fail('geo lookup is injected') }) t.after(stopNodeEgressSocks) const proxy = { id: local.id, scheme: 'local', host: 'local', port: 0 } diff --git a/test/unit/proxy-ipv6.test.mjs b/test/unit/proxy-ipv6.test.mjs index cabeb9cf8..c453fe13e 100644 --- a/test/unit/proxy-ipv6.test.mjs +++ b/test/unit/proxy-ipv6.test.mjs @@ -12,7 +12,7 @@ import { isCodexSlotReady, pickCodexSlots } from '../../src/lib/pool/codex-slot- async function makePool(t) { const pool = await new ProxyPool({ db: await openDatabase({ dbPath: ':memory:' }) }).init() - pool.updateConfig({ enabled: false }) + await pool.updateConfig({ enabled: false }) t.after(async () => { pool.stopScheduler() await closeDatabase() @@ -22,7 +22,7 @@ async function makePool(t) { test('IPv6 aliases normalize and deduplicate after loading bracketed historical hosts', async (t) => { const pool = await makePool(t) - const first = pool.importLines('socks5h://a%3Ab:p%40%3A%2F%25@[2001:0db8:0:0:0:0:0:1]:1080') + const first = await pool.importLines('socks5h://a%3Ab:p%40%3A%2F%25@[2001:0db8:0:0:0:0:0:1]:1080') const id = first.items[0].id const full = pool.getProxyByIdWithAuth(id) assert.equal(full.host, '2001:db8::1') @@ -34,7 +34,7 @@ test('IPv6 aliases normalize and deduplicate after loading bracketed historical await pool.db.prepare('UPDATE proxies SET host = $1 WHERE id = $2').run('[2001:0db8::1]', id) await pool.reload() assert.equal(pool.state.proxies[0].host, '2001:db8::1') - const result = pool.importLines('', { + const result = await pool.importLines('', { fields: [{ host: '2001:db8::1', port: 1080, username: 'a:b', password: 'p@:/%' }], }) assert.equal(result.added, 0) @@ -77,7 +77,7 @@ test('malformed historical IPv6 hosts cannot silently probe localhost', async (t }) const { items: [proxy], - } = pool.importLines(`127.0.0.1:${server.address().port}`) + } = await pool.importLines(`127.0.0.1:${server.address().port}`) await pool.flush() await pool.db.prepare('UPDATE proxies SET host = $1 WHERE id = $2').run('[::1', proxy.id) await pool.reload() @@ -90,24 +90,25 @@ test('IPv6 policy survives reload and preserves binding and health instead of re const pool = await makePool(t) const { items: [ipv6], - } = pool.importLines('[2001:db8::1]:1080\n127.0.0.1:1080') - assert.equal(pool.bind(ipv6.id, 'vm-01').error, 'ipv6_disabled') - assert.equal(pool.allocateForVm('vm-02').host, '127.0.0.1') + } = await pool.importLines('[2001:db8::1]:1080\n127.0.0.1:1080') + assert.equal((await pool.bind(ipv6.id, 'vm-01')).error, 'ipv6_disabled') + assert.equal((await pool.allocateForVm('vm-02')).host, '127.0.0.1') assert.equal((await pool.detectGeo(ipv6.id)).error, 'ipv6_disabled') assert.throws(() => boundProxyUrl({ host: '::1', port: 1080 }), { code: 'ipv6_disabled' }) - pool.updateConfig({ ipv6_enabled: true }) - assert.equal(pool.bind(ipv6.id, 'vm-01').ok, true) + await pool.updateConfig({ ipv6_enabled: true }) + assert.equal((await pool.bind(ipv6.id, 'vm-01')).ok, true) const proxy = pool.state.proxies.find((p) => p.id === ipv6.id) Object.assign(proxy, { status: 'ok', latency_ms: 20, last_probe_at: '2026-10-01T00:00:00Z', consecutive_failures: 2 }) const history = { status: proxy.status, last_probe_at: proxy.last_probe_at, failures: proxy.consecutive_failures } - pool.updateConfig({ ipv6_enabled: false, disconnect_on_error: true }) + await pool.save() + await pool.updateConfig({ ipv6_enabled: false, disconnect_on_error: true }) await pool.reload() assert.equal(pool.snapshot().config.ipv6_enabled, false) assert.equal(pool.getProxyForVm('vm-01'), null) - assert.equal(pool.ensureBoundToVm('vm-01', ipv6.id), null) - assert.equal(pool.ensureBoundToVm('vm-03', ipv6.id), null) - assert.equal(pool.allocateForVm('vm-01'), null) - assert.equal(pool.reportRuntimeFailure('vm-01', 'connection_closed').reason, 'ipv6_disabled') + assert.equal(await pool.ensureBoundToVm('vm-01', ipv6.id), null) + assert.equal(await pool.ensureBoundToVm('vm-03', ipv6.id), null) + assert.equal(await pool.allocateForVm('vm-01'), null) + assert.equal((await pool.reportRuntimeFailure('vm-01', 'connection_closed')).reason, 'ipv6_disabled') const result = await pool.probeById(ipv6.id) assert.equal(result.probe.scope, 'policy') assert.equal(result.proxy.blocked_reason, 'ipv6_disabled') @@ -129,7 +130,7 @@ test('IPv6 policy survives reload and preserves binding and health instead of re const codex = { ...vm, platform: 'openai', has_token: true, status: 'running' } assert.equal(isCodexSlotReady(codex), false) assert.equal(pickCodexSlots([codex], { pin: codex.id }).error, 'ipv6_disabled') - pool.updateConfig({ ipv6_enabled: true }) + await pool.updateConfig({ ipv6_enabled: true }) await pool.reload() assert.equal(pool.getProxyForVm('vm-01').id, ipv6.id) assert.equal(boundProxyUrl(vm.proxy), 'socks5h://[2001:db8::1]:1080') @@ -156,18 +157,18 @@ test('URL-imported IPv6 probes connect only when enabled and closing cancels in- }) const { items: [proxy], - } = pool.importLines(`socks5h://[::1]:${server.address().port}`) + } = await pool.importLines(`socks5h://[::1]:${server.address().port}`) assert.equal((await pool.probeById(proxy.id)).probe.error, 'ipv6_disabled') assert.equal((await pool.probeAll()).total, 0) assert.equal(connections, 0) - pool.updateConfig({ ipv6_enabled: true }) + await pool.updateConfig({ ipv6_enabled: true }) assert.equal((await pool.probeById(proxy.id)).probe.ok, true) const history = pool.state.proxies[0].last_probe_at const connected = once(server, 'connection') const pending = pool.probeById(proxy.id) const [socket] = await connected const closed = once(socket, 'close') - pool.updateConfig({ ipv6_enabled: false }) + await pool.updateConfig({ ipv6_enabled: false }) assert.equal((await pending).probe.error, 'ipv6_disabled') await closed assert.equal(pool.state.proxies[0].status, 'ok') diff --git a/test/unit/proxy-pool-bind.test.mjs b/test/unit/proxy-pool-bind.test.mjs index 3cdc6db97..6eb2c4747 100644 --- a/test/unit/proxy-pool-bind.test.mjs +++ b/test/unit/proxy-pool-bind.test.mjs @@ -21,9 +21,9 @@ test('proxy pool persists custom dns_primary and rejects invalid URLs without ch t.after(() => fs.rmSync(dir, { recursive: true, force: true })) const pool = await new ProxyPool({ dataDir: dir }).init() pool.stopScheduler() - assert.equal(pool.updateConfig({ dns_primary: '8.8.8.8:53' }).ok, true) + assert.equal((await pool.updateConfig({ dns_primary: '8.8.8.8:53' })).ok, true) const primary = 'https://cloudflare-dns.com:8443/dns-query?key=a%2Cb' - assert.equal(pool.updateConfig({ dns_primary: primary }).ok, true) + assert.equal((await pool.updateConfig({ dns_primary: primary })).ok, true) for (const invalid of [ '9.9.9.9:53', 'http://cloudflare-dns.com/dns-query', @@ -32,7 +32,7 @@ test('proxy pool persists custom dns_primary and rejects invalid URLs without ch 'https://cloudflare-dns.com/dns-query?key=a,b', 'https://cloudflare-dns.com:65536/dns-query', ]) { - const result = pool.updateConfig({ dns_primary: invalid }) + const result = await pool.updateConfig({ dns_primary: invalid }) assert.equal(result.ok, false, invalid) assert.equal(result.error, 'invalid_dns_primary', invalid) assert.equal(pool.snapshot().config.dns_primary, primary) @@ -48,9 +48,9 @@ test('DNS type suppression persists both toggle states and rejects non-booleans' const pool = await new ProxyPool({ dataDir: dir }).init() pool.stopScheduler() assert.equal(pool.snapshot().config.dns_disable_svcb_https, false) - assert.equal(pool.updateConfig({ dns_disable_svcb_https: true }).ok, true) + assert.equal((await pool.updateConfig({ dns_disable_svcb_https: true })).ok, true) for (const value of ['false', 1, null, [], {}]) { - const result = pool.updateConfig({ dns_disable_svcb_https: value, dns_primary: '8.8.8.8:53' }) + const result = await pool.updateConfig({ dns_disable_svcb_https: value, dns_primary: '8.8.8.8:53' }) assert.equal(result.error, 'invalid_dns_disable_svcb_https') assert.equal(pool.snapshot().config.dns_disable_svcb_https, true) assert.equal(pool.snapshot().config.dns_primary, 'auto') @@ -58,7 +58,7 @@ test('DNS type suppression persists both toggle states and rejects non-booleans' const reopened = await new ProxyPool({ dataDir: dir }).init() reopened.stopScheduler() assert.equal(reopened.snapshot().config.dns_disable_svcb_https, true) - assert.equal(reopened.updateConfig({ dns_disable_svcb_https: false }).ok, true) + assert.equal((await reopened.updateConfig({ dns_disable_svcb_https: false })).ok, true) const disabled = await new ProxyPool({ dataDir: dir }).init() disabled.stopScheduler() assert.equal(disabled.snapshot().config.dns_disable_svcb_https, false) @@ -76,20 +76,20 @@ test('clampBindLimit defaults to 5 and stays in 1..32', () => { test('bind_limit defaults to 5 and is configurable', async () => { const pool = await makePool() - pool.importLines('10.0.0.1:1080') + await pool.importLines('10.0.0.1:1080') const id = pool.snapshot().proxies[0].id assert.equal(pool.bindLimit(), 5) assert.equal(pool.snapshot().config.bind_limit, 5) for (let i = 1; i <= 5; i++) { - const r = pool.bind(id, `vm-0${i}`) + const r = await pool.bind(id, `vm-0${i}`) assert.equal(r.ok, true, `bind vm-0${i}`) } - const full = pool.bind(id, 'vm-06') + const full = await pool.bind(id, 'vm-06') assert.equal(full.ok, false) assert.equal(full.error, 'proxy_bind_limit') assert.equal(full.max, 5) - const cfg = pool.updateConfig({ bind_limit: 2 }) + const cfg = await pool.updateConfig({ bind_limit: 2 }) pool.stopScheduler() assert.equal(cfg.ok, true) assert.equal(pool.bindLimit(), 2) @@ -97,24 +97,24 @@ test('bind_limit defaults to 5 and is configurable', async () => { assert.equal(pool.snapshot().totals.bind_limit, 2) // existing binds stay; new binds still rejected at the new cap assert.equal(pool.snapshot().proxies[0].bound_count, 5) - assert.equal(pool.bind(id, 'vm-07').ok, false) + assert.equal((await pool.bind(id, 'vm-07')).ok, false) - pool.unbind(id) - assert.equal(pool.bind(id, 'vm-01').ok, true) - assert.equal(pool.bind(id, 'vm-02').ok, true) - assert.equal(pool.bind(id, 'vm-03').ok, false) - assert.equal(pool.bind(id, 'vm-03').max, 2) + await pool.unbind(id) + assert.equal((await pool.bind(id, 'vm-01')).ok, true) + assert.equal((await pool.bind(id, 'vm-02')).ok, true) + assert.equal((await pool.bind(id, 'vm-03')).ok, false) + assert.equal((await pool.bind(id, 'vm-03')).max, 2) }) test('remove deletes a proxy and updateConfig rejects bad bind_limit', async () => { const pool = await makePool() - pool.importLines('10.0.0.2:1080') + await pool.importLines('10.0.0.2:1080') const id = pool.snapshot().proxies[0].id - const removed = pool.remove(id) + const removed = await pool.remove(id) assert.equal(removed.ok, true) assert.equal(pool.snapshot().proxies.length, 0) - assert.equal(pool.updateConfig({ bind_limit: 0 }).ok, false) - assert.equal(pool.updateConfig({ bind_limit: 33 }).ok, false) + assert.equal((await pool.updateConfig({ bind_limit: 0 })).ok, false) + assert.equal((await pool.updateConfig({ bind_limit: 33 })).ok, false) pool.stopScheduler() }) @@ -137,7 +137,7 @@ test('probeOne marks egress_down after SOCKS greeting if helper is dead', async }).init() pool.stopScheduler() try { - pool.importLines(`${socks.host}:${socks.port}`) + await pool.importLines(`${socks.host}:${socks.port}`) const result = await pool.probeOne(pool.state.proxies[0]) assert.equal(result.ok, false) assert.equal(result.error, 'not_running') @@ -159,7 +159,7 @@ test('probeOne repairs egress then passes', async () => { }).init() pool.stopScheduler() try { - pool.importLines(`${socks.host}:${socks.port}`) + await pool.importLines(`${socks.host}:${socks.port}`) const result = await pool.probeOne(pool.state.proxies[0]) assert.equal(result.ok, true) assert.equal(healthy, true) @@ -182,7 +182,7 @@ test('startScheduler immediately repairs enabled egress', async () => { }, }).init() try { - pool.importLines(`${socks.host}:${socks.port}`) + await pool.importLines(`${socks.host}:${socks.port}`) pool.startScheduler() const deadline = Date.now() + 1000 while (!healthy && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10)) @@ -202,7 +202,7 @@ test('egress_down does not mark a live SOCKS proxy dead', async () => { }).init() pool.stopScheduler() try { - pool.importLines(`${socks.host}:${socks.port}`) + await pool.importLines(`${socks.host}:${socks.port}`) const id = pool.snapshot().proxies[0].id await pool.probeById(id) await pool.probeById(id) @@ -223,7 +223,7 @@ test('TCP connect then close is not a SOCKS death', async () => { const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() try { - pool.importLines(`${addr.address}:${addr.port}`) + await pool.importLines(`${addr.address}:${addr.port}`) const result = await pool._probeSocks(pool.state.proxies[0]) assert.equal(result.ok, true) } finally { @@ -237,12 +237,13 @@ test('probeAll resurrects a probe-killed proxy', async () => { const pool = await new ProxyPool({ dataDir: tmpDir() }).init() pool.stopScheduler() try { - pool.importLines(`${socks.host}:${socks.port}`) + await pool.importLines(`${socks.host}:${socks.port}`) const p = pool.state.proxies[0] p.enabled = false p.status = 'dead' p.last_error = 'connection_closed' p.consecutive_failures = 2 + await pool.save() const report = await pool.probeAll() assert.equal(report.ok, true) assert.equal(pool.state.proxies[0].enabled, true) @@ -255,15 +256,15 @@ test('probeAll resurrects a probe-killed proxy', async () => { test('ensureLocal adds a single local egress row', async () => { const pool = await makePool() - const first = pool.ensureLocal() - const second = pool.ensureLocal() + const first = await pool.ensureLocal() + const second = await pool.ensureLocal() assert.equal(first.ok, true) assert.equal(first.created, true) assert.equal(first.proxy.kind, 'local') assert.equal(first.proxy.id, 'px-local') assert.equal(second.created, false) assert.equal(pool.snapshot().proxies.filter((p) => p.kind === 'local').length, 1) - const bound = pool.bind('px-local', 'vm-01') + const bound = await pool.bind('px-local', 'vm-01') assert.equal(bound.ok, true) const auth = pool.getProxyForVm('vm-01') assert.equal(auth.scheme, 'local') @@ -278,8 +279,8 @@ test('local egress probe stays direct when kin-egress is down', async () => { onDisableVm: (vmId) => disabled.push(vmId), }).init() pool.stopScheduler() - pool.ensureLocal() - pool.bind('px-local', 'vm-01') + await pool.ensureLocal() + await pool.bind('px-local', 'vm-01') const row = pool.state.proxies.find((p) => p.id === 'px-local') const result = await pool.probeOne(row) assert.equal(result.ok, true) diff --git a/test/unit/proxy-pool-import.test.mjs b/test/unit/proxy-pool-import.test.mjs index fb6d6c0f3..3f538bfa1 100644 --- a/test/unit/proxy-pool-import.test.mjs +++ b/test/unit/proxy-pool-import.test.mjs @@ -15,7 +15,7 @@ async function makePool(t) { test('import: different passwords at the same endpoint are distinct within a batch and after reload', async (t) => { const pool = await makePool(t) - const first = pool.importLines('1.2.3.4:1080:alice:first\n1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:first') + const first = await pool.importLines('1.2.3.4:1080:alice:first\n1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:first') assert.equal(first.added, 2) assert.equal(first.skipped, 1) assert.equal(first.skip_details[0].reason, 'duplicate') @@ -23,7 +23,7 @@ test('import: different passwords at the same endpoint are distinct within a bat await pool.reload() assert.deepEqual(pool.state.proxies.map((p) => p.password).sort(), ['first', 'second']) - const next = pool.importLines('1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:third') + const next = await pool.importLines('1.2.3.4:1080:alice:second\n1.2.3.4:1080:alice:third') assert.equal(next.added, 1) assert.equal(next.skipped, 1) assert.equal(pool.state.proxies.length, 3) @@ -31,7 +31,7 @@ test('import: different passwords at the same endpoint are distinct within a bat test('import: structured credentials containing colons do not collide, equivalent URLs still deduplicate', async (t) => { const pool = await makePool(t) - const result = pool.importLines('', { + const result = await pool.importLines('', { fields: [ { host: '1.2.3.4', port: 1080, username: 'alice:region', password: 'secret' }, { host: '1.2.3.4', port: 1080, username: 'alice', password: 'region:secret' }, @@ -39,7 +39,7 @@ test('import: structured credentials containing colons do not collide, equivalen }) assert.equal(result.added, 2) assert.equal(result.skipped, 0) - const duplicate = pool.importLines('socks5://alice:region%3Asecret@1.2.3.4:1080') + const duplicate = await pool.importLines('socks5://alice:region%3Asecret@1.2.3.4:1080') assert.equal(duplicate.added, 0) assert.equal(duplicate.skipped, 1) }) diff --git a/test/unit/proxy-pool-shared.test.mjs b/test/unit/proxy-pool-shared.test.mjs index ea67efbdb..0a90003ad 100644 --- a/test/unit/proxy-pool-shared.test.mjs +++ b/test/unit/proxy-pool-shared.test.mjs @@ -35,3 +35,259 @@ test('a stale replica probe and flush preserve a proxy created by another connec 'a probe must not replace the whole proxy table with its stale working set', ) }) + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +test('an old probe cannot overwrite a new endpoint, credentials, owner, binding or manual disable', async (t) => { + const [first, second] = await replicas(t) + const added = await first.importLines('192.0.2.3:1080:user:old') + const id = added.items[0].id + await first.bind(id, 'vm-old') + const entered = deferred() + const done = deferred() + let callbacks = 0 + second.onEnableVm = second.onDisableVm = () => { + callbacks++ + } + second.probeOne = async () => { + entered.resolve() + await done.promise + return { ok: true, scope: 'socks', latency_ms: 1 } + } + const probing = second.probeById(id) + try { + await entered.promise + await first.update(id, { host: '192.0.2.4', port: 2080, username: 'new-user', password: 'new-password' }) + await first.repo.patchConfig(id, { owner_user_id: 'owner-new' }) + await first.unbind(id, 'vm-old') + await first.bind(id, 'vm-new') + await first.setEnabled(id, false) + done.resolve() + const result = await probing + assert.equal(result.stale, true) + const stored = await first.repo.getById(id) + assert.equal(stored.host, '192.0.2.4') + assert.equal(stored.port, 2080) + assert.equal(stored.username, 'new-user') + assert.equal(stored.password, 'new-password') + assert.equal(stored.owner_user_id, 'owner-new') + assert.equal(stored.bound_vm_id, 'vm-new') + assert.equal(stored.enabled, false) + assert.equal(stored.last_probe_at, null) + assert.equal(callbacks, 0) + } finally { + done.resolve() + } +}) + +test('stale configuration edits merge only supplied fields with the current PostgreSQL row', async (t) => { + const [first, second] = await replicas(t) + const added = await first.importLines('192.0.2.5:1080:user:old') + const id = added.items[0].id + await second.refresh() + await first.update(id, { username: 'peer-user', password: 'peer-password', label: 'peer-label' }) + await first.bind(id, 'vm-peer') + await second.update(id, { port: 2081 }) + const row = await first.repo.getById(id) + assert.equal(row.port, 2081) + assert.equal(row.username, 'peer-user') + assert.equal(row.password, 'peer-password') + assert.equal(row.label, 'peer-label') + assert.equal(row.bound_vm_id, 'vm-peer') +}) + +test('a completed probe never recreates a proxy deleted by another replica', async (t) => { + const [first, second] = await replicas(t) + const { + items: [proxy], + } = await first.importLines('192.0.2.6:1080') + const entered = deferred() + const done = deferred() + second.probeOne = async () => { + entered.resolve() + await done.promise + return { ok: true, latency_ms: 1 } + } + const probing = second.probeById(proxy.id) + try { + await entered.promise + await first.remove(proxy.id) + done.resolve() + assert.equal((await probing).stale, true) + assert.equal(await first.repo.getById(proxy.id), null) + } finally { + done.resolve() + } +}) + +test('parallel imports retain both new endpoints and deduplicate identical credentials', async (t) => { + const [first, second] = await replicas(t) + const distinct = await Promise.all([first.importLines('192.0.2.7:1080'), second.importLines('192.0.2.8:1080')]) + assert.equal( + distinct.reduce((count, result) => count + result.added, 0), + 2, + ) + const same = await Promise.all([ + first.importLines('192.0.2.9:1080:user:password'), + second.importLines('192.0.2.9:1080:user:password'), + ]) + assert.equal( + same.reduce((count, result) => count + result.added, 0), + 1, + ) + assert.equal( + same.reduce((count, result) => count + result.skipped, 0), + 1, + ) + assert.equal((await first.repo.loadAll()).length, 3) +}) + +test('binding capacity and one-proxy-per-VM checks use current PostgreSQL bindings', async (t) => { + const [first, second] = await replicas(t) + await first.updateConfig({ enabled: false, bind_limit: 1 }) + const { + items: [proxy, other], + } = await first.importLines('192.0.2.10:1080\n192.0.2.11:1080') + const results = await Promise.all([first.bind(proxy.id, 'vm-a'), second.bind(proxy.id, 'vm-b')]) + assert.equal(results.filter((result) => result.ok).length, 1) + assert.equal(results.find((result) => !result.ok).error, 'proxy_bind_limit') + const winner = (await first.repo.getById(proxy.id)).bound_vm_id + assert.equal((await second.bind(other.id, winner)).ok, true) + assert.equal((await first.repo.getById(proxy.id)).bound_vm_id, null) + assert.equal((await first.repo.getById(other.id)).bound_vm_id, winner) +}) + +test('failed health persistence never fires VM state callbacks or publishes optimistic health', async (t) => { + const [first, second] = await replicas(t) + await first.updateConfig({ enabled: false, max_failures: 1 }) + const { + items: [proxy], + } = await first.importLines('192.0.2.12:1080') + await first.bind(proxy.id, 'vm-a') + await second.refresh() + let callbacks = 0 + second.onDisableVm = () => { + callbacks++ + } + second.probeOne = async () => ({ ok: false, scope: 'socks', error: 'mock_failure', latency_ms: null }) + await first.db.exec(`CREATE FUNCTION reject_proxy_health() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'mock health persistence failure' USING ERRCODE = 'P0001'; END $$; + CREATE TRIGGER reject_proxy_health BEFORE UPDATE OF status ON proxies + FOR EACH ROW EXECUTE FUNCTION reject_proxy_health();`) + await assert.rejects(second.probeById(proxy.id), { code: 'P0001' }) + assert.equal(callbacks, 0) + assert.equal(second.snapshot().proxies[0].status, 'unknown') + assert.equal((await first.repo.getById(proxy.id)).enabled, true) +}) + +test('refresh does not wait for a network probe and a late refresh cannot overwrite newer state', { + timeout: 2000, +}, async (t) => { + const [first, second] = await replicas(t) + const { + items: [proxy], + } = await first.importLines('192.0.2.13:1080') + const entered = deferred() + const done = deferred() + second.probeOne = async () => { + entered.resolve() + await done.promise + return { ok: true, latency_ms: 1 } + } + const probing = second.probeById(proxy.id) + try { + await entered.promise + await first.update(proxy.id, { label: 'peer-label' }) + await second.refresh() + assert.equal(second.labelOf(proxy.id), 'peer-label') + done.resolve() + await probing + } finally { + done.resolve() + } + const captured = deferred() + const release = deferred() + const snapshot = second.repo.snapshot.bind(second.repo) + second.repo.snapshot = async () => { + const data = await snapshot() + captured.resolve() + await release.promise + return data + } + const older = second.refresh() + await captured.promise + second.repo.snapshot = snapshot + await first.update(proxy.id, { label: 'latest-label' }) + await second.refresh() + release.resolve() + await older + assert.equal(second.labelOf(proxy.id), 'latest-label') +}) + +test('pool configuration edits preserve fields committed by another replica', async (t) => { + const [first, second] = await replicas(t) + await first.updateConfig({ enabled: false, bind_limit: 7 }) + await second.updateConfig({ geo_timeout_ms: 1200 }) + const config = await first.repo.getConfig() + assert.equal(config.enabled, false) + assert.equal(config.bind_limit, 7) + assert.equal(config.geo_timeout_ms, 1200) +}) + +test('saving a stale explicit state edit preserves peer creations and rejects changed rows', async (t) => { + const [first, second] = await replicas(t) + const { + items: [proxy], + } = await first.importLines('192.0.2.14:1080') + await second.refresh() + const { + items: [peer], + } = await first.importLines('192.0.2.15:1080') + second.state.proxies[0].label = 'local-label' + await second.save() + assert.equal((await first.repo.getById(proxy.id)).label, 'local-label') + assert.ok(await first.repo.getById(peer.id)) + await first.update(proxy.id, { password: 'peer-password', username: 'peer-user' }) + second.state.proxies.find((row) => row.id === proxy.id).label = 'outdated-edit' + await assert.rejects(second.save(), { code: 'proxy_changed' }) + assert.equal((await first.repo.getById(proxy.id)).password, 'peer-password') +}) + +test('an old probe does not repair egress with obsolete connection configuration', async (t) => { + const [first, second] = await replicas(t) + const { + items: [proxy], + } = await first.importLines('192.0.2.16:1080') + await second.refresh() + const entered = deferred() + const done = deferred() + second._probeSocks = async () => { + entered.resolve() + await done.promise + return { ok: true, latency_ms: 1 } + } + second.egressCheck = () => ({ ok: false, reason: 'mock_egress_down' }) + let repairs = 0 + second.repairEgress = () => { + repairs++ + } + second.probeOne = ProxyPool.prototype.probeOne + const probing = second.probeById(proxy.id) + try { + await entered.promise + await first.update(proxy.id, { port: 2080 }) + done.resolve() + const result = await probing + assert.equal(result.stale, true) + assert.equal(result.probe.error, 'proxy_changed') + assert.equal(repairs, 0) + } finally { + done.resolve() + } +}) diff --git a/test/unit/proxy-pool-update.test.mjs b/test/unit/proxy-pool-update.test.mjs index a0cc5579c..274c0401c 100644 --- a/test/unit/proxy-pool-update.test.mjs +++ b/test/unit/proxy-pool-update.test.mjs @@ -10,16 +10,16 @@ async function makePool() { pool.stopScheduler() return pool } -function seed(pool, line = 'socks5://alice:s3cret@1.2.3.4:1080') { - pool.importLines(line) +async function seed(pool, line = 'socks5://alice:s3cret@1.2.3.4:1080') { + await pool.importLines(line) return pool.state.proxies[0] } test('update: 只改 port，账密保留；raw 明文密码被清除', async () => { const pool = await makePool() - const p = seed(pool) + const p = await seed(pool) assert.match(p.raw, /s3cret/, '前提：导入后 raw 含明文密码') - const r = pool.update(p.id, { port: 1081 }) + const r = await pool.update(p.id, { port: 1081 }) assert.equal(r.ok, true) assert.equal(p.port, 1081) assert.equal(p.username, 'alice') @@ -30,16 +30,16 @@ test('update: 只改 port，账密保留；raw 明文密码被清除', async () test('update: username 传空串清除账密（password 一并清）', async () => { const pool = await makePool() - const p = seed(pool) - assert.equal(pool.update(p.id, { username: '' }).ok, true) + const p = await seed(pool) + assert.equal((await pool.update(p.id, { username: '' })).ok, true) assert.equal(p.username, null) assert.equal(p.password, null) }) test('update: 不传的字段保持不动', async () => { const pool = await makePool() - const p = seed(pool) - pool.update(p.id, { password: 'newpw' }) + const p = await seed(pool) + await pool.update(p.id, { password: 'newpw' }) assert.equal(p.host, '1.2.3.4') assert.equal(p.port, 1080) assert.equal(p.username, 'alice') @@ -48,37 +48,37 @@ test('update: 不传的字段保持不动', async () => { test('update: 只给密码不给用户名被拒，不静默存成「仍无账密」', async () => { const pool = await makePool() - pool.importLines('9.9.9.9:1080') + await pool.importLines('9.9.9.9:1080') const p = pool.state.proxies[0] assert.equal(p.username, null, '前提：这条本来没有账密') - assert.equal(pool.update(p.id, { password: 'onlypw' }).error, 'password_without_username') + assert.equal((await pool.update(p.id, { password: 'onlypw' })).error, 'password_without_username') assert.equal(p.password, null, '被拒后不留半套账密') // 清了账密还想在同一次请求里设密码，同样是无效组合 - assert.equal(pool.update(p.id, { username: '', password: 'x' }).error, 'password_without_username') + assert.equal((await pool.update(p.id, { username: '', password: 'x' })).error, 'password_without_username') }) test('update: 只换用户名时沿用原密码（不传的键就是不动）', async () => { const pool = await makePool() - const p = seed(pool) - assert.equal(pool.update(p.id, { username: 'bob' }).ok, true) + const p = await seed(pool) + assert.equal((await pool.update(p.id, { username: 'bob' })).ok, true) assert.equal(p.username, 'bob') assert.equal(p.password, 's3cret', 'password 没传，按语义保持原值') }) test('update: 非法值与空 patch 被拒，且不改动原记录', async () => { const pool = await makePool() - const p = seed(pool) - assert.equal(pool.update(p.id, { port: 99999 }).error, 'invalid_proxy') - assert.equal(pool.update(p.id, { host: '' }).error, 'invalid_proxy') - assert.equal(pool.update(p.id, {}).error, 'no_editable_fields') - assert.equal(pool.update('nope', { port: 1080 }).error, 'proxy_not_found') + const p = await seed(pool) + assert.equal((await pool.update(p.id, { port: 99999 })).error, 'invalid_proxy') + assert.equal((await pool.update(p.id, { host: '' })).error, 'invalid_proxy') + assert.equal((await pool.update(p.id, {})).error, 'no_editable_fields') + assert.equal((await pool.update('nope', { port: 1080 })).error, 'proxy_not_found') assert.equal(p.port, 1080, '被拒后原值不变') }) test('update: 响应里的 publicProxy 不含账密', async () => { const pool = await makePool() - const p = seed(pool) - const r = pool.update(p.id, { port: 1081 }) + const p = await seed(pool) + const r = await pool.update(p.id, { port: 1081 }) const s = JSON.stringify(r.proxy) assert.doesNotMatch(s, /alice|s3cret|password|username/, 'update 响应不得泄漏账密') assert.equal(r.proxy.has_auth, true, '但要能看出有账密') @@ -86,35 +86,35 @@ test('update: 响应里的 publicProxy 不含账密', async () => { test('getProxyByIdWithAuth: 拼出完整 URI，特殊字符转义', async () => { const pool = await makePool() - const p = seed(pool) + const p = await seed(pool) assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://alice:s3cret@1.2.3.4:1080') - pool.update(p.id, { username: 'u@x', password: 'p:w/d' }) + await pool.update(p.id, { username: 'u@x', password: 'p:w/d' }) assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://u%40x:p%3Aw%2Fd@1.2.3.4:1080') - pool.update(p.id, { username: '' }) + await pool.update(p.id, { username: '' }) assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://1.2.3.4:1080', '无账密时不带 @') }) test('getProxyByIdWithAuth: 禁用/失效的代理仍可读回（getProxyForVm 做不到）', async () => { const pool = await makePool() - const p = seed(pool) - pool.bind(p.id, 'vm-01') + const p = await seed(pool) + await pool.bind(p.id, 'vm-01') assert.ok(pool.getProxyForVm('vm-01'), '前提：启用时 getProxyForVm 能取到') - pool.setEnabled(p.id, false) + await pool.setEnabled(p.id, false) assert.equal(pool.getProxyForVm('vm-01'), null, 'getProxyForVm 对禁用代理返回 null') assert.equal(pool.getProxyByIdWithAuth(p.id).url, 'socks5://alice:s3cret@1.2.3.4:1080', '新方法不受影响') }) test('snapshot 回归：列表接口永不返回账密', async () => { const pool = await makePool() - seed(pool) + await seed(pool) const s = JSON.stringify(pool.snapshot()) assert.doesNotMatch(s, /alice|s3cret|socks5:\/\//, 'GET /proxies 的载荷不得含账密') }) test('update: 只改代理名称不动连接字段，也不要求重载槽位；重开后名称仍在', async () => { const pool = await makePool() - const p = seed(pool) - const r = pool.update(p.id, { label: ' 东京-2 号机 ' }) + const p = await seed(pool) + const r = await pool.update(p.id, { label: ' 东京-2 号机 ' }) assert.equal(r.ok, true) assert.equal(r.connection_changed, false) assert.equal(r.proxy.label, '东京-2 号机') @@ -126,17 +126,17 @@ test('update: 只改代理名称不动连接字段，也不要求重载槽位； test('update: 代理名称传空串清除；超长或非字符串被拒', async () => { const pool = await makePool() - const p = seed(pool) - pool.update(p.id, { label: 'a' }) - assert.equal(pool.update(p.id, { label: '' }).proxy.label, null) - assert.equal(pool.update(p.id, { label: 'x'.repeat(65) }).error, 'label_too_long') - assert.equal(pool.update(p.id, { label: 123 }).error, 'invalid_label') + const p = await seed(pool) + await pool.update(p.id, { label: 'a' }) + assert.equal((await pool.update(p.id, { label: '' })).proxy.label, null) + assert.equal((await pool.update(p.id, { label: 'x'.repeat(65) })).error, 'label_too_long') + assert.equal((await pool.update(p.id, { label: 123 })).error, 'invalid_label') }) test('update: 连接字段与名称同改时报告 connection_changed', async () => { const pool = await makePool() - const p = seed(pool) - const r = pool.update(p.id, { port: 1082, label: 'box' }) + const p = await seed(pool) + const r = await pool.update(p.id, { port: 1082, label: 'box' }) assert.equal(r.connection_changed, true) assert.equal(p.port, 1082) assert.equal(p.label, 'box') @@ -144,9 +144,9 @@ test('update: 连接字段与名称同改时报告 connection_changed', async () test('labelOf 只给名称，不带账密；未知代理返回 null', async () => { const pool = await makePool() - const p = seed(pool) + const p = await seed(pool) assert.equal(pool.labelOf(p.id), null) - pool.update(p.id, { label: 'tw-box' }) + await pool.update(p.id, { label: 'tw-box' }) assert.equal(pool.labelOf(p.id), 'tw-box') assert.equal(pool.labelOf('px-missing'), null) }) diff --git a/test/unit/proxy-resolve.test.mjs b/test/unit/proxy-resolve.test.mjs index a7888755f..cf7f601c9 100644 --- a/test/unit/proxy-resolve.test.mjs +++ b/test/unit/proxy-resolve.test.mjs @@ -14,9 +14,9 @@ function tmpDir() { test('prefers pool credentials over host-only vm.proxy', async () => { const pool = await new ProxyPool({ dataDir: tmpDir() }).init() - pool.importLines('socks5://user:s3cret@10.0.0.5:1080') + await pool.importLines('socks5://user:s3cret@10.0.0.5:1080') const id = pool.snapshot().proxies[0].id - pool.bind(id, 'vm-02') + await pool.bind(id, 'vm-02') pool.stopScheduler() const resolved = resolveImportProxy({ vm: { id: 'vm-02', proxy: { id, host: '10.0.0.5', port: 1080 } }, @@ -28,10 +28,10 @@ test('prefers pool credentials over host-only vm.proxy', async () => { test('blocks import when bound proxy is fail or dead', async () => { const pool = await new ProxyPool({ dataDir: tmpDir() }).init() - pool.importLines('10.0.0.6:1080') + await pool.importLines('10.0.0.6:1080') const id = pool.snapshot().proxies[0].id - pool.bind(id, 'vm-03') - pool.setEnabled(id, false) + await pool.bind(id, 'vm-03') + await pool.setEnabled(id, false) pool.stopScheduler() const resolved = resolveImportProxy({ vm: { id: 'vm-03', proxy: { id, host: '10.0.0.6', port: 1080, url: 'socks5h://10.0.0.6:1080' } }, From 7197b4650f1e43d37d5afcdb1024518c747288bb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:25:06 +0800 Subject: [PATCH 112/473] test: reproduce reordered account observation tickets --- test/unit/shared-account-quota.test.mjs | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/unit/shared-account-quota.test.mjs b/test/unit/shared-account-quota.test.mjs index 188387671..f2d2b42f9 100644 --- a/test/unit/shared-account-quota.test.mjs +++ b/test/unit/shared-account-quota.test.mjs @@ -394,3 +394,33 @@ test('an older refresh finishing late cannot replace a newer committed remote sn await older assert.equal(left.repo.get('shared-account').max_concurrency, 29) }) + +test('ticket allocation preserves local invocation order when an earlier PG allocation is delayed', async (t) => { + const { left } = await pair(t) + const allocate = left.persistence.beginObservation.bind(left.persistence) + let entered + let resume + const started = new Promise((resolve) => { + entered = resolve + }) + const gate = new Promise((resolve) => { + resume = resolve + }) + let calls = 0 + left.persistence.beginObservation = async () => { + if (++calls === 1) { + entered() + await gate + } + return allocate() + } + const older = left.beginObservation() + await started + const newer = left.beginObservation() + await Promise.resolve() + const callsBeforeRelease = calls + resume() + const [oldTicket, newTicket] = await Promise.all([older, newer]) + assert.equal(callsBeforeRelease, 1) + assert.ok(BigInt(oldTicket) < BigInt(newTicket)) +}) From 12437c3b8b851be4c55a76cfcdb4abb3ebc652fd Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:25:29 +0800 Subject: [PATCH 113/473] fix: preserve account observation ticket invocation order --- src/lib/pool/account-quota.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/pool/account-quota.mjs b/src/lib/pool/account-quota.mjs index 4b696ef91..0a6d5c7b0 100644 --- a/src/lib/pool/account-quota.mjs +++ b/src/lib/pool/account-quota.mjs @@ -68,6 +68,7 @@ export class AccountQuota { this.revisions = new Map() this.refreshStarted = 0 this.refreshPublished = 0 + this.observationTail = Promise.resolve() this.allocations = new Map() this.seedAccounts = accounts || [] this.repo = this._workingSet() @@ -186,7 +187,9 @@ export class AccountQuota { /** Reserve before remote I/O when the producer can identify when an observation starts. */ beginObservation() { - return this.persistence.beginObservation() + const observation = this.observationTail.then(() => this.persistence.beginObservation()) + this.observationTail = observation.catch(() => {}) + return observation } /** Re-bind to a fresh DB connection (after backup restore). */ From 26882a041729cc414eacbc9913320dd614f00c91 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:25:47 +0800 Subject: [PATCH 114/473] test(k8s): reproduce cached scheduling state overwrites --- test/unit/shared-scheduling-runtime.test.mjs | 208 +++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 test/unit/shared-scheduling-runtime.test.mjs diff --git a/test/unit/shared-scheduling-runtime.test.mjs b/test/unit/shared-scheduling-runtime.test.mjs new file mode 100644 index 000000000..537456cde --- /dev/null +++ b/test/unit/shared-scheduling-runtime.test.mjs @@ -0,0 +1,208 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SchedulingRuntime } from '../../src/lib/pool/scheduling-runtime.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function pair(t, seed = {}) { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + const left = await new SchedulingRuntime(db).init() + left.upsert({ account_id: 'runtime-account', vm_id: 'vm-old', status: 'ready', priority: 3, weight: 2, ...seed }) + await left.flush() + const right = await new SchedulingRuntime(other).init() + return { left, right, stored: () => left.persistence.get('runtime-account') } +} + +for (const producer of ['window', 'heartbeat']) { + test(`a stale ${producer} update preserves another controller's VM, priority and cooldown`, async (t) => { + const { left, right, stored } = await pair(t) + const until = Date.now() + 120_000 + left.upsert({ account_id: 'runtime-account', vm_id: 'vm-new', priority: 19, weight: 7 }) + left.markCooldown('runtime-account', { vmId: 'vm-new', until, reason: 'account_quota_exhausted' }) + await left.flush() + if (producer === 'window') + right.updateWindow('runtime-account', { vmId: 'vm-old', sessionWindowStatus: 'rejected' }) + else + right.upsert({ + account_id: 'runtime-account', + vm_id: 'vm-old', + worker_heartbeat_at: Date.now(), + worker_status: { ok: true }, + }) + await right.flush() + const state = await stored() + assert.equal(state.vm_id, 'vm-new') + assert.equal(state.priority, 19) + assert.equal(state.weight, 7) + assert.equal(state.cooldown_until, until) + assert.equal(state.rate_limit_reset_at, until) + }) +} + +test('different model cooldowns merge across independent runtime caches', async (t) => { + const { left, right, stored } = await pair(t) + const until = Date.now() + 90_000 + left.markCooldown('runtime-account', { vmId: 'vm-old', model: 'claude-a', until, reason: 'model-a-limit' }) + await left.flush() + right.markCooldown('runtime-account', { + vmId: 'vm-old', + model: 'claude-b', + until: until + 1000, + reason: 'model-b-limit', + }) + await right.flush() + const state = await stored() + assert.deepEqual(Object.keys(state.model_states).sort(), ['claude-a', 'claude-b']) + assert.equal(state.model_states['claude-a'].cooldown_until, until) + assert.equal(state.model_states['claude-b'].cooldown_until, until + 1000) +}) + +test("a late local expiry cannot clear another controller's extended account, hard or model cooldown", async (t) => { + const now = Date.now() + const { left, right, stored } = await pair(t, { + status: 'cooldown', + cooldown_until: now - 1, + cooldown_reason: 'account_quota_exhausted', + rate_limit_reset_at: now - 1, + overload_until: now - 1, + model_states: { 'claude-a': { cooldown_until: now - 1, reason: 'old' } }, + }) + const until = now + 120_000 + left.markCooldown('runtime-account', { vmId: 'vm-old', until, reason: 'account_quota_exhausted' }) + left.updateWindow('runtime-account', { overloadUntil: until }) + left.markCooldown('runtime-account', { vmId: 'vm-old', model: 'claude-a', until, reason: 'new' }) + await left.flush() + right.clearExpired(now) + await right.flush() + const state = await stored() + assert.equal(state.cooldown_until, until) + assert.equal(state.rate_limit_reset_at, until) + assert.equal(state.overload_until, until) + assert.equal(state.model_states['claude-a'].cooldown_until, until) +}) + +test('a stale auth clear preserves a new quota cooldown while explicit management clear removes the current state', async (t) => { + const { left, right, stored } = await pair(t, { + status: 'cooldown', + cooldown_until: Date.now() + 1000, + cooldown_reason: 'authentication_failed_after_refresh', + }) + const until = Date.now() + 90_000 + left.markCooldown('runtime-account', { vmId: 'vm-old', until, reason: 'account_quota_exhausted' }) + left.markCooldown('runtime-account', { vmId: 'vm-old', until, reason: 'model-limit', model: 'claude-a' }) + await left.flush() + right.clearAuthCooldown('runtime-account') + await right.flush() + assert.equal((await stored()).cooldown_until, until) + assert.equal((await stored()).cooldown_reason, 'account_quota_exhausted') + right.clearAccountCooldown('runtime-account') + await right.flush() + const cleared = await stored() + assert.equal(cleared.cooldown_until, null) + assert.equal(cleared.rate_limit_reset_at, null) + assert.deepEqual(cleared.model_states, {}) + assert.equal(cleared.priority, 3) +}) + +test('database observation tickets reject an older worker or allowed-window completion', async (t) => { + const { left, right, stored } = await pair(t) + const older = await left.beginObservation() + const newer = await right.beginObservation() + const until = Date.now() + 90_000 + right.upsert( + { + account_id: 'runtime-account', + vm_id: 'vm-old', + worker_status: { ok: true, generation: 2 }, + worker_heartbeat_at: 200, + }, + { observation: newer }, + ) + right.updateWindow('runtime-account', { + rateLimitResetAt: until, + sessionWindowStatus: 'rejected', + observation: newer, + }) + await right.flush() + left.upsert( + { + account_id: 'runtime-account', + vm_id: 'vm-old', + worker_status: { ok: false, generation: 1 }, + worker_heartbeat_at: 100, + }, + { observation: older }, + ) + left.updateWindow('runtime-account', { rateLimitResetAt: null, sessionWindowStatus: 'allowed', observation: older }) + await left.flush() + const state = await stored() + assert.deepEqual(state.worker_status, { ok: true, generation: 2 }) + assert.equal(state.worker_heartbeat_at, 200) + assert.equal(state.rate_limit_reset_at, until) + assert.equal(state.session_window_status, 'rejected') +}) + +test('runtime refresh publishes committed peer rows and preserves a later local mutation', async (t) => { + const { left, right } = await pair(t) + right.upsert({ account_id: 'runtime-account', vm_id: 'vm-new', priority: 21 }) + await right.flush() + await left.refresh() + assert.equal(left.get('runtime-account').vm_id, 'vm-new') + assert.equal(left.get('runtime-account').priority, 21) + const entered = deferred() + const resume = deferred() + const list = left.persistence.list.bind(left.persistence) + left.persistence.list = async () => { + const rows = await list() + entered.resolve() + await resume.promise + return rows + } + const pending = left.refresh() + try { + await entered.promise + left.upsert({ account_id: 'runtime-account', vm_id: 'vm-new', priority: 31 }) + await left.flush() + } finally { + resume.resolve() + await pending + } + assert.equal(left.get('runtime-account').priority, 31) +}) + +test('an older runtime refresh finishing late cannot replace a newer committed peer snapshot', async (t) => { + const { left, right } = await pair(t) + const entered = deferred() + const resume = deferred() + const list = left.persistence.list.bind(left.persistence) + let first = true + left.persistence.list = async () => { + const rows = await list() + if (first) { + first = false + entered.resolve() + await resume.promise + } + return rows + } + const pending = left.refresh() + try { + await entered.promise + right.upsert({ account_id: 'runtime-account', vm_id: 'vm-new', priority: 41 }) + await right.flush() + await left.refresh() + } finally { + resume.resolve() + await pending + } + assert.equal(left.get('runtime-account').priority, 41) +}) From a4beb970fb4b6c9ba125e36047549748bc588e5c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:25:55 +0800 Subject: [PATCH 115/473] feat(k8s): refresh shared state and order account observations per attempt --- src/lib/admin/panel-routes.mjs | 2 +- src/lib/pool/rate-limit-service.mjs | 1 + src/lib/protocol/handle-protocol.mjs | 27 +++++++++++++++++++--- src/server.mjs | 1 + test/e2e/kubernetes-admission.e2e.test.mjs | 16 +++++++++++++ 5 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index c16a94514..3cc365948 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -3547,7 +3547,7 @@ export function createPanelHandler(ctx) { try { const repo = ctx.accountQuota?.repo const row = repo && typeof repo.get === 'function' ? await repo.get(id) : null - if (row && typeof repo.save === 'function') await repo.save({ ...row, email: null, name: id }) + if (row && typeof repo.patch === 'function') repo.patch(id, { email: null, name: id }) } catch {} const saved = getVm(cfg.paths.project, id) || vm return json( diff --git a/src/lib/pool/rate-limit-service.mjs b/src/lib/pool/rate-limit-service.mjs index 17dcb2a9d..1a19df66a 100755 --- a/src/lib/pool/rate-limit-service.mjs +++ b/src/lib/pool/rate-limit-service.mjs @@ -141,6 +141,7 @@ export class RateLimitService { exhausted: true, status: 429, countRequest: false, + observation: result.accountObservation, }) } catch {} if (fallback) this.requestUsageProbe({ accountId, vmId }) diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 0b4cab7b0..c9330873d 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -445,6 +445,7 @@ export function createHandleProtocol(deps) { requestContext = null, routing = {}, noGoFallback = false, + accountObservation = null, }) { const assembler = createClaudeMessageAssembler() const workerResult = await dispatchStreamInference({ @@ -472,6 +473,7 @@ export function createHandleProtocol(deps) { }, }) Object.assign(workerResult, mergeAssembledAssistantHop(workerResult, assembler.message)) + workerResult.accountObservation = accountObservation if (workerResult?.body) { workerResult.body = restoreToolNames(workerResult.body, toolNames) } @@ -1415,7 +1417,16 @@ export function createHandleProtocol(deps) { }) noteCachePrefix(selected, attemptSessionId, hopBody) logBag.outbound_session_id = sessionIdForLog(sessionIdFromOutboundBody(hopBody) || attemptSessionId) - return { body: hopBody, meta: { toolNames: {}, sessionId: attemptSessionId, cliHop: true, requestContext } } + return { + body: hopBody, + meta: { + toolNames: {}, + sessionId: attemptSessionId, + cliHop: true, + requestContext, + accountObservation: await accountQuota?.beginObservation?.(), + }, + } } cacheTtl = requestedCacheTtl @@ -1491,7 +1502,14 @@ export function createHandleProtocol(deps) { }) noteCachePrefix(selected, attemptSessionId, prepared.body) logBag.outbound_session_id = sessionIdForLog(sessionIdFromOutboundBody(prepared.body) || attemptSessionId) - return { body: prepared.body, meta: { toolNames: prepared.toolNames, sessionId: attemptSessionId } } + return { + body: prepared.body, + meta: { + toolNames: prepared.toolNames, + sessionId: attemptSessionId, + accountObservation: await accountQuota?.beginObservation?.(), + }, + } }, callAttempt: async ({ candidate, body, attemptMeta, deliveryMode: attemptDelivery, signal, onCommit }) => { if (!clientStream) { @@ -1511,6 +1529,7 @@ export function createHandleProtocol(deps) { want1m, routing: getRouting(), noGoFallback: !!pinVmId, + accountObservation: attemptMeta?.accountObservation, }) } let state @@ -1539,7 +1558,7 @@ export function createHandleProtocol(deps) { if (anthropicKeepalive) activeAttemptCommit = onCommit keepalive.arm() try { - return await dispatchStreamInference({ + const workerResult = await dispatchStreamInference({ exec: candidate.exec, onExecutionStart: candidate.bindExecution, onExecutionEnd: candidate.completeExecution, @@ -1593,6 +1612,7 @@ export function createHandleProtocol(deps) { writeChunks(claudeSSELineToResponsesEvents(line, state)) }, }) + return { ...workerResult, accountObservation: attemptMeta?.accountObservation } } finally { if (anthropicKeepalive) { activeAttemptCommit = null @@ -1663,6 +1683,7 @@ export function createHandleProtocol(deps) { exhausted, status: result.status, countRequest: !healthReal, + observation: result?.accountObservation, }) const pool = typeof deps.getPoolScheduler === 'function' ? deps.getPoolScheduler() : deps.poolScheduler // Live `5h-status` on every response; `allowed` is the only early unblock. diff --git a/src/server.mjs b/src/server.mjs index d4f2c0c4c..6cf24f374 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -817,6 +817,7 @@ async function requireAuth(req, res) { refreshModelPolicy(snapshot.model_policy) refreshGptModelPolicy(snapshot.gpt_model_policy) invalidatePriceCatalog() + await accountQuota.refresh?.() await proxyPool.refresh?.() } // Panel session diff --git a/test/e2e/kubernetes-admission.e2e.test.mjs b/test/e2e/kubernetes-admission.e2e.test.mjs index b92f35b37..e64ad2735 100644 --- a/test/e2e/kubernetes-admission.e2e.test.mjs +++ b/test/e2e/kubernetes-admission.e2e.test.mjs @@ -68,3 +68,19 @@ test('Kubernetes request authentication reloads a model policy changed by anothe await gw.stop() } }) + +test('Kubernetes panel proxy reads reload a record committed by another process connection', async () => { + const gw = await startGateway({ env }) + try { + const added = await api(gw, 'POST', '/api/panel/proxies/import', { body: { text: '127.0.0.1:1080' } }) + assert.equal(added.status, 200, added.text) + const id = (added.json.data || added.json).items[0].id + await gw.db.prepare('UPDATE proxies SET label = $2 WHERE id = $1').run(id, 'external-label') + const listed = await api(gw, 'GET', '/api/panel/proxies') + assert.equal(listed.status, 200, listed.text) + const current = (listed.json.data || listed.json).proxies.find((row) => row.id === id) + assert.equal(current?.label, 'external-label') + } finally { + await gw.stop() + } +}) From e2d02afbb32b870f328f572652f7a5ea17ec70ce Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:27:32 +0800 Subject: [PATCH 116/473] test: reproduce stale account window cleanup overwrites --- test/unit/shared-account-quota.test.mjs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/unit/shared-account-quota.test.mjs b/test/unit/shared-account-quota.test.mjs index f2d2b42f9..7550ed99f 100644 --- a/test/unit/shared-account-quota.test.mjs +++ b/test/unit/shared-account-quota.test.mjs @@ -424,3 +424,24 @@ test('ticket allocation preserves local invocation order when an earlier PG allo assert.equal(callsBeforeRelease, 1) assert.ok(BigInt(oldTicket) < BigInt(newTicket)) }) + +test('elapsed-window cleanup from a stale replica cannot clear a newer window with the same utilization', async (t) => { + const { left, right } = await pair(t) + left.ingestHeaders('shared-account', { + 'anthropic-ratelimit-unified-5h-utilization': '0.9', + 'anthropic-ratelimit-unified-5h-reset': new Date(Date.now() - 60_000).toISOString(), + }) + await left.flush() + await right.refresh() + const newReset = new Date(Date.now() + 3600_000).toISOString() + left.ingestHeaders('shared-account', { + 'anthropic-ratelimit-unified-5h-utilization': '0.9', + 'anthropic-ratelimit-unified-5h-reset': newReset, + }) + await left.flush() + right.persistEffectiveWindows('shared-account') + await right.flush() + const saved = await left.persistence.get('shared-account') + assert.equal(saved.unified.headers['5h'].utilization, 0.9) + assert.equal(saved.unified.headers['5h'].reset, newReset) +}) From 2ccae1994f720eb32bd19dca5a7fd46810a28cb9 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:28:03 +0800 Subject: [PATCH 117/473] test: reject stale cross-replica sticky mutations --- test/unit/sticky-router-shared.test.mjs | 116 ++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 test/unit/sticky-router-shared.test.mjs diff --git a/test/unit/sticky-router-shared.test.mjs b/test/unit/sticky-router-shared.test.mjs new file mode 100644 index 000000000..62fa8a45f --- /dev/null +++ b/test/unit/sticky-router-shared.test.mjs @@ -0,0 +1,116 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { StickyRouter } from '../../src/lib/pool/sticky-router.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' + +async function replicas() { + const db = getDb() + const peer = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: db.schema }) + const config = { sticky: { enabled: true, ttl_seconds: 60 } } + return Promise.all([ + new StickyRouter({ db, config, shared: true }).init(), + new StickyRouter({ db: peer, config, shared: true }).init(), + ]) +} + +// Keep the regression runnable before introducing the asynchronous shared API. +async function bind(router, key, value, options) { + if (router.bindShared) return router.bindShared(key, value, options) + const result = router.bind(key, value, options) + await router.flush() + return result +} + +async function unbind(router, key, options) { + if (router.unbindShared) return router.unbindShared(key, options) + const result = router.unbind(key, options) + await router.flush() + return result +} + +async function rebindFamily(router, key, value, options) { + if (router.rebindFamilyShared) return router.rebindFamilyShared(key, value, options) + const result = router.rebindFamily(key, value, options) + await router.flush() + return result +} + +test('a delayed bind cannot overwrite a family migration committed by another connection', async () => { + const [first, second] = await replicas() + const key = 'family2:shared-family' + await bind(first, key, { accountId: 'account-old', vmId: 'vm-old', sessionId: 'native-old' }) + await second.reload() + const previous = second.resolve(key) + await rebindFamily(first, key, { accountId: 'account-new', vmId: 'vm-new' }) + const migrated = await first.persistence.get(key) + + await bind( + second, + key, + { accountId: previous.accountId, vmId: previous.vmId, sessionId: previous.sessionId }, + { + ifGeneration: previous.generation, + }, + ) + const stored = await first.persistence.get(key) + assert.equal(stored.account_id, 'account-new') + assert.equal(stored.vm_id, 'vm-new') + assert.equal(stored.generation, migrated.generation) +}) + +test('a delayed generation-conditional unbind cannot delete a newer family binding', async () => { + const [first, second] = await replicas() + const key = 'family2:unbind-family' + await bind(first, key, { accountId: 'account-old', vmId: 'vm-old' }) + await second.reload() + const previous = second.resolve(key) + await rebindFamily(first, key, { accountId: 'account-new', vmId: 'vm-new' }) + + await unbind(second, key, { ifGeneration: previous.generation }) + const stored = await first.persistence.get(key) + assert.ok(stored, 'the binding committed after the observed generation must survive') + assert.equal(stored.vm_id, 'vm-new') +}) + +test('a delayed bind cannot resurrect a sticky key unbound by another connection', async () => { + const [first, second] = await replicas() + const key = 'sess:unbound-session' + await bind(first, key, { accountId: 'account', vmId: 'vm', sessionId: 'native-before-delete' }) + await second.reload() + const previous = second.resolve(key) + await unbind(first, key, { ifGeneration: previous.generation }) + + await bind( + second, + key, + { accountId: 'account', vmId: 'vm', sessionId: 'native-before-delete' }, + { + ifGeneration: previous.generation, + }, + ) + assert.equal(await first.persistence.get(key), null, 'an unbind must invalidate every older binding token') +}) + +test('delete and recreate never reuse a generation accepted by a stale replica', async () => { + const [first, second] = await replicas() + const key = 'sess:recreated-session' + await bind(first, key, { accountId: 'account', vmId: 'vm-old' }) + await second.reload() + const previous = second.resolve(key) + await unbind(first, key, { ifGeneration: previous.generation }) + await bind(first, key, { accountId: 'account', vmId: 'vm-new' }) + const recreated = await first.persistence.get(key) + assert.notEqual(recreated.generation, previous.generation, 'a deleted binding token must never become current again') +}) + +test('a replica that observed no binding cannot replace the committed native session ID', async () => { + const [first, second] = await replicas() + const key = 'sess:one-native-session' + assert.equal(second.resolve(key), null) + await bind(first, key, { accountId: 'account', vmId: 'vm', sessionId: 'native-first' }, { ifGeneration: 0 }) + await bind(second, key, { accountId: 'account', vmId: 'vm', sessionId: 'native-second' }, { ifGeneration: 0 }) + + await second.reload() + assert.equal(second.resolve(key).sessionId, 'native-first') + assert.equal((await first.persistence.get(key)).session_id, 'native-first') +}) From 1c6e04c2cf3e3cbaf9870d57f2aa2e678ea4d120 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:28:20 +0800 Subject: [PATCH 118/473] fix: compare account quota windows before stale cleanup --- src/lib/db/repos/accounts-repo.mjs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/lib/db/repos/accounts-repo.mjs b/src/lib/db/repos/accounts-repo.mjs index acd2a0858..570ee9a3a 100644 --- a/src/lib/db/repos/accounts-repo.mjs +++ b/src/lib/db/repos/accounts-repo.mjs @@ -66,6 +66,15 @@ export function accountChanges(before, after, { paths = [] } = {}) { observationChanges(before[field], after[field], [field], observations) } for (const path of paths) touchObservation(valueAt(before, path), valueAt(after, path), path, observations) + for (const change of observations) { + const path = change.path + // Reset/status/utilization describe one window. A stale cleanup must compare the whole window. + const window = + path[0] === 'unified' && ['headers', 'official'].includes(path[1]) ? path.slice(0, 3) : path.slice(0, 2) + change.guardPath = + path[0] === 'unified' && ['5h', '7d', '7d_oi', 'last_probe', 'fable'].includes(window.at(-1)) ? window : path + change.guardBefore = valueAt(before, change.guardPath) + } return { fields, deltas, observations } } @@ -290,8 +299,13 @@ export class AccountsRepo { const extra = parse(row.extra, {}) || {} const versions = row.observation_versions || {} const probeChanged = probeBase !== undefined && !isDeepStrictEqual(extra.unified?.last_probe ?? null, probeBase) + const applicable = changes.map( + (change) => + !conditional || + isDeepStrictEqual(valueAt(extra, change.guardPath || change.path), change.guardBefore ?? change.before), + ) let changed = false - for (const change of changes) { + for (const [index, change] of changes.entries()) { if (!['unified', 'last_blocked', 'last_cli_rate_limit'].includes(change.path[0])) { throw new Error('unsupported account observation') } @@ -302,7 +316,7 @@ export class AccountsRepo { ) continue if (probeChanged && change.path[0] === 'unified' && change.path[1] === 'last_probe') continue - if (conditional && !isDeepStrictEqual(valueAt(extra, change.path), change.before)) continue + if (!applicable[index]) continue setAt(extra, change.path, change.value) versions[JSON.stringify(change.path)] = sequence.toString() changed = true From 6590bfbb3e4df6ab99d0451da0d7e5a657c526b5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:34:24 +0800 Subject: [PATCH 119/473] test(pool): cover observation context and initial runtime binding --- test/unit/shared-scheduling-runtime.test.mjs | 34 ++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/unit/shared-scheduling-runtime.test.mjs b/test/unit/shared-scheduling-runtime.test.mjs index 537456cde..df26a30d0 100644 --- a/test/unit/shared-scheduling-runtime.test.mjs +++ b/test/unit/shared-scheduling-runtime.test.mjs @@ -2,6 +2,7 @@ import test from 'node:test' import assert from 'node:assert/strict' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { SchedulingRuntime } from '../../src/lib/pool/scheduling-runtime.mjs' +import { AccountsRepo } from '../../src/lib/db/repos/accounts-repo.mjs' function deferred() { let resolve @@ -22,6 +23,16 @@ async function pair(t, seed = {}) { return { left, right, stored: () => left.persistence.get('runtime-account') } } +test('an existing account without a VM can acquire its initial runtime binding from observation context', async () => { + const db = getDb() + await new AccountsRepo(db).insert({ account_id: 'unbound-runtime-account' }) + const runtime = await new SchedulingRuntime(db).init() + runtime.updateWindow('unbound-runtime-account', { vmId: 'vm-initial', sessionWindowStatus: 'allowed' }) + await runtime.flush() + assert.equal((await runtime.persistence.get('unbound-runtime-account')).vm_id, 'vm-initial') + assert.equal(runtime.get('unbound-runtime-account').session_window_status, 'allowed') +}) + for (const producer of ['window', 'heartbeat']) { test(`a stale ${producer} update preserves another controller's VM, priority and cooldown`, async (t) => { const { left, right, stored } = await pair(t) @@ -151,6 +162,29 @@ test('database observation tickets reject an older worker or allowed-window comp assert.equal(state.session_window_status, 'rejected') }) +test('an observational VM context cannot rebind the authoritative account and quota clearing compares the cached cooldown', async (t) => { + const until = Date.now() + 90_000 + const { left, right, stored } = await pair(t, { + status: 'cooldown', + cooldown_until: until, + cooldown_reason: 'quota_guard', + }) + left.upsert({ account_id: 'runtime-account', vm_id: 'vm-new' }) + left.markCooldown('runtime-account', { vmId: 'vm-new', until: until + 1000, reason: 'quota_guard' }) + await left.flush() + right.clearQuotaCooldown('runtime-account') + await right.flush() + assert.equal((await stored()).cooldown_until, until + 1000) + const observation = await right.beginObservation() + right.upsert({ account_id: 'runtime-account', worker_status: { ok: true } }, { vmId: 'vm-old', observation }) + await right.flush() + assert.equal((await stored()).vm_id, 'vm-new') + await right.refresh() + assert.equal(right.clearQuotaCooldown('runtime-account'), true) + await right.flush() + assert.equal((await stored()).cooldown_until, null) +}) + test('runtime refresh publishes committed peer rows and preserves a later local mutation', async (t) => { const { left, right } = await pair(t) right.upsert({ account_id: 'runtime-account', vm_id: 'vm-new', priority: 21 }) From a163d7d409b157de53007167f556d1aea6cf0719 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:36:01 +0800 Subject: [PATCH 120/473] fix(k8s): patch shared runtime fields with ordered observations --- src/lib/db/repos/account-runtime-repo.mjs | 491 +++++++++++----------- src/lib/pool/scheduling-runtime.mjs | 360 +++++++++------- 2 files changed, 458 insertions(+), 393 deletions(-) diff --git a/src/lib/db/repos/account-runtime-repo.mjs b/src/lib/db/repos/account-runtime-repo.mjs index 41b43eee3..a95d2cd03 100644 --- a/src/lib/db/repos/account-runtime-repo.mjs +++ b/src/lib/db/repos/account-runtime-repo.mjs @@ -11,6 +11,7 @@ */ import { getDb, withTransaction } from '../database.mjs' +import { isDeepStrictEqual } from 'node:util' function parse(value, fallback = null) { if (value == null) return fallback @@ -38,7 +39,7 @@ function mergeState(rt, acc) { if (!rt && !acc) return null return { account_id: rt?.account_id ?? acc?.id, - vm_id: rt?.vm_id ?? acc?.vm_id ?? null, + vm_id: acc?.vm_id ?? rt?.vm_id ?? null, status: rt?.status || 'unknown', priority: Number(acc?.priority) || 0, weight: Math.max(0, Number(acc?.load_factor) || 0), @@ -56,303 +57,303 @@ function mergeState(rt, acc) { session_window_start: toMs(acc?.session_window_start), session_window_end: toMs(acc?.session_window_end), session_window_status: acc?.session_window_status || null, - updated_at: rt?.updated_at ?? acc?.updated_at ?? null, + updated_at: acc?.updated_at ?? rt?.updated_at ?? null, } } +const ACCOUNT_FIELDS = { + vm_id: 'vm_id', + priority: 'priority', + weight: 'load_factor', + last_used_at: 'last_used_at', + cooldown_until: 'temp_unschedulable_until', + cooldown_reason: 'temp_unschedulable_reason', + rate_limited_at: 'rate_limited_at', + rate_limit_reset_at: 'rate_limit_reset_at', + overload_until: 'overload_until', + session_window_start: 'session_window_start', + session_window_end: 'session_window_end', + session_window_status: 'session_window_status', +} +const RUNTIME_FIELDS = { + vm_id: 'vm_id', + status: 'status', + credential_generation: 'credential_generation', + refresh_status: 'refresh_status', + worker_heartbeat_at: 'worker_heartbeat_at', + worker_status: 'worker_status_json', +} +const TIMES = new Set([ + 'last_used_at', + 'cooldown_until', + 'rate_limited_at', + 'rate_limit_reset_at', + 'overload_until', + 'session_window_start', + 'session_window_end', +]) + +function overlaps(left, right) { + return left.slice(0, Math.min(left.length, right.length)).every((key, i) => key === right[i]) +} + +function accepts(versions, path, sequence) { + // Share 008's sequence and row-locked ledger with account observations; + // the runtime prefix keeps their field versions independent. + return !Object.entries(versions).some(([key, value]) => overlaps(JSON.parse(key), path) && BigInt(value) > sequence) +} + export class AccountRuntimeRepo { constructor(db = getDb()) { this.db = db - this._getRt = db.prepare('SELECT * FROM account_runtime WHERE account_id = $1') - this._getAcc = db.prepare(` - SELECT id, vm_id, priority, load_factor, last_used_at, - temp_unschedulable_until, temp_unschedulable_reason, - rate_limited_at, rate_limit_reset_at, overload_until, - session_window_start, session_window_end, session_window_status, - updated_at - FROM accounts WHERE id = $1 - `) - this._listIds = db.prepare(` - SELECT account_id FROM account_runtime - UNION SELECT id FROM accounts - ORDER BY 1 - `) - this._ensureAcc = db.prepare(` - INSERT INTO accounts (id, vm_id, name, extra, created_at, updated_at) - VALUES ($1, $2, $3, '{}', $4, $5) - ON CONFLICT(id) DO NOTHING - `) - this._linkDefaultGroup = db.prepare(` - INSERT INTO account_groups (account_id, group_id, priority, created_at) - VALUES ($1, 1, 50, $2) - ON CONFLICT(account_id, group_id) DO NOTHING - `) - this._updateAcc = db.prepare(` - UPDATE accounts SET - vm_id = $1, - priority = $2, load_factor = $3, last_used_at = $4, - temp_unschedulable_until = $5, temp_unschedulable_reason = $6, - rate_limited_at = $7, rate_limit_reset_at = $8, overload_until = $9, - session_window_start = $10, session_window_end = $11, session_window_status = $12, - updated_at = $13 - WHERE id = $14 - `) - this._upsertRt = db.prepare(` - INSERT INTO account_runtime ( - account_id, vm_id, status, model_states_json, credential_generation, - refresh_status, worker_heartbeat_at, worker_status_json, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT(account_id) DO UPDATE SET - vm_id = excluded.vm_id, - status = excluded.status, - model_states_json = excluded.model_states_json, - credential_generation = excluded.credential_generation, - refresh_status = excluded.refresh_status, - worker_heartbeat_at = excluded.worker_heartbeat_at, - worker_status_json = excluded.worker_status_json, - updated_at = excluded.updated_at - `) - this._removeRt = db.prepare('DELETE FROM account_runtime WHERE account_id = $1') + } + + async beginObservation() { + return (await this.db.prepare("SELECT nextval('account_observation_seq') AS observation").get()).observation } async get(accountId) { - const rt = await this._getRt.get(accountId) - const acc = await this._getAcc.get(accountId) - if (!rt && !acc) return null - return mergeState(rt, acc) + const row = await this.db + .prepare(`SELECT to_jsonb(r) AS runtime, to_jsonb(a) AS account + FROM account_runtime r FULL OUTER JOIN accounts a ON a.id = r.account_id + WHERE COALESCE(r.account_id, a.id) = $1`) + .get(accountId) + return row ? mergeState(row.runtime, row.account) : null } async list() { - const ids = await this._listIds.all() - return (await Promise.all(ids.map((r) => this.get(r.account_id ?? r.id)))).filter(Boolean) + return ( + await this.db + .prepare(`SELECT to_jsonb(r) AS runtime, to_jsonb(a) AS account + FROM account_runtime r FULL OUTER JOIN accounts a ON a.id = r.account_id + ORDER BY COALESCE(r.account_id, a.id)`) + .all() + ).map((row) => mergeState(row.runtime, row.account)) } - async upsert(input = {}) { - return withTransaction(this.db, async () => { - await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(input.account_id)) - - if (!input.account_id || !input.vm_id) return null - const current = (await this.get(input.account_id)) || {} - const next = { - ...current, - ...input, - status: input.status ?? current.status ?? 'unknown', - priority: Number(input.priority ?? current.priority ?? 0), - weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), - model_states: input.model_states ?? current.model_states ?? {}, - updated_at: new Date().toISOString(), - } - await this._ensureAcc.run(next.account_id, next.vm_id, next.account_id, next.updated_at, next.updated_at) - await this._linkDefaultGroup.run(next.account_id, next.updated_at) - await this._updateAcc.run( - next.vm_id, - next.priority, - next.weight, - toIso(next.last_used_at), - toIso(next.cooldown_until), - next.cooldown_reason ?? null, - toIso(next.rate_limited_at), - toIso(next.rate_limit_reset_at), - toIso(next.overload_until), - toIso(next.session_window_start), - toIso(next.session_window_end), - next.session_window_status ?? null, - next.updated_at, - next.account_id, - ) - await this._upsertRt.run( - next.account_id, - next.vm_id, - next.status, - JSON.stringify(next.model_states || {}), - next.credential_generation ?? 0, - next.refresh_status ?? null, - next.worker_heartbeat_at ?? null, - next.worker_status != null ? JSON.stringify(next.worker_status) : null, - next.updated_at, - ) - return await this.get(next.account_id) - }) + async lockAccount(accountId, vmId) { + if (vmId) { + const now = new Date().toISOString() + await this.db + .prepare(`INSERT INTO accounts (id, vm_id, name, extra, priority, load_factor, created_at, updated_at) + VALUES ($1,$2,$1,'{}',0,1,$3,$3) ON CONFLICT(id) DO NOTHING`) + .run(accountId, vmId, now) + await this.db + .prepare(`INSERT INTO account_groups (account_id, group_id, priority, created_at) + VALUES ($1,1,50,$2) ON CONFLICT(account_id,group_id) DO NOTHING`) + .run(accountId, now) + } + const account = await this.db.prepare('SELECT * FROM accounts WHERE id = $1 FOR UPDATE').get(accountId) + if (account && !account.vm_id && vmId) { + return this.db + .prepare('UPDATE accounts SET vm_id = $2 WHERE id = $1 AND vm_id IS NULL RETURNING *') + .get(accountId, vmId) + } + return account } - /** - * Structured rate-limit / session-window write-through - * (sub2api account.rate_limited_at / rate_limit_reset_at / overload_until / - * session_window_* counterpart). Only provided fields are updated. - */ - async updateWindow( - accountId, - { - vmId = null, - rateLimitedAt, - rateLimitResetAt, - overloadUntil, - sessionWindowStart, - sessionWindowEnd, - sessionWindowStatus, - } = {}, - ) { + async patch(accountId, fields, { vmId, observation, modelPatch, clearModels = false, expected, condition } = {}) { + const sequence = BigInt(await (observation ?? this.beginObservation())) return withTransaction(this.db, async () => { - await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) - - const current = (await this.get(accountId)) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, + const account = await this.lockAccount(accountId, vmId) + if (!account) return null + const runtime = await this.db.prepare('SELECT * FROM account_runtime WHERE account_id = $1').get(accountId) + const current = mergeState(runtime, account) + if (!current.vm_id && !vmId && !fields.vm_id) return null + if (condition && !condition(current)) return null + if (expected && Object.entries(expected).some(([key, value]) => !isDeepStrictEqual(current[key], value))) + return null + const versions = account.observation_versions || {} + const acc = {} + const rt = {} + let accepted = false + for (const [field, value] of Object.entries(fields)) { + if (value === undefined || (!Object.hasOwn(ACCOUNT_FIELDS, field) && !Object.hasOwn(RUNTIME_FIELDS, field))) + continue + const path = ['runtime', field] + if (!accepts(versions, path, sequence)) continue + versions[JSON.stringify(path)] = sequence.toString() + accepted = true + if (Object.hasOwn(ACCOUNT_FIELDS, field)) acc[ACCOUNT_FIELDS[field]] = TIMES.has(field) ? toIso(value) : value + if (Object.hasOwn(RUNTIME_FIELDS, field)) + rt[RUNTIME_FIELDS[field]] = field === 'worker_status' && value != null ? JSON.stringify(value) : value + } + let models = { ...(current.model_states || {}) } + let modelsChanged = false + if (clearModels && accepts(versions, ['runtime', 'model_states'], sequence)) { + models = {} + versions[JSON.stringify(['runtime', 'model_states'])] = sequence.toString() + modelsChanged = true + } + for (const [model, value] of Object.entries(modelPatch || {})) { + const path = ['runtime', 'model_states', model] + if (!accepts(versions, path, sequence)) continue + if (value == null) delete models[model] + else Object.defineProperty(models, model, { value, enumerable: true, writable: true, configurable: true }) + versions[JSON.stringify(path)] = sequence.toString() + modelsChanged = true } - if (!current.vm_id && vmId) current.vm_id = vmId - if (!current.vm_id) return null - if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt - if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt - if (overloadUntil !== undefined) current.overload_until = overloadUntil - if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart - if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd - if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus - return await this.upsert(current) + if (modelsChanged) { + rt.model_states_json = JSON.stringify(models) + accepted = true + } + if (!accepted) return current + const now = new Date().toISOString() + await this.db + .prepare(`INSERT INTO account_runtime (account_id, vm_id, updated_at) + VALUES ($1,$2,$3) ON CONFLICT(account_id) DO NOTHING`) + .run(accountId, current.vm_id || vmId, now) + if (Object.keys(rt).length) { + const values = Object.values(rt) + const assignments = Object.keys(rt).map((key, i) => `${key} = $${i + 1}`) + values.push(now, accountId) + await this.db + .prepare(`UPDATE account_runtime SET ${assignments.join(', ')}, updated_at = $${values.length - 1} + WHERE account_id = $${values.length}`) + .run(...values) + } + const values = [...Object.values(acc), JSON.stringify(versions), now, accountId] + const assignments = Object.keys(acc).map((key, i) => `${key} = $${i + 1}`) + assignments.push(`observation_versions = $${values.length - 2}::jsonb`, `updated_at = $${values.length - 1}`) + await this.db.prepare(`UPDATE accounts SET ${assignments.join(', ')} WHERE id = $${values.length}`).run(...values) + return this.get(accountId) }) } - async markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { - return withTransaction(this.db, async () => { - await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + async upsert(input = {}, options = {}) { + if (!input.account_id || (input.vm_id !== undefined && !input.vm_id)) return null + const fields = { ...input } + if (input.priority !== undefined) fields.priority = Number(input.priority) || 0 + if (input.weight !== undefined) fields.weight = Math.max(0, Number(input.weight) || 0) + if (input.credential_generation !== undefined) + fields.credential_generation = Number(input.credential_generation) || 0 + return this.patch(input.account_id, fields, { vmId: input.vm_id, modelPatch: input.model_states, ...options }) + } - const current = (await this.get(accountId)) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, - } - if (model) { - current.model_states = { ...(current.model_states || {}) } - current.model_states[model] = { - status, - cooldown_until: Number(until) || null, - reason: reason || null, - updated_at: new Date().toISOString(), - } - } else { - current.status = status - current.cooldown_until = Number(until) || null - current.cooldown_reason = reason || null - // Structured mirror of the protocol-level limit (sub2api account columns). - // overload_until is only the 529 hard column (RateLimitService.handle529). - // A 502 failover also uses reason provider_overloaded for 15s; writing it - // here paints every hopped VM as 过载冷却 and removes them from the pool. - if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { - current.rate_limited_at = Date.now() - current.rate_limit_reset_at = Number(until) || null - } - } - return await this.upsert(current) - }) + async updateWindow(accountId, { vmId = null, observation, ...input } = {}) { + const fields = {} + for (const [key, field] of Object.entries({ + rateLimitedAt: 'rate_limited_at', + rateLimitResetAt: 'rate_limit_reset_at', + overloadUntil: 'overload_until', + sessionWindowStart: 'session_window_start', + sessionWindowEnd: 'session_window_end', + sessionWindowStatus: 'session_window_status', + })) { + if (input[key] !== undefined) fields[field] = input[key] + } + return this.patch(accountId, fields, { vmId, observation }) + } + + async markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown', observation } = {}) { + const cooldown = Number(until) || null + if (model) + return this.patch( + accountId, + {}, + { + vmId, + observation, + modelPatch: { + [model]: { status, cooldown_until: cooldown, reason: reason || null, updated_at: new Date().toISOString() }, + }, + }, + ) + return this.patch( + accountId, + { + status, + cooldown_until: cooldown, + cooldown_reason: reason || null, + ...(/rate_limited|quota_exhausted/i.test(String(reason || '')) + ? { rate_limited_at: Date.now(), rate_limit_reset_at: cooldown } + : {}), + }, + { vmId, observation }, + ) } - /** Stale-access 401 / 403 permission — not quota, Fable, or 429. */ static AUTH_COOLDOWN = /authentication_failed_after_refresh|permission_denied/i - /** Grant-death leftover. A later live ticket must drop this or the panel stays revoke. */ static GRANT_REVOKE_COOLDOWN = /oauth_revoked|oauth_invalid_grant|invalid_grant|token has been revoked|oauth_no_refresh/i isAuthCooldown(state) { return AccountRuntimeRepo.AUTH_COOLDOWN.test(String(state?.cooldown_reason || '')) } - isGrantRevokeCooldown(state) { + return AccountRuntimeRepo.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || '')) + } + isQuotaCooldown(state, now = Date.now()) { return ( - AccountRuntimeRepo.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || '')) || - (String(state?.status || '') === 'disabled' && - AccountRuntimeRepo.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || state?.status || ''))) + !(Number(state?.rate_limit_reset_at) > now || Number(state?.overload_until) > now) && + (!state?.cooldown_reason || /quota|account_quota_exhausted|rate_limited/i.test(String(state.cooldown_reason))) && + !!(state?.cooldown_until || state?.cooldown_reason) ) } - /** Worker fresh / refresh success: drop only auth leftover so quota cools stay. */ - async clearAuthCooldown(accountId, { vmId = null } = {}) { - return withTransaction(this.db, async () => { - await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) - - const current = await this.get(accountId) - if (!current || !this.isAuthCooldown(current)) return false - return !!(await this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - })) - }) + async clearAuthCooldown(accountId, { vmId, observation, expected } = {}) { + return !!(await this.patch( + accountId, + { status: 'ready', cooldown_until: null, cooldown_reason: null }, + { vmId, observation, expected, condition: (state) => this.isAuthCooldown(state) }, + )) } - /** Live login / import: drop leftover oauth_revoked disable. Quota cools stay. */ - async clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { - return withTransaction(this.db, async () => { - await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) - - const current = await this.get(accountId) - if (!current || !this.isGrantRevokeCooldown(current)) return false - return !!(await this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - })) - }) + async clearGrantRevokeCooldown(accountId, { vmId, observation, expected } = {}) { + return !!(await this.patch( + accountId, + { status: 'ready', cooldown_until: null, cooldown_reason: null }, + { vmId, observation, expected, condition: (state) => this.isGrantRevokeCooldown(state) }, + )) } - /** Operator / panel: drop account + per-model cooldown leftovers. */ - async clearAccountCooldown(accountId, { vmId = null } = {}) { - return withTransaction(this.db, async () => { - await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) + async clearAccountCooldown(accountId, { vmId, observation } = {}) { + return !!(await this.patch( + accountId, + { status: 'ready', cooldown_until: null, cooldown_reason: null, rate_limit_reset_at: null, overload_until: null }, + { vmId, observation, clearModels: true }, + )) + } - const current = await this.get(accountId) - if (!current) return false - return !!(await this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - rate_limit_reset_at: null, - overload_until: null, - model_states: {}, - })) - }) + async clearQuotaCooldown(accountId, { vmId, observation, expected } = {}) { + return !!(await this.patch( + accountId, + { status: 'ready', cooldown_until: null, cooldown_reason: null, rate_limit_reset_at: null }, + { vmId, observation, expected, condition: (state) => this.isQuotaCooldown(state) }, + )) } async clearExpired(now = Date.now()) { - for (const id of await this._listIds.all()) { + for (const row of await this.list()) { await withTransaction(this.db, async () => { - const accountId = id.account_id ?? id.id - await this.db.prepare('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))').get(String(accountId)) - const state = await this.get(accountId) - if (!state?.vm_id) return - let changed = false - if (state.cooldown_until && state.cooldown_until <= now) { - state.status = 'ready' - state.cooldown_until = null - state.cooldown_reason = null - changed = true + const account = await this.lockAccount(row.account_id) + if (!account) return + const current = await this.get(row.account_id) + const fields = {} + const expected = {} + if (current.cooldown_until && current.cooldown_until <= now) { + Object.assign(fields, { status: 'ready', cooldown_until: null, cooldown_reason: null }) + expected.cooldown_until = current.cooldown_until + expected.cooldown_reason = current.cooldown_reason } - if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { - state.rate_limit_reset_at = null - changed = true + for (const field of ['rate_limit_reset_at', 'overload_until']) { + if (current[field] && current[field] <= now) { + fields[field] = null + expected[field] = current[field] + } } - if (state.overload_until && state.overload_until <= now) { - state.overload_until = null - changed = true + const modelPatch = {} + for (const [model, state] of Object.entries(current.model_states || {})) { + if (state?.cooldown_until && Number(state.cooldown_until) <= now) modelPatch[model] = null } - const models = { ...(state.model_states || {}) } - for (const [model, modelState] of Object.entries(models)) { - if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { - delete models[model] - changed = true - } + if (Object.keys(fields).length || Object.keys(modelPatch).length) { + await this.patch(row.account_id, fields, { expected, modelPatch }) } - if (changed) await this.upsert({ ...state, model_states: models }) }) } } async remove(accountId) { - return (await this._removeRt.run(accountId)).changes > 0 + return (await this.db.prepare('DELETE FROM account_runtime WHERE account_id = $1').run(accountId)).changes > 0 } } diff --git a/src/lib/pool/scheduling-runtime.mjs b/src/lib/pool/scheduling-runtime.mjs index db233fac5..878ffd801 100644 --- a/src/lib/pool/scheduling-runtime.mjs +++ b/src/lib/pool/scheduling-runtime.mjs @@ -1,208 +1,272 @@ /** Loaded scheduling state keeps eligibility checks and reservations synchronous. */ +import { isDeepStrictEqual } from 'node:util' import { AccountRuntimeRepo } from '../db/repos/account-runtime-repo.mjs' import { WriteQueue } from './write-queue.mjs' +const WINDOW_FIELDS = { + rateLimitedAt: 'rate_limited_at', + rateLimitResetAt: 'rate_limit_reset_at', + overloadUntil: 'overload_until', + sessionWindowStart: 'session_window_start', + sessionWindowEnd: 'session_window_end', + sessionWindowStatus: 'session_window_status', +} + +function state(input, current = {}) { + return { + cooldown_until: null, + cooldown_reason: null, + last_used_at: null, + credential_generation: 0, + refresh_status: null, + worker_heartbeat_at: null, + worker_status: null, + rate_limited_at: null, + rate_limit_reset_at: null, + overload_until: null, + session_window_start: null, + session_window_end: null, + session_window_status: null, + ...current, + ...structuredClone(input), + status: input.status ?? current.status ?? 'unknown', + priority: Number(input.priority ?? current.priority ?? 0), + weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), + model_states: structuredClone(input.model_states ?? current.model_states ?? {}), + updated_at: new Date().toISOString(), + } +} + export class SchedulingRuntime { constructor(db) { this.persistence = new AccountRuntimeRepo(db) this.rows = new Map() + this.revisions = new Map() + this.refreshStarted = 0 + this.refreshPublished = 0 this.writes = this.persistence.db.schedulingWriteQueue ||= new WriteQueue(this.persistence.db) } async init() { - await this.flush() - this.rows = new Map((await this.persistence.list()).map((row) => [row.account_id, row])) + await this.refresh() return this } - flush() { return this.writes.flush() } - + beginObservation() { + return this.persistence.beginObservation() + } get(id) { return structuredClone(this.rows.get(id) || null) } - list() { return [...this.rows.values()].map((row) => structuredClone(row)) } - upsert(input = {}) { - if (!input.account_id || !input.vm_id) return null - const current = this.get(input.account_id) || {} - const next = { - cooldown_until: null, - cooldown_reason: null, - last_used_at: null, - credential_generation: 0, - refresh_status: null, - worker_heartbeat_at: null, - worker_status: null, - rate_limited_at: null, - rate_limit_reset_at: null, - overload_until: null, - session_window_start: null, - session_window_end: null, - session_window_status: null, - ...current, - ...structuredClone(input), - status: input.status ?? current.status ?? 'unknown', - priority: Number(input.priority ?? current.priority ?? 0), - weight: Math.max(0, Number(input.weight ?? current.weight ?? 1)), - model_states: structuredClone(input.model_states ?? current.model_states ?? {}), - updated_at: new Date().toISOString(), + async refresh() { + await this.flush() + const refresh = ++this.refreshStarted + const revisions = new Map(this.revisions) + const rows = await this.persistence.list() + if (refresh < this.refreshPublished) return + this.refreshPublished = refresh + const present = new Set(rows.map((row) => row.account_id)) + for (const row of rows) { + if (this.revisions.get(row.account_id) === revisions.get(row.account_id)) this.rows.set(row.account_id, row) + } + for (const id of this.rows.keys()) { + if (!present.has(id) && this.revisions.get(id) === revisions.get(id)) this.rows.delete(id) } - this.rows.set(next.account_id, next) - this.writes.enqueue(() => this.persistence.upsert(next)) - return structuredClone(next) } - remove(id) { - const removed = this.rows.delete(id) - this.writes.enqueue(() => this.persistence.remove(id)) - return removed + ticket(observation) { + const ticket = observation ?? this.beginObservation() + ticket?.catch?.(() => {}) + return ticket } - updateWindow( - accountId, - { - vmId = null, - rateLimitedAt, - rateLimitResetAt, - overloadUntil, - sessionWindowStart, - sessionWindowEnd, - sessionWindowStatus, - } = {}, - ) { - const current = this.get(accountId) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, - } - if (!current.vm_id && vmId) current.vm_id = vmId - if (!current.vm_id) return null - if (rateLimitedAt !== undefined) current.rate_limited_at = rateLimitedAt - if (rateLimitResetAt !== undefined) current.rate_limit_reset_at = rateLimitResetAt - if (overloadUntil !== undefined) current.overload_until = overloadUntil - if (sessionWindowStart !== undefined) current.session_window_start = sessionWindowStart - if (sessionWindowEnd !== undefined) current.session_window_end = sessionWindowEnd - if (sessionWindowStatus !== undefined) current.session_window_status = sessionWindowStatus - return this.upsert(current) + publish(next, write) { + const id = next.account_id + const revision = (this.revisions.get(id) || 0) + 1 + this.revisions.set(id, revision) + this.rows.set(id, next) + this.writes.enqueue(async () => { + const stored = await write() + if (stored && this.revisions.get(id) === revision) this.rows.set(id, stored) + }) + return structuredClone(next) } - markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown' } = {}) { - const current = this.get(accountId) || { - account_id: accountId, - vm_id: vmId, - model_states: {}, + upsert(input = {}, { observation, vmId } = {}) { + const current = this.get(input.account_id) || {} + if (!input.account_id || (input.vm_id !== undefined && !input.vm_id) || !(input.vm_id || current.vm_id || vmId)) + return null + const next = state(input, { ...current, vm_id: current.vm_id || vmId }) + const patch = { account_id: input.account_id } + for (const key of Object.keys(input)) { + if (['account_id', 'model_states', 'updated_at'].includes(key)) continue + // A caller can still supply a cached merged record. Persist its mutations, + // while an explicit observation also versions unchanged observed fields. + if (observation !== undefined || !isDeepStrictEqual(current[key], next[key])) patch[key] = next[key] } - if (model) { - current.model_states = { ...(current.model_states || {}) } - current.model_states[model] = { - status, - cooldown_until: Number(until) || null, - reason: reason || null, - updated_at: new Date().toISOString(), - } - } else { - current.status = status - current.cooldown_until = Number(until) || null - current.cooldown_reason = reason || null - // Structured mirror of the protocol-level limit (sub2api account columns). - // overload_until is only the 529 hard column (RateLimitService.handle529). - // A 502 failover also uses reason provider_overloaded for 15s; writing it - // here paints every hopped VM as 过载冷却 and removes them from the pool. - if (/rate_limited|quota_exhausted/i.test(String(reason || ''))) { - current.rate_limited_at = Date.now() - current.rate_limit_reset_at = Number(until) || null + const modelPatch = {} + if (input.model_states !== undefined) { + for (const model of new Set([ + ...Object.keys(current.model_states || {}), + ...Object.keys(input.model_states || {}), + ])) { + if ( + observation !== undefined || + !isDeepStrictEqual(current.model_states?.[model], input.model_states?.[model]) + ) { + modelPatch[model] = input.model_states?.[model] ?? null + } } } - return this.upsert(current) + const ticket = this.ticket(observation) + return this.publish(next, () => + this.persistence.upsert(patch, { observation: ticket, modelPatch, vmId: vmId || input.vm_id || current.vm_id }), + ) } - static AUTH_COOLDOWN = /authentication_failed_after_refresh|permission_denied/i - - static GRANT_REVOKE_COOLDOWN = - /oauth_revoked|oauth_invalid_grant|invalid_grant|token has been revoked|oauth_no_refresh/i + remove(id) { + const removed = this.rows.delete(id) + this.revisions.set(id, (this.revisions.get(id) || 0) + 1) + this.writes.enqueue(() => this.persistence.remove(id)) + return removed + } - isAuthCooldown(state) { - return SchedulingRuntime.AUTH_COOLDOWN.test(String(state?.cooldown_reason || '')) + updateWindow(accountId, { vmId = null, observation, ...input } = {}) { + const current = this.get(accountId) || { account_id: accountId, vm_id: vmId } + current.vm_id ||= vmId + if (!current.vm_id) return null + const fields = {} + for (const [key, field] of Object.entries(WINDOW_FIELDS)) if (input[key] !== undefined) fields[field] = input[key] + const ticket = this.ticket(observation) + return this.publish(state(fields, current), () => + this.persistence.updateWindow(accountId, { vmId, ...input, observation: ticket }), + ) } - isGrantRevokeCooldown(state) { - return ( - SchedulingRuntime.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || '')) || - (String(state?.status || '') === 'disabled' && - SchedulingRuntime.GRANT_REVOKE_COOLDOWN.test(String(state?.cooldown_reason || state?.status || ''))) + markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown', observation } = {}) { + const current = this.get(accountId) || { account_id: accountId, vm_id: vmId } + current.vm_id ||= vmId + if (!current.vm_id) return null + const fields = model + ? { + model_states: { + ...(current.model_states || {}), + [model]: { + status, + cooldown_until: Number(until) || null, + reason: reason || null, + updated_at: new Date().toISOString(), + }, + }, + } + : { + status, + cooldown_until: Number(until) || null, + cooldown_reason: reason || null, + ...(/rate_limited|quota_exhausted/i.test(String(reason || '')) + ? { rate_limited_at: Date.now(), rate_limit_reset_at: Number(until) || null } + : {}), + } + const ticket = this.ticket(observation) + return this.publish(state(fields, current), () => + this.persistence.markCooldown(accountId, { vmId, until, reason, model, status, observation: ticket }), ) } - clearAuthCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current || !this.isAuthCooldown(current)) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - }) + static AUTH_COOLDOWN = AccountRuntimeRepo.AUTH_COOLDOWN + static GRANT_REVOKE_COOLDOWN = AccountRuntimeRepo.GRANT_REVOKE_COOLDOWN + isAuthCooldown(current) { + return this.persistence.isAuthCooldown(current) + } + isGrantRevokeCooldown(current) { + return this.persistence.isGrantRevokeCooldown(current) } - clearGrantRevokeCooldown(accountId, { vmId = null } = {}) { - const current = this.get(accountId) - if (!current || !this.isGrantRevokeCooldown(current)) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - }) + clearAuthCooldown(accountId, options = {}) { + return this.clearCooldown('clearAuthCooldown', accountId, options) + } + clearGrantRevokeCooldown(accountId, options = {}) { + return this.clearCooldown('clearGrantRevokeCooldown', accountId, options) + } + clearAccountCooldown(accountId, options = {}) { + return this.clearCooldown('clearAccountCooldown', accountId, options) + } + clearQuotaCooldown(accountId, options = {}) { + return this.clearCooldown('clearQuotaCooldown', accountId, options) } - clearAccountCooldown(accountId, { vmId = null } = {}) { + clearCooldown(method, accountId, { vmId = null, observation } = {}) { const current = this.get(accountId) if (!current) return false - return !!this.upsert({ - ...current, - vm_id: current.vm_id || vmId, + if (method === 'clearAuthCooldown' && !this.isAuthCooldown(current)) return false + if (method === 'clearGrantRevokeCooldown' && !this.isGrantRevokeCooldown(current)) return false + if (method === 'clearQuotaCooldown' && !this.persistence.isQuotaCooldown(current)) return false + const management = method === 'clearAccountCooldown' + const fields = { status: 'ready', cooldown_until: null, cooldown_reason: null, - rate_limit_reset_at: null, - overload_until: null, - model_states: {}, + ...(management ? { rate_limit_reset_at: null, overload_until: null, model_states: {} } : {}), + ...(method === 'clearQuotaCooldown' ? { rate_limit_reset_at: null } : {}), + } + const ticket = this.ticket(observation) + this.publish(state(fields, current), async () => { + await this.persistence[method](accountId, { + vmId, + observation: ticket, + ...(management + ? {} + : { + expected: { + cooldown_until: current.cooldown_until, + cooldown_reason: current.cooldown_reason, + ...(method === 'clearQuotaCooldown' + ? { rate_limit_reset_at: current.rate_limit_reset_at, overload_until: current.overload_until } + : {}), + }, + }), + }) + return this.persistence.get(accountId) }) + return true } clearExpired(now = Date.now()) { - for (const state of this.list()) { - if (!state.vm_id) continue - let changed = false - if (state.cooldown_until && state.cooldown_until <= now) { - state.status = 'ready' - state.cooldown_until = null - state.cooldown_reason = null - changed = true - } - if (state.rate_limit_reset_at && state.rate_limit_reset_at <= now) { - state.rate_limit_reset_at = null - changed = true - } - if (state.overload_until && state.overload_until <= now) { - state.overload_until = null - changed = true - } - const models = { ...(state.model_states || {}) } - for (const [model, modelState] of Object.entries(models)) { - if (modelState?.cooldown_until && Number(modelState.cooldown_until) <= now) { + const changed = new Map() + for (const current of this.list()) { + const fields = {} + if (current.cooldown_until && current.cooldown_until <= now) + Object.assign(fields, { status: 'ready', cooldown_until: null, cooldown_reason: null }) + for (const field of ['rate_limit_reset_at', 'overload_until']) + if (current[field] && current[field] <= now) fields[field] = null + const models = { ...(current.model_states || {}) } + for (const [model, value] of Object.entries(models)) + if (value?.cooldown_until && Number(value.cooldown_until) <= now) { delete models[model] - changed = true + fields.model_states = models } - } - if (changed) this.upsert({ ...state, model_states: models }) + if (!Object.keys(fields).length) continue + const id = current.account_id + const revision = (this.revisions.get(id) || 0) + 1 + this.revisions.set(id, revision) + this.rows.set(id, state(fields, current)) + changed.set(id, revision) } + if (!changed.size) return + this.writes.enqueue(async () => { + // Re-evaluate committed values under the account row lock. A local + // snapshot expiring cannot delete a newer or extended remote cooldown. + await this.persistence.clearExpired(now) + for (const [id, revision] of changed) { + const stored = await this.persistence.get(id) + if (stored && this.revisions.get(id) === revision) this.rows.set(id, stored) + } + }) } } From c89c391f5a71c18e510c9dfef87a6baa2d4aa2bb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:39:31 +0800 Subject: [PATCH 121/473] fix: guard shared sticky bindings with persistent generations --- .../011_sticky_revisions.sql | 4 + src/lib/db/repos/sticky-repo.mjs | 149 +++++++++- src/lib/pool/sticky-router.mjs | 271 +++++++++++++++++- test/unit/sticky-router-shared.test.mjs | 245 ++++++++++++++++ 4 files changed, 660 insertions(+), 9 deletions(-) create mode 100644 src/lib/db/postgres-migrations/011_sticky_revisions.sql diff --git a/src/lib/db/postgres-migrations/011_sticky_revisions.sql b/src/lib/db/postgres-migrations/011_sticky_revisions.sql new file mode 100644 index 000000000..64e0d740f --- /dev/null +++ b/src/lib/db/postgres-migrations/011_sticky_revisions.sql @@ -0,0 +1,4 @@ +ALTER TABLE sticky_sessions ADD COLUMN deleted_at bigint; +CREATE SEQUENCE sticky_generation_seq AS bigint OWNED BY sticky_sessions.generation; +SELECT setval('sticky_generation_seq', GREATEST(COALESCE((SELECT max(generation) FROM sticky_sessions), 0) + 1, 1), false); +ALTER TABLE sticky_sessions ALTER COLUMN generation SET DEFAULT nextval('sticky_generation_seq'); diff --git a/src/lib/db/repos/sticky-repo.mjs b/src/lib/db/repos/sticky-repo.mjs index 032543557..868d8ebd2 100644 --- a/src/lib/db/repos/sticky-repo.mjs +++ b/src/lib/db/repos/sticky-repo.mjs @@ -2,7 +2,7 @@ * sticky_sessions repository — conversation key → account/VM bindings. */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' function rowToEntry(row) { if (!row) return null @@ -22,7 +22,7 @@ function rowToEntry(row) { export class StickyRepo { constructor(db = getDb()) { this.db = db - this._get = db.prepare('SELECT * FROM sticky_sessions WHERE key = $1') + this._get = db.prepare('SELECT * FROM sticky_sessions WHERE key = $1 AND deleted_at IS NULL') this._upsert = db.prepare(` INSERT INTO sticky_sessions (key, account_id, vm_id, session_id, device_id, bound_at, expires_at, hits, generation, slot_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) @@ -35,7 +35,8 @@ export class StickyRepo { expires_at = excluded.expires_at, hits = excluded.hits, generation = excluded.generation, - slot_index = excluded.slot_index + slot_index = excluded.slot_index, + deleted_at = NULL `) this._delete = db.prepare('DELETE FROM sticky_sessions WHERE key = $1') this._deleteByAccount = db.prepare('DELETE FROM sticky_sessions WHERE account_id = $1 OR vm_id = $2') @@ -49,8 +50,8 @@ export class StickyRepo { OR ascii_fold(key) LIKE '%:ch:%' OR ascii_fold(key) LIKE '%:dev:%' `) - this._all = db.prepare('SELECT * FROM sticky_sessions') - this._count = db.prepare('SELECT COUNT(*) c FROM sticky_sessions') + this._all = db.prepare('SELECT * FROM sticky_sessions WHERE deleted_at IS NULL') + this._count = db.prepare('SELECT COUNT(*) c FROM sticky_sessions WHERE deleted_at IS NULL') } async get(key) { @@ -99,4 +100,142 @@ export class StickyRepo { async count() { return (await this._count.get()).c } + + async snapshot() { + const sessions = {} + const versions = {} + for (const row of await this.db.prepare('SELECT * FROM sticky_sessions').all()) { + versions[row.key] = Number(row.generation) + if (row.deleted_at == null) sessions[row.key] = rowToEntry(row) + } + return { sessions, versions } + } + + async withKeys(keys, change) { + return withTransaction(this.db, async () => { + for (const key of [...new Set(keys.filter(Boolean))].sort()) { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ + `vm2api:sticky:${this.db.schema}`, + key, + ]) + } + return change() + }) + } + + async current(key) { + const row = await this.db.prepare('SELECT * FROM sticky_sessions WHERE key = $1 FOR UPDATE').get(key) + if (!row) return { entry: null, generation: 0 } + const generation = Number(row.generation) + if (!Number.isSafeInteger(generation) || generation < 0) throw new Error('invalid sticky generation') + return { entry: row.deleted_at == null ? rowToEntry(row) : null, generation } + } + + async nextGeneration(minimum = 0) { + // Legacy imports/single-process writes can carry explicit generations. Keep + // the sequence ahead of them before allocating a shared token. This lock + // covers only the short metadata transaction, never inference execution. + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ + `vm2api:sticky-generation:${this.db.schema}`, + 'allocator', + ]) + const sequence = await this.db.prepare('SELECT last_value FROM sticky_generation_seq').get() + if (Number(sequence.last_value) <= minimum) { + await this.db.prepare("SELECT setval('sticky_generation_seq', $1, true)").get(minimum) + } + const row = await this.db.prepare("SELECT nextval('sticky_generation_seq') AS generation").get() + const generation = Number(row.generation) + if (!Number.isSafeInteger(generation) || generation < 1) throw new Error('invalid sticky generation') + return generation + } + + /** The callback derives the new entry from the locked PG row, never a cached snapshot. */ + async compareAndSet(key, ifGeneration, change, { now = Date.now(), expire = true } = {}) { + if (!Number.isSafeInteger(ifGeneration) || ifGeneration < 0) throw new Error('sticky generation is required') + return this.withKeys([key], async () => { + let current = await this.current(key) + if (expire && current.entry?.expires_at != null && current.entry.expires_at < now) { + const generation = await this.nextGeneration(current.generation) + await this.db + .prepare(`UPDATE sticky_sessions SET deleted_at = $3, generation = $2 + WHERE key = $1 AND generation = $4`) + .run(key, generation, now, current.generation) + current = await this.current(key) + } + if (current.generation !== ifGeneration) return { applied: false, ...current, reason: 'sticky_changed' } + const next = change(current.entry) + if (!next) { + const generation = await this.nextGeneration(current.generation) + const result = + current.generation === 0 + ? await this.db + .prepare(`INSERT INTO sticky_sessions (key, generation, deleted_at) VALUES ($1, $2, $3) + ON CONFLICT (key) DO NOTHING RETURNING generation`) + .get(key, generation, now) + : await this.db + .prepare(`UPDATE sticky_sessions SET deleted_at = $3, generation = $2 + WHERE key = $1 AND generation = $4 RETURNING generation`) + .get(key, generation, now, current.generation) + if (!result) return { applied: false, ...(await this.current(key)), reason: 'sticky_changed' } + return { applied: true, entry: null, generation } + } + const identity = ['account_id', 'vm_id', 'session_id', 'device_id', 'slot_index'] + const changed = + !current.entry || identity.some((field) => (current.entry[field] ?? null) !== (next[field] ?? null)) + const generation = changed ? await this.nextGeneration(current.generation) : current.generation + const fields = ['account_id', 'vm_id', 'session_id', 'device_id', 'bound_at', 'expires_at', 'hits', 'slot_index'] + const values = fields.map((field) => next[field] ?? null) + let row + if (current.generation === 0) { + row = await this.db + .prepare(`INSERT INTO sticky_sessions (key, generation, ${fields.join(', ')}) + VALUES ($1, $2, ${fields.map((_, index) => `$${index + 3}`).join(', ')}) + ON CONFLICT (key) DO NOTHING RETURNING *`) + .get(key, generation, ...values) + } else { + row = await this.db + .prepare(`UPDATE sticky_sessions SET generation = $2, deleted_at = NULL, + ${fields.map((field, index) => `${field} = $${index + 3}`).join(', ')} + WHERE key = $1 AND generation = $11 RETURNING *`) + .get(key, generation, ...values, current.generation) + } + if (!row) return { applied: false, ...(await this.current(key)), reason: 'sticky_changed' } + return { applied: true, entry: rowToEntry(row), generation } + }) + } + + /** Bulk invalidation selects authoritative PG keys even when the caller cache is empty. */ + async invalidateShared(where, params = [], now = Date.now()) { + const keys = ( + await this.db.prepare(`SELECT key FROM sticky_sessions WHERE deleted_at IS NULL AND (${where})`).all(...params) + ).map((row) => row.key) + return this.withKeys(keys, async () => { + const results = [] + for (const key of keys.sort()) { + const row = await this.db + .prepare(`SELECT generation FROM sticky_sessions + WHERE key = $1 AND deleted_at IS NULL AND (${where.replace(/\$(\d+)/g, (_, index) => `$${Number(index) + 1}`)}) + FOR UPDATE`) + .get(key, ...params) + if (!row) continue + const result = await this.compareAndSet(key, Number(row.generation), () => null, { now, expire: false }) + if (result.applied) results.push({ key, ...result }) + } + return results + }) + } + + async removeByAccountShared({ accountId = null, vmId = null } = {}) { + if (!accountId && !vmId) return [] + return this.invalidateShared('account_id = $1 OR vm_id = $2', [accountId || '', vmId || '']) + } + + async purgeExpiredShared(now = Date.now()) { + return this.invalidateShared('expires_at < $1', [now], now) + } + + async dropAliasKeysShared() { + return this.invalidateShared(`key = 'envelope' OR ascii_fold(key) LIKE 'ch:%' OR ascii_fold(key) LIKE 'dev:%' + OR ascii_fold(key) LIKE '%:envelope' OR ascii_fold(key) LIKE '%:ch:%' OR ascii_fold(key) LIKE '%:dev:%'`) + } } diff --git a/src/lib/pool/sticky-router.mjs b/src/lib/pool/sticky-router.mjs index b4e5309a7..fa2167bfb 100644 --- a/src/lib/pool/sticky-router.mjs +++ b/src/lib/pool/sticky-router.mjs @@ -1,5 +1,5 @@ /** - * Configurable sticky / conversation-continuity routing (SQLite-backed). + * Configurable sticky / conversation-continuity routing (PostgreSQL-backed). * Binds conversation key → account/VM for the TTL window. */ @@ -8,6 +8,7 @@ import { ENVELOPE_NEEDLES, extractPrompt } from '../core/distill-detect.mjs' import { resolveStoreDb } from '../db/database.mjs' import { StickyRepo } from '../db/repos/sticky-repo.mjs' import { WriteQueue } from './write-queue.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { claudeCodeAgentRootSession, extractCallerSession, @@ -203,25 +204,35 @@ export function childDeclaredWithoutParent(body = {}, headers = {}) { } export class StickyRouter { - constructor({ dataDir, db, config }) { + constructor({ dataDir, db, config, shared = isKubernetesMode() }) { this.db = resolveStoreDb({ db, dataDir }) this.persistence = new StickyRepo(this.db) this.writes = new WriteQueue(this.db) + this.sharedOperations = new Set() + this.db.pendingWriteQueues.add(this) this.sessions = {} + this.shared = shared + this.versions = {} + this.cacheWrites = 0 + this.refreshIssued = 0 + this.refreshPublished = 0 this.config = mergeStickyConfig(config) this.repo = { get: (key) => structuredClone(this.sessions[key] || null), all: () => structuredClone(this.sessions), upsert: (key, entry) => { + this._assertSyncWrite() const next = { ...entry, device_id: entry.device_id || this.sessions[key]?.device_id || null } this.sessions[key] = structuredClone(next) this.writes.enqueue(() => this.persistence.upsert(key, next)) }, remove: (key) => { + this._assertSyncWrite() delete this.sessions[key] this.writes.enqueue(() => this.persistence.remove(key)) }, removeByAccount: ({ accountId = null, vmId = null } = {}) => { + this._assertSyncWrite() let removed = 0 for (const [key, entry] of Object.entries(this.sessions)) { if ((accountId && entry.account_id === accountId) || (vmId && entry.vm_id === vmId)) { @@ -233,6 +244,7 @@ export class StickyRouter { return removed }, purgeExpired: (now) => { + this._assertSyncWrite() let removed = 0 for (const [key, entry] of Object.entries(this.sessions)) { if (entry.expires_at != null && entry.expires_at < now) { @@ -249,6 +261,10 @@ export class StickyRouter { /** Kept for API compat + post-restore hook (state lives in DB). */ async init() { for (const queue of this.db.pendingWriteQueues || []) await queue.flush() + if (this.shared) { + await this.persistence.dropAliasKeysShared() + return this.refresh() + } await this.persistence.dropAliasKeys() this.sessions = await this.persistence.all() return this @@ -256,21 +272,262 @@ export class StickyRouter { async reload() { await this.flush() + if (this.shared) return this.refresh() return await this.init() } - flush() { - return this.writes.flush() + async flush() { + await this.writes.flush() + while (this.sharedOperations.size) await Promise.all([...this.sharedOperations]) } /** Re-bind to a fresh DB connection (after backup restore). */ async rebind(db) { await this.flush() + this.db.pendingWriteQueues?.delete(this.writes) + this.db.pendingWriteQueues?.delete(this) this.db = db this.persistence = new StickyRepo(db) + this.writes = new WriteQueue(db) + db.pendingWriteQueues.add(this) + this.cacheWrites++ return await this.init() } + _assertSyncWrite() { + if (this.shared) + throw Object.assign(new Error('shared sticky mutations require the asynchronous API'), { + code: 'shared_sticky_async_required', + }) + } + + _trackShared(operation) { + this.sharedOperations.add(operation) + operation.finally(() => this.sharedOperations.delete(operation)).catch(() => {}) + return operation + } + + /** Pure committed-state refresh: no alias cleanup and no write-queue drain. */ + async refresh() { + const issued = ++this.refreshIssued + const writes = this.cacheWrites + const snapshot = await this.persistence.snapshot() + if (issued > this.refreshPublished && writes === this.cacheWrites) { + this.refreshPublished = issued + this.sessions = snapshot.sessions + this.versions = snapshot.versions + if (this.shared) this._purge() + } + return this + } + + versionOf(key) { + return this.versions[key] ?? (Number(this.sessions[key]?.generation) || 0) + } + + _record(key, entry) { + if (!entry) return null + return { + key, + accountId: entry.account_id, + vmId: entry.vm_id, + sessionId: entry.session_id || null, + generation: entry.generation, + slotIndex: entry.slot_index == null ? null : Number(entry.slot_index), + } + } + + _acceptShared(key, result) { + this.db.afterCommit(() => { + this.cacheWrites++ + if (result.generation < this.versionOf(key)) return + this.versions[key] = result.generation + if (result.entry) this.sessions[key] = structuredClone(result.entry) + else delete this.sessions[key] + }) + const record = this._record(key, result.entry) + return { + applied: result.applied, + record, + generation: result.generation, + nativeSessionId: record?.sessionId || null, + ...(result.reason ? { reason: result.reason } : {}), + } + } + + async _changeShared(key, options, change) { + if (!key || !this.config.enabled) + return { applied: false, record: null, generation: 0, nativeSessionId: null, reason: 'sticky_disabled' } + const expected = options?.ifGeneration ?? this.versionOf(key) + return this._trackShared( + this.persistence.compareAndSet(key, expected, change).then((result) => this._acceptShared(key, result)), + ) + } + + async bindShared(key, { accountId, vmId, sessionId = null, deviceId = null, slotIndex = null } = {}, options = {}) { + if (!accountId || !vmId) throw new Error('sticky account and VM are required') + return this._changeShared(key, options, (previous) => { + const prev = previous || {} + const locked = !!( + prev.vm_id && + vmId && + prev.vm_id !== vmId && + prev.account_id && + accountId && + prev.account_id !== accountId + ) + const nextVm = locked ? prev.vm_id : vmId + const sameVm = !!(prev.vm_id && nextVm === prev.vm_id) + const nextSession = locked + ? prev.session_id || sessionId || null + : options.replaceSession + ? sessionId || null + : sameVm + ? prev.session_id || sessionId || null + : sessionId || prev.session_id || null + const now = Date.now() + return { + account_id: locked ? prev.account_id : accountId, + vm_id: nextVm, + session_id: nextSession, + device_id: String(deviceId || '').trim() || prev.device_id || null, + slot_index: locked + ? (prev.slot_index ?? null) + : options.clearSlot + ? null + : slotIndex == null + ? (prev.slot_index ?? null) + : Number(slotIndex), + bound_at: now, + expires_at: now + (this.config.ttl_seconds || 86400) * 1000, + hits: (prev.hits || 0) + (options.countHit === false ? 0 : 1), + } + }) + } + + async rebindFamilyShared(key, { accountId, vmId } = {}, options = {}) { + if (!accountId || !vmId) throw new Error('sticky account and VM are required') + return this._changeShared(key, options, (prev) => ({ + account_id: accountId, + vm_id: vmId, + session_id: null, + slot_index: null, + device_id: prev?.device_id || null, + bound_at: Date.now(), + expires_at: Date.now() + (this.config.ttl_seconds || 86400) * 1000, + hits: (prev?.hits || 0) + (options.countHit === true ? 1 : 0), + })) + } + + async bindDeviceAffinityShared(key, value = {}, options = {}) { + return this.rebindFamilyShared(key, value, options) + } + + async unbindShared(key, options = {}) { + return this._changeShared(key, options, () => null) + } + + async unbindByAccountShared(options = {}) { + return this._trackShared( + this.persistence.removeByAccountShared(options).then((results) => { + for (const result of results) this._acceptShared(result.key, result) + return results.length + }), + ) + } + + async purgeExpiredShared(now = Date.now()) { + return this._trackShared( + this.persistence.purgeExpiredShared(now).then((results) => { + for (const result of results) this._acceptShared(result.key, result) + return results.length + }), + ) + } + + /** A legacy source and every destination are checked under the same ordered key locks. */ + async migrateLegacyIdentityShared(legacyKey, { sessionId = '', deviceId = '' } = {}) { + const sessionKey = this.canonicalSessionKey(sessionId) + const deviceKey = this.canonicalDeviceKey(deviceId) + const result = { sessionKey: null, deviceKey: null } + if (!legacyKey || !this.config.enabled) return result + const sourceVersion = this.versionOf(legacyKey) + const sessionVersion = sessionKey ? this.versionOf(sessionKey) : null + const deviceVersion = deviceKey ? this.versionOf(deviceKey) : null + return this._trackShared( + this.persistence.withKeys([legacyKey, sessionKey, deviceKey], async () => { + const source = await this.persistence.current(legacyKey) + if (!source.entry || source.generation !== sourceVersion || Date.now() > source.entry.expires_at) return result + const sourceRecord = source.entry + if (sessionKey && sessionKey !== legacyKey && sessionVersion === 0) { + const applied = await this.bindShared( + sessionKey, + { + accountId: sourceRecord.account_id, + vmId: sourceRecord.vm_id, + sessionId: sourceRecord.session_id || sessionId, + deviceId, + }, + { ifGeneration: 0, countHit: false }, + ) + if (applied.applied) result.sessionKey = sessionKey + } + if (deviceKey && deviceVersion === 0) { + const applied = await this.bindDeviceAffinityShared( + deviceKey, + { accountId: sourceRecord.account_id, vmId: sourceRecord.vm_id }, + { ifGeneration: 0 }, + ) + if (applied.applied) result.deviceKey = deviceKey + } + return result + }), + ) + } + + async sessionPoolKeysShared(req, body = {}, options = {}) { + await this.refresh() + const { sessionId = '', deviceId = '', migrate = true } = options + if (!this.config.enabled) return { stickyKey: null, stickyKeys: [] } + const canonical = this.canonicalSessionKey(sessionId) + if (!canonical) + return { + stickyKey: this.extractPoolKey(req, body, { platform: 'anthropic' }), + stickyKeys: this.collectPoolKeys(req, body, { platform: 'anthropic' }), + } + const legacy = [ + ...new Set([...this.collectPoolKeys(req, body, { platform: 'anthropic' }), ...this.collectPoolKeys(req, body)]), + ].filter((key) => key !== canonical && this.resolve(key)) + if (!this.resolve(canonical) && legacy.length) { + if (!migrate) return { stickyKey: legacy[0], stickyKeys: legacy } + await this.migrateLegacyIdentityShared(legacy[0], { sessionId, deviceId }) + } + return { stickyKey: canonical, stickyKeys: [canonical, ...legacy] } + } + + async familyPoolKeyShared(req, sessionId, { trusted = false } = {}) { + await this.refresh() + const legacy = this.familyKey(req, sessionId, 'anthropic') + if (!trusted || !legacy) return legacy + const canonical = this.canonicalFamilyKey(sessionId) + const sourceVersion = this.versionOf(legacy) + if (this.versionOf(canonical) === 0) { + await this._trackShared( + this.persistence.withKeys([legacy, canonical], async () => { + const source = await this.persistence.current(legacy) + if (source.entry && source.generation === sourceVersion && Date.now() <= source.entry.expires_at) { + await this.rebindFamilyShared( + canonical, + { accountId: source.entry.account_id, vmId: source.entry.vm_id }, + { ifGeneration: 0 }, + ) + } + }), + ) + } + return canonical + } + isolateKey(raw, req = null) { const id = req?.apiKeyRecord?.id if (id == null || id === '') return String(raw) @@ -653,6 +910,12 @@ export class StickyRouter { } _purge() { + if (this.shared) { + for (const [key, entry] of Object.entries(this.sessions)) { + if (entry.expires_at != null && entry.expires_at < Date.now()) delete this.sessions[key] + } + return + } this.repo.purgeExpired(Date.now()) } diff --git a/test/unit/sticky-router-shared.test.mjs b/test/unit/sticky-router-shared.test.mjs index 62fa8a45f..4890caec3 100644 --- a/test/unit/sticky-router-shared.test.mjs +++ b/test/unit/sticky-router-shared.test.mjs @@ -1,5 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' +import { setImmediate } from 'node:timers/promises' import { StickyRouter } from '../../src/lib/pool/sticky-router.mjs' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' @@ -114,3 +115,247 @@ test('a replica that observed no binding cannot replace the committed native ses assert.equal(second.resolve(key).sessionId, 'native-first') assert.equal((await first.persistence.get(key)).session_id, 'native-first') }) + +test('shared writes return committed native sessions and invalidate same-VM replacements', async () => { + const [first, second] = await replicas() + const key = 'sess:native-replacement' + const initial = await first.bindShared(key, { accountId: 'account', vmId: 'vm', sessionId: 'native-1', slotIndex: 0 }) + assert.equal(initial.applied, true) + assert.equal(initial.nativeSessionId, 'native-1') + assert.equal(initial.record.generation, initial.generation) + await second.refresh() + const touched = await first.bindShared(key, { accountId: 'account', vmId: 'vm', sessionId: 'ignored' }) + assert.equal(touched.nativeSessionId, 'native-1') + assert.equal(touched.generation, initial.generation) + const replaced = await first.bindShared( + key, + { accountId: 'account', vmId: 'vm', sessionId: 'native-2' }, + { + replaceSession: true, + clearSlot: true, + }, + ) + assert.ok(replaced.generation > initial.generation) + assert.equal(replaced.nativeSessionId, 'native-2') + assert.equal(replaced.record.slotIndex, null) + const stale = await second.bindShared( + key, + { accountId: 'account', vmId: 'vm', sessionId: 'native-1' }, + { + ifGeneration: initial.generation, + replaceSession: true, + }, + ) + assert.equal(stale.applied, false) + assert.equal(stale.reason, 'sticky_changed') + assert.equal(stale.nativeSessionId, 'native-2') + assert.equal(stale.generation, replaced.generation) +}) + +test('shared same-identity touches increment hits from PG instead of a stale cache', async () => { + const [first, second] = await replicas() + const key = 'sess:atomic-hits' + const initial = await first.bindShared(key, { accountId: 'account', vmId: 'vm', sessionId: 'native' }) + await second.refresh() + const touches = await Promise.all( + [first, second].map((router) => + router.bindShared(key, { accountId: 'account', vmId: 'vm' }, { ifGeneration: initial.generation }), + ), + ) + assert.ok(touches.every((result) => result.applied)) + assert.equal((await first.persistence.get(key)).hits, 3) +}) + +test('credential invalidation finds peer-only bindings and retains tombstone versions', async () => { + const [first, second] = await replicas() + await first.bindShared('sess:credential-a', { accountId: 'dead-account', vmId: 'vm-a' }) + await first.bindShared('sess:credential-b', { accountId: 'dead-account', vmId: 'vm-b' }) + await first.bindShared('sess:unrelated', { accountId: 'live-account', vmId: 'vm-c' }) + assert.equal(second.stats().active_sessions, 0) + assert.equal(await second.unbindByAccountShared({ accountId: 'dead-account' }), 2) + await first.refresh() + assert.equal(first.resolve('sess:credential-a'), null) + assert.equal(first.resolve('sess:credential-b'), null) + assert.ok(first.versionOf('sess:credential-a') > 0) + assert.equal(first.resolve('sess:unrelated').vmId, 'vm-c') +}) + +test('unbinding an absent key invalidates a pending first bind', async () => { + const [first, second] = await replicas() + const key = 'sess:absent-unbind' + const deleted = await first.unbindShared(key, { ifGeneration: 0 }) + assert.equal(deleted.applied, true) + assert.ok(deleted.generation > 0) + const stale = await second.bindShared(key, { accountId: 'account', vmId: 'vm' }, { ifGeneration: 0 }) + assert.equal(stale.applied, false) + assert.equal(stale.record, null) + assert.equal(stale.generation, deleted.generation) +}) + +test('expiry retires its version and a stale binder cannot revive it', async () => { + const [first, second] = await replicas() + const key = 'sess:expired' + const initial = await first.bindShared(key, { accountId: 'account', vmId: 'vm' }) + await second.refresh() + const entry = await first.persistence.get(key) + assert.equal(await first.purgeExpiredShared(entry.expires_at + 1), 1) + const stale = await second.bindShared(key, { accountId: 'account', vmId: 'vm' }, { ifGeneration: initial.generation }) + assert.equal(stale.applied, false) + assert.equal(stale.record, null) + assert.ok(stale.generation > initial.generation) + assert.equal(first.stats().active_sessions, 0) +}) + +test('legacy migration validates its source and never restores a deleted canonical destination', async () => { + const [first, second] = await replicas() + const legacy = 'p:anthropic:kkey:legacy-session' + await first.bindShared(legacy, { accountId: 'account', vmId: 'vm', sessionId: 'native' }) + await second.refresh() + await first.unbindShared(legacy) + assert.deepEqual(await second.migrateLegacyIdentityShared(legacy, { sessionId: 'new-session', deviceId: 'device' }), { + sessionKey: null, + deviceKey: null, + }) + assert.equal(await first.persistence.get('sess:new-session'), null) + assert.equal(await first.persistence.get('dev2:device'), null) + + await first.bindShared(legacy, { accountId: 'account', vmId: 'vm', sessionId: 'native' }) + await first.unbindShared('sess:retired-canonical', { ifGeneration: 0 }) + await second.refresh() + const result = await second.migrateLegacyIdentityShared(legacy, { sessionId: 'retired-canonical' }) + assert.equal(result.sessionKey, null) + assert.equal(await first.persistence.get('sess:retired-canonical'), null) +}) + +test('canonical migration shares native ID while preserving the existing device home and family separation', async () => { + const [first, second] = await replicas() + const req = { apiKeyRecord: { id: 'key' }, headers: {} } + const body = { metadata: { user_id: { session_id: 'trusted-session' } } } + const legacy = first.collectPoolKeys(req, body, { platform: 'anthropic' })[0] + await first.bindShared(legacy, { accountId: 'account', vmId: 'vm', sessionId: 'native' }) + await first.bindDeviceAffinityShared('dev2:device', { accountId: 'home-account', vmId: 'home-vm' }) + const keys = await second.sessionPoolKeysShared(req, body, { sessionId: 'trusted-session', deviceId: 'device' }) + assert.equal(keys.stickyKey, 'sess:trusted-session') + assert.ok(keys.stickyKeys.includes(legacy)) + assert.equal(second.resolve(keys.stickyKey).sessionId, 'native') + assert.equal(second.resolve('dev2:device').vmId, 'home-vm') + assert.equal(second.resolve('dev2:device').slotIndex, null) + assert.equal(second.resolve('dev2:device').sessionId, null) + await first.refresh() + assert.equal(first.resolve(keys.stickyKey).sessionId, 'native') + const familyLegacy = first.familyKey(req, 'parent', 'anthropic') + await first.bindShared(familyLegacy, { accountId: 'account', vmId: 'vm', sessionId: 'unused-session', slotIndex: 2 }) + assert.equal(await second.familyPoolKeyShared(req, 'parent', { trusted: true }), 'family2:parent') + assert.equal(second.resolve('family2:parent').sessionId, null) + assert.equal(second.resolve('family2:parent').slotIndex, null) +}) + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +test('refresh never drains writes and delayed reads cannot overwrite a newer commit or refresh', async () => { + const [first, second] = await replicas() + const key = 'sess:refresh' + await first.bindShared(key, { accountId: 'account', vmId: 'vm', sessionId: 'native' }) + const queued = deferred() + first.writes.enqueue(() => queued.promise) + try { + await first.refresh() + assert.equal(first.resolve(key).sessionId, 'native') + } finally { + queued.resolve() + } + const snapshot = second.persistence.snapshot.bind(second.persistence) + const entered = deferred() + const done = deferred() + second.persistence.snapshot = async () => { + const value = await snapshot() + entered.resolve() + await done.promise + return value + } + const old = second.refresh() + await entered.promise + second.persistence.snapshot = snapshot + await first.bindShared(key, { accountId: 'account', vmId: 'vm-new', sessionId: 'native-new' }) + await second.refresh() + done.resolve() + await old + assert.equal(second.resolve(key).sessionId, 'native-new') +}) + +test('shared flush waits for pending CAS and failed persistence never publishes optimistic state', async () => { + const [first] = await replicas() + const key = 'sess:durable' + const initial = await first.bindShared(key, { accountId: 'account', vmId: 'vm', sessionId: 'native' }) + const entered = deferred() + const done = deferred() + const compareAndSet = first.persistence.compareAndSet.bind(first.persistence) + first.persistence.compareAndSet = async (...args) => { + entered.resolve() + await done.promise + return compareAndSet(...args) + } + const binding = first.bindShared(key, { accountId: 'account', vmId: 'vm-new' }) + await entered.promise + let flushed = false + const flush = first.flush().then(() => { + flushed = true + }) + await setImmediate() + assert.equal(flushed, false) + done.resolve() + await binding + await flush + first.persistence.compareAndSet = compareAndSet + const before = first.resolve(key) + await first.db.exec(`CREATE FUNCTION reject_sticky_write() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'mock sticky persistence failure' USING ERRCODE = 'P0001'; END $$; + CREATE TRIGGER reject_sticky_write BEFORE UPDATE ON sticky_sessions FOR EACH ROW EXECUTE FUNCTION reject_sticky_write()`) + await assert.rejects(first.bindShared(key, { accountId: 'account', vmId: 'vm-other' }), { code: 'P0001' }) + assert.deepEqual(first.resolve(key), before) + assert.ok(before.generation > initial.generation) + assert.equal((await first.persistence.get(key)).vm_id, 'vm-new') +}) + +test('shared synchronous mutation entry points fail before changing cached state', async () => { + const [first] = await replicas() + const key = 'sess:sync-forbidden' + await first.bindShared(key, { accountId: 'account', vmId: 'vm' }) + const before = first.resolve(key) + for (const call of [ + () => first.bind(key, { accountId: 'account', vmId: 'vm-new' }), + () => first.unbind(key), + () => first.rebindFamily(key, { accountId: 'other-account', vmId: 'other-vm' }), + () => first.bindDeviceAffinity('dev2:device', { accountId: 'account', vmId: 'vm' }), + () => first.unbindByAccount({ accountId: 'account' }), + ]) + assert.throws(call, { code: 'shared_sticky_async_required' }) + assert.deepEqual(first.resolve(key), before) +}) + +test('the sticky generation sequence is owned by its table column', async () => { + const [first] = await replicas() + const row = await first.db.prepare("SELECT pg_get_serial_sequence('sticky_sessions', 'generation') AS owned").get() + assert.match(row.owned, /sticky_generation_seq$/) +}) + +test('shared generations advance past an explicitly imported legacy generation', async () => { + const [first] = await replicas() + const key = 'sess:imported-generation' + await first.persistence.upsert(key, { + account_id: 'account', + vm_id: 'vm', + generation: 500, + expires_at: Date.now() + 60_000, + }) + await first.refresh() + const moved = await first.bindShared(key, { accountId: 'account', vmId: 'vm-new' }, { ifGeneration: 500 }) + assert.equal(moved.applied, true) + assert.ok(moved.generation > 500) +}) From 87db8ac22ecd2c40bdef1a25bccb0c0d20e7ab12 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:40:28 +0800 Subject: [PATCH 122/473] test: reproduce VM mutation and execution admission races --- test/unit/shared-vm-operations.test.mjs | 246 ++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 test/unit/shared-vm-operations.test.mjs diff --git a/test/unit/shared-vm-operations.test.mjs b/test/unit/shared-vm-operations.test.mjs new file mode 100644 index 000000000..d36c7951b --- /dev/null +++ b/test/unit/shared-vm-operations.test.mjs @@ -0,0 +1,246 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { createInterface } from 'node:readline' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { withVmLock, vmLockBusy } from '../../src/lib/vm/vm-file.mjs' + +function kubernetes(t) { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) +} + +async function otherDatabase(t) { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + return other +} + +async function operations(db, ownerId) { + const { SharedVmOperations } = await import('../../src/lib/vm/shared-vm-operations.mjs') + return new SharedVmOperations(db, { ownerId }) +} + +function execution(patch = {}) { + return { accountId: 'account-a', vmId: 'vm-a', accountLimit: 20, vmLimit: 20, fableLimit: 0, rpmLimit: 0, ...patch } +} + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +test('Kubernetes VM mutation refuses an active execution on another PG connection', async (t) => { + kubernetes(t) + const other = await otherDatabase(t) + const gate = await new ExecutionAdmissionsRepo(other).tryReserve(execution()) + let called = false + try { + await assert.rejects( + withVmLock('/pod-a/vms/vm-a.json', () => { + called = true + }), + { code: 'vm_execution_active', status: 409 }, + ) + assert.equal(called, false) + } finally { + await gate.release() + } +}) + +test('an active Kubernetes VM mutation blocks new execution on another PG connection', async (t) => { + kubernetes(t) + const other = await otherDatabase(t) + const entered = deferred() + const finish = deferred() + const running = withVmLock('/pod-a/vms/vm-a.json', async () => { + entered.resolve() + await finish.promise + }) + await entered.promise + let gate + try { + gate = await new ExecutionAdmissionsRepo(other).tryReserve(execution()) + assert.equal(gate.ok, false) + assert.equal(gate.code, 'vm_operation_active') + } finally { + await gate?.release?.() + finish.resolve() + await running + } +}) + +test('different Pod paths identify the same VM mutation resource', async (t) => { + kubernetes(t) + await otherDatabase(t) + const entered = deferred() + const finish = deferred() + const running = withVmLock('/pod-a/vms/vm-a.json', async () => { + entered.resolve() + await finish.promise + }) + await entered.promise + try { + await assert.rejects( + withVmLock('/pod-b/vms/vm-a.json', () => {}), + { code: 'vm_operation_active', status: 409 }, + ) + } finally { + finish.resolve() + await running + } +}) + +test('concurrent operation and execution admission cannot both acquire the same VM', async (t) => { + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = new ExecutionAdmissionsRepo(other) + for (let i = 0; i < 8; i++) { + const vmId = `race-${i}` + const [operation, admission] = await Promise.all([ + first.tryAcquire({ vmId }), + second.tryReserve(execution({ vmId })), + ]) + assert.equal(Number(operation.ok) + Number(admission.ok), 1) + if (operation.ok) { + assert.equal(admission.code, 'vm_operation_active') + await operation.release() + } else { + assert.equal(operation.code, 'vm_execution_active') + await admission.release() + } + } +}) + +test('same-VM nested callbacks reuse one persistent operation and preserve callback values', async (t) => { + kubernetes(t) + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = await operations(other, 'owner-b') + const result = await first.run('vm-a', async (outer) => { + return withVmLock( + '/another-pod/vms/vm-a.json', + async (inner) => { + assert.equal(inner.operationId, outer.operationId) + assert.equal( + (await other.prepare('SELECT count(*) AS n FROM vm_operations WHERE released_at IS NULL').get()).n, + 1, + ) + return 42 + }, + { operations: second }, + ) + }) + assert.equal(result, 42) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM vm_operations WHERE released_at IS NULL').get()).n, 0) +}) + +test('normal completion releases, exceptions and explicit uncertainty retain the VM without TTL', async (t) => { + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = await operations(other, 'owner-b') + await first.run('vm-normal', async () => 'done') + const next = await second.tryAcquire({ vmId: 'vm-normal' }) + assert.equal(next.ok, true) + await next.release() + await assert.rejects( + first.run('vm-error', async () => { + throw new Error('synthetic unknown remote result') + }), + /synthetic unknown/, + ) + await first.run('vm-unknown', async () => ({ vmOperationUncertain: true })) + await getDb().prepare("UPDATE vm_operations SET created_at = clock_timestamp() - interval '10 years'").run() + for (const vmId of ['vm-error', 'vm-unknown']) { + assert.equal((await second.tryAcquire({ vmId })).code, 'vm_operation_active') + assert.equal((await new ExecutionAdmissionsRepo(other).tryReserve(execution({ vmId }))).code, 'vm_operation_active') + } +}) + +test('explicitly known not-started errors release and rethrow, while unknown release never frees capacity', async (t) => { + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = await operations(other, 'owner-b') + await assert.rejects( + first.run('vm-safe', () => { + throw Object.assign(new Error('validation failed'), { vmOperationNotStarted: true }) + }), + /validation failed/, + ) + const next = await second.tryAcquire({ vmId: 'vm-safe' }) + assert.equal(next.ok, true) + assert.deepEqual(await first.release({ operationId: next.operationId, ownerId: 'owner-a' }), { released: false }) + await next.markUncertain() + assert.deepEqual(await next.release(), { released: false }) + assert.equal((await first.tryAcquire({ vmId: 'vm-safe' })).code, 'vm_operation_active') +}) + +test('closing the owner PG connection leaves the committed operation unavailable', async (t) => { + const other = await otherDatabase(t) + const owner = await operations(other, 'owner-b') + const observer = await operations(getDb(), 'owner-a') + assert.equal((await owner.tryAcquire({ vmId: 'vm-disconnected' })).ok, true) + await other.close() + assert.equal((await observer.tryAcquire({ vmId: 'vm-disconnected' })).code, 'vm_operation_active') +}) + +test('SIGKILL of an operation owner does not make its VM available', async (t) => { + const observer = await operations(getDb(), 'observer') + const databaseUrl = new URL('../../src/lib/db/database.mjs', import.meta.url).href + const serviceUrl = new URL('../../src/lib/vm/shared-vm-operations.mjs', import.meta.url).href + const script = `import { createDatabase } from ${JSON.stringify(databaseUrl)}; + import { SharedVmOperations } from ${JSON.stringify(serviceUrl)}; + const db = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: process.env.KIN_DATABASE_SCHEMA }); + const claim = await new SharedVmOperations(db, { ownerId: 'killed-owner' }).tryAcquire({ vmId: 'vm-killed' }); + process.stdout.write(JSON.stringify({ ok: claim.ok }) + '\\n'); + setInterval(() => {}, 1000);` + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'pipe'] }) + t.after(() => child.kill('SIGKILL')) + const lines = createInterface({ input: child.stdout }) + const ready = await Promise.race([ + once(lines, 'line').then(([line]) => JSON.parse(line)), + once(child, 'exit').then(() => { + throw new Error('owner exited before claim') + }), + ]) + assert.equal(ready.ok, true) + const exited = once(child, 'exit') + child.kill('SIGKILL') + await exited + assert.equal((await observer.tryAcquire({ vmId: 'vm-killed' })).code, 'vm_operation_active') +}) + +test('Kubernetes canonical key validation and PG failures never execute a local fallback', async (t) => { + kubernetes(t) + let called = false + await assert.rejects( + withVmLock('/pod-a/elsewhere/vm-a.json', () => { + called = true + }), + { code: 'invalid_vm_operation' }, + ) + const other = await otherDatabase(t) + const service = await operations(other, 'owner-b') + await other.close() + await assert.rejects( + withVmLock( + 'explicit-key', + () => { + called = true + }, + { operations: service, vmId: 'vm-a' }, + ), + ) + assert.equal(called, false) + assert.equal(vmLockBusy('explicit-key'), false) +}) From 1bac28f1f7fb12dcf1feadf5aa60f46a98e1922c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:43:15 +0800 Subject: [PATCH 123/473] test: reproduce stale runtime producer observations --- test/unit/runtime-producers.test.mjs | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test/unit/runtime-producers.test.mjs diff --git a/test/unit/runtime-producers.test.mjs b/test/unit/runtime-producers.test.mjs new file mode 100644 index 000000000..32cd8707c --- /dev/null +++ b/test/unit/runtime-producers.test.mjs @@ -0,0 +1,67 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' +import { SchedulingRuntime } from '../../src/lib/pool/scheduling-runtime.mjs' +import { getDb } from '../../src/lib/db/database.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +test('worker health acquires its observation ticket before remote I/O', async () => { + const events = [] + let persisted + const pool = new PoolScheduler({ + projectRoot: '/tmp/vm2api-648-runtime-producer', + workerHealth: async () => { + events.push('health') + return { ok: true, credential: { generation: 1 } } + }, + runtimeRepo: { + beginObservation: async () => { + events.push('ticket') + return 41 + }, + get: () => null, + upsert: (patch, options) => { + persisted = { patch, options } + }, + }, + }) + await pool.getWorkerHealth({ vmId: 'vm-health', accountId: 'account-health' }) + assert.deepEqual(events, ['ticket', 'health']) + assert.equal(persisted.options?.observation, 41) +}) + +test('request success cannot restore the worker snapshot captured before a newer heartbeat', async () => { + const runtime = await new SchedulingRuntime(getDb()).init() + runtime.upsert({ account_id: 'account-success', vm_id: 'vm-success', worker_status: { generation: 1 } }) + await runtime.flush() + const selected = { accountId: 'account-success', vmId: 'vm-success', state: runtime.get('account-success') } + runtime.upsert({ account_id: 'account-success', worker_status: { generation: 2 } }) + await runtime.flush() + const pool = new PoolScheduler({ projectRoot: '/tmp/vm2api-648-runtime-producer', runtimeRepo: runtime }) + pool.markSuccess(selected) + await runtime.flush() + assert.equal((await runtime.persistence.get('account-success')).worker_status.generation, 2) +}) + +test('runtime observation tickets retain local invocation order while allocation is delayed', async () => { + const runtime = new SchedulingRuntime(getDb()) + const first = deferred() + let calls = 0 + runtime.persistence.beginObservation = () => { + calls++ + return calls === 1 ? first.promise : Promise.resolve(2) + } + const a = runtime.beginObservation() + const b = runtime.beginObservation() + await Promise.resolve() + assert.equal(calls, 1) + first.resolve(1) + assert.deepEqual(await Promise.all([a, b]), [1, 2]) +}) From f74ac9f29e6d7f2808f790669c916c8b88790e7b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:44:05 +0800 Subject: [PATCH 124/473] test: cover VM operation ownership and callback boundaries --- test/unit/shared-vm-operations.test.mjs | 122 +++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/test/unit/shared-vm-operations.test.mjs b/test/unit/shared-vm-operations.test.mjs index d36c7951b..8a982f760 100644 --- a/test/unit/shared-vm-operations.test.mjs +++ b/test/unit/shared-vm-operations.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import { spawn } from 'node:child_process' import { once } from 'node:events' import { createInterface } from 'node:readline' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { withVmLock, vmLockBusy } from '../../src/lib/vm/vm-file.mjs' @@ -68,6 +68,7 @@ test('an active Kubernetes VM mutation blocks new execution on another PG connec await finish.promise }) await entered.promise + assert.equal(vmLockBusy('/pod-a/vms/vm-a.json'), true) let gate try { gate = await new ExecutionAdmissionsRepo(other).tryReserve(execution()) @@ -77,6 +78,7 @@ test('an active Kubernetes VM mutation blocks new execution on another PG connec await gate?.release?.() finish.resolve() await running + assert.equal(vmLockBusy('/pod-a/vms/vm-a.json'), false) } }) @@ -244,3 +246,121 @@ test('Kubernetes canonical key validation and PG failures never execute a local assert.equal(called, false) assert.equal(vmLockBusy('explicit-key'), false) }) + +test('two independent operation owners have one winner and unrelated VMs remain available', async (t) => { + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = await operations(other, 'owner-b') + const claims = await Promise.all([first.tryAcquire({ vmId: 'vm-a' }), second.tryAcquire({ vmId: 'vm-a' })]) + assert.equal(claims.filter((claim) => claim.ok).length, 1) + assert.equal(claims.find((claim) => !claim.ok).code, 'vm_operation_active') + const independent = await second.tryAcquire({ vmId: 'vm-b' }) + assert.equal(independent.ok, true) + await independent.release() + const winner = claims.find((claim) => claim.ok) + assert.deepEqual(await winner.release(), { released: true }) + assert.deepEqual(await winner.release(), { released: true }) +}) + +test('a failed release remains persistent and retries PostgreSQL rather than caching its rejection', async (t) => { + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = await operations(other, 'owner-b') + const claim = await first.tryAcquire({ vmId: 'vm-a' }) + await getDb().exec(`CREATE FUNCTION refuse_vm_operation_release() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.released_at IS NOT NULL THEN RAISE EXCEPTION 'synthetic release failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER refuse_vm_operation_release BEFORE UPDATE ON vm_operations FOR EACH ROW EXECUTE FUNCTION refuse_vm_operation_release()`) + await assert.rejects(claim.release(), { code: 'P0001' }) + assert.equal((await second.tryAcquire({ vmId: 'vm-a' })).code, 'vm_operation_active') + await getDb().exec('DROP TRIGGER refuse_vm_operation_release ON vm_operations') + assert.deepEqual(await claim.release(), { released: true }) + const next = await second.tryAcquire({ vmId: 'vm-a' }) + assert.equal(next.ok, true) + await next.release() +}) + +test('different schemas cannot reuse one another nested operation context', async (t) => { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema + '_other' }) + t.after(() => other.close()) + const first = await operations(db, 'owner-a') + const second = await operations(other, 'owner-b') + await first.run('vm-a', async (outer) => + second.run('vm-a', async (inner) => { + assert.notEqual(inner.operationId, outer.operationId) + assert.equal( + (await other.prepare('SELECT count(*) AS n FROM vm_operations WHERE released_at IS NULL').get()).n, + 1, + ) + }), + ) +}) + +test('a claim cannot be created in an outer transaction whose later rollback would erase ownership', async () => { + const service = await operations(getDb(), 'owner-a') + let called = false + await assert.rejects( + withTransaction(getDb(), () => + service.run('vm-a', () => { + called = true + }), + ), + { code: 'vm_operation_transaction_active' }, + ) + assert.equal(called, false) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM vm_operations').get()).n, 0) +}) + +test('async work escaping a finished callback cannot reuse its old claim', async (t) => { + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = await operations(other, 'owner-b') + const proceed = deferred() + let late + await first.run('vm-a', async () => { + late = proceed.promise.then(() => first.run('vm-a', () => 'unexpected callback')) + }) + const current = await second.tryAcquire({ vmId: 'vm-a' }) + proceed.resolve() + await assert.rejects(late, { code: 'vm_operation_active' }) + await current.release() +}) + +test('async work escaping a failed callback cannot reuse the claim while uncertainty is being persisted', async (t) => { + const other = await otherDatabase(t) + const first = await operations(getDb(), 'owner-a') + const second = await operations(other, 'owner-b') + const mark = first.markUncertain.bind(first) + const marking = deferred() + const finish = deferred() + const proceed = deferred() + first.markUncertain = async (identity) => { + marking.resolve() + await finish.promise + return mark(identity) + } + let late + const failed = assert.rejects( + first.run('vm-a', async () => { + late = proceed.promise.then(() => second.run('vm-a', () => 'unexpected callback')) + throw new Error('synthetic callback failure') + }), + /synthetic callback failure/, + ) + await marking.promise + proceed.resolve() + try { + await assert.rejects(late, { code: 'vm_operation_active' }) + } finally { + finish.resolve() + await failed + } +}) + +test('an explicitly supplied VM id still rejects reserved file identities', async (t) => { + kubernetes(t) + await assert.rejects( + withVmLock('explicit-key', () => {}, { vmId: 'active' }), + { code: 'invalid_vm_operation' }, + ) +}) From 01acbab793ab85833d0590348f9f74611e0a6515 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:44:42 +0800 Subject: [PATCH 125/473] test: reproduce shared sticky runner submission races --- test/unit/shared-sticky-runner.test.mjs | 100 ++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 test/unit/shared-sticky-runner.test.mjs diff --git a/test/unit/shared-sticky-runner.test.mjs b/test/unit/shared-sticky-runner.test.mjs new file mode 100644 index 000000000..00035a283 --- /dev/null +++ b/test/unit/shared-sticky-runner.test.mjs @@ -0,0 +1,100 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { StickyRouter } from '../../src/lib/pool/sticky-router.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' + +async function fixture(t) { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const config = { sticky: { enabled: true, ttl_seconds: 60 } } + const [router, peer] = await Promise.all([ + new StickyRouter({ db, config, shared: true }).init(), + new StickyRouter({ db: other, config, shared: true }).init(), + ]) + t.after(async () => { + await router.flush() + await peer.flush() + await other.close() + }) + const scheduler = { + async selectAndReserve() { + return { ok: true, accountId: 'account-a', vmId: 'vm-a', release() {} } + }, + markSuccess() {}, + } + return { router, peer, runner: new FailoverRunner({ scheduler, stickyRouter: router }) } +} + +function success() { + return { ok: true, status: 200, terminalState: 'verified', body: { content: [{ type: 'text', text: 'ok' }] } } +} + +for (const initial of ['absent', 'tombstone']) { + test(`runner durably binds ${initial} shared session before kernel submission`, async (t) => { + const fx = await fixture(t) + const key = `runner-${initial}` + if (initial === 'tombstone') { + await fx.router.bindShared(key, { accountId: 'account-a', vmId: 'vm-a' }) + await fx.router.unbindShared(key) + } + let calls = 0 + const result = await fx.runner.run({ + requestId: key, + stickyKey: key, + canonicalBody: { model: 'claude-test' }, + applyAttempt: () => ({ body: {}, meta: { sessionId: 'native-a' } }), + callAttempt: async () => { + calls++ + const stored = await fx.peer.persistence.get(key) + assert.equal(stored.account_id, 'account-a') + assert.equal(stored.session_id, 'native-a') + return success() + }, + }) + assert.equal(result.ok, true) + assert.equal(calls, 1) + }) +} + +test('runner rejects shared native-session CAS conflict before submission', async (t) => { + const fx = await fixture(t) + const key = 'runner-pre-submit-conflict' + let calls = 0 + const result = await fx.runner.run({ + requestId: key, + stickyKey: key, + canonicalBody: { model: 'claude-test' }, + applyAttempt: async () => { + await fx.peer.refresh() + await fx.peer.rebindFamilyShared(key, { accountId: 'account-b', vmId: 'vm-b' }, { ifGeneration: fx.peer.versionOf(key) }) + return { body: {}, meta: { sessionId: 'native-a' } } + }, + callAttempt: async () => { calls++; return success() }, + }) + assert.equal(calls, 0) + assert.equal(result.status, 503) + assert.equal(result.body.error.code, 'sticky_binding_conflict') + assert.equal((await fx.peer.persistence.get(key)).vm_id, 'vm-b') +}) + +test('runner late success preserves shared pin moved during execution', async (t) => { + const fx = await fixture(t) + const key = 'runner-late-success' + const result = await fx.runner.run({ + requestId: key, + stickyKey: key, + canonicalBody: { model: 'claude-test' }, + applyAttempt: () => ({ body: {}, meta: { sessionId: 'native-a' } }), + callAttempt: async () => { + await fx.peer.refresh() + await fx.peer.rebindFamilyShared(key, { accountId: 'account-b', vmId: 'vm-b' }, { ifGeneration: fx.peer.versionOf(key) }) + await fx.router.refresh() + return success() + }, + }) + assert.equal(result.ok, true) + const stored = await fx.peer.persistence.get(key) + assert.equal(stored.vm_id, 'vm-b') + assert.equal(stored.session_id, null) +}) From f27455d9ddc2b34a702a2a9bc5abe65888d964de Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:44:50 +0800 Subject: [PATCH 126/473] test(k8s): require shared sticky confirmation before Codex execution --- test/unit/shared-sticky-producers.test.mjs | 235 +++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 test/unit/shared-sticky-producers.test.mjs diff --git a/test/unit/shared-sticky-producers.test.mjs b/test/unit/shared-sticky-producers.test.mjs new file mode 100644 index 000000000..b350d7dd0 --- /dev/null +++ b/test/unit/shared-sticky-producers.test.mjs @@ -0,0 +1,235 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { setImmediate } from 'node:timers/promises' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { StickyRouter } from '../../src/lib/pool/sticky-router.mjs' +import { resetOpenAIAccountRuntime } from '../../src/lib/pool/openai-account-runtime.mjs' +import { handleCodexProtocol } from '../../src/lib/protocol/handle-codex.mjs' +import { peekCurrentAccount } from '../../src/lib/protocol/user-count-tokens.mjs' +import { clearVmCooldown } from '../../src/lib/admin/panel-api.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function fixture(t) { + resetOpenAIAccountRuntime() + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-sticky-producer-')) + fs.mkdirSync(path.join(root, 'vms', 'vm-gpt-a'), { recursive: true }) + fs.writeFileSync( + path.join(root, 'vms', 'vm-gpt-a.json'), + JSON.stringify({ + id: 'vm-gpt-a', + platform: 'openai', + family: 'codex', + schedulable: true, + status: 'running', + has_token: true, + }), + ) + fs.writeFileSync( + path.join(root, 'vms', 'vm-gpt-a', 'codex-credentials.json'), + JSON.stringify({ + accounts: [ + { + id: 'synthetic-account', + access_token: 'synthetic-access', + refresh_token: 'synthetic-refresh', + chatgpt_account_id: 'synthetic-account', + }, + ], + }), + ) + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const config = { sticky: { enabled: true, ttl_seconds: 60 } } + const [router, peer] = await Promise.all([ + new StickyRouter({ db, config, shared: true }).init(), + new StickyRouter({ db: other, config, shared: true }).init(), + ]) + t.after(async () => { + await router.flush() + await peer.flush() + await other.close() + fs.rmSync(root, { recursive: true, force: true }) + }) + const req = { headers: { 'x-session-id': 'shared-codex-session' }, apiKeyKind: 'user' } + const body = { model: 'gpt-5.4', input: 'synthetic request', stream: false } + const key = router.extractPoolKey(req, body, { platform: 'openai' }) + const response = { headersSent: false, write() {}, end() {} } + const jobs = [] + return { + root, + router, + peer, + key, + response, + jobs, + run({ ensure, hop } = {}) { + return handleCodexProtocol({ + req, + res: response, + protocol: 'openai.responses', + ctx: { body }, + inbound: body, + logBag: {}, + stats: { errors: 0, requests: 0, by_route: {} }, + projectRoot: root, + stickyRouter: router, + json(_res, status, value) { + response.status = status + response.body = value + return value + }, + writeSSEHeaders() { + response.headersSent = true + }, + ops: { + writeCodexKernelConfig() {}, + ensureCodexKernel: ensure || (async () => ({ ok: true })), + streamCodexKernel: async (args) => { + jobs.push(args) + if (hop) await hop(args) + return { ok: true, status: 200, terminalState: 'verified', body: { id: 'resp-synthetic' } } + }, + }, + }) + }, + } +} + +test('Codex awaits durable shared native-session binding before submitting a kernel job', async (t) => { + const fx = await fixture(t) + await fx.run({ + hop: async ({ envelope }) => { + const stored = await fx.peer.persistence.get(fx.key) + assert.ok(stored, 'a shared conversation must be bound before kernel submission') + assert.equal(stored.vm_id, 'vm-gpt-a') + assert.equal(stored.session_id, envelope.session.session_id) + assert.equal(envelope.body.prompt_cache_key, stored.session_id) + }, + }) + assert.equal(fx.jobs.length, 1) + assert.equal(fx.response.status, 200) +}) + +test('a shared Codex pre-submission CAS conflict returns 503 without sending the old body', async (t) => { + const fx = await fixture(t) + await fx.run({ + ensure: async () => { + await fx.peer.bindShared( + fx.key, + { accountId: 'vm-gpt-b', vmId: 'vm-gpt-b', sessionId: 'native-peer' }, + { ifGeneration: 0 }, + ) + return { ok: true } + }, + hop: () => assert.fail('a conflicting native session cannot receive the stale outbound body'), + }) + assert.equal(fx.jobs.length, 0) + assert.equal(fx.response.status, 503) + assert.equal(fx.response.body.error.code, 'sticky_binding_conflict') + assert.equal((await fx.peer.persistence.get(fx.key)).session_id, 'native-peer') +}) + +test('a late Codex success cannot restore a pin moved by another controller', async (t) => { + const fx = await fixture(t) + await fx.run({ + hop: async () => { + await fx.peer.refresh() + await fx.peer.rebindFamilyShared( + fx.key, + { accountId: 'vm-gpt-b', vmId: 'vm-gpt-b' }, + { ifGeneration: fx.peer.versionOf(fx.key) }, + ) + }, + }) + assert.equal(fx.response.status, 200) + const stored = await fx.peer.persistence.get(fx.key) + assert.equal(stored.vm_id, 'vm-gpt-b') + assert.equal(stored.session_id, null) +}) + +test('count_tokens awaits the shared read-only session helper with migrate disabled', async () => { + const entered = deferred() + const resume = deferred() + let peeked = false + const pending = peekCurrentAccount({ + req: { headers: {}, apiKeyKind: 'master' }, + inbound: { + metadata: { user_id: JSON.stringify({ session_id: 'native-count-session', device_id: 'count-device' }) }, + }, + model: 'claude-test', + stickyRouter: { + shared: true, + sessionPoolKeys() { + throw new Error('shared callers must not use the synchronous migration helper') + }, + async sessionPoolKeysShared(_req, _body, options) { + assert.equal(options.migrate, false) + entered.resolve() + await resume.promise + return { stickyKey: 'shared-count-key', stickyKeys: ['shared-count-key'] } + }, + }, + poolScheduler: { + async peekAccount({ stickyKey }) { + peeked = true + assert.equal(stickyKey, 'shared-count-key') + return { ok: true } + }, + }, + }) + try { + await Promise.race([entered.promise, pending.then(() => assert.fail('read-only helper was not awaited'))]) + await setImmediate() + assert.equal(peeked, false) + } finally { + resume.resolve() + } + assert.equal((await pending).ok, true) +}) + +test('panel cooldown clearing awaits shared sticky deletion before returning', async (t) => { + const fx = await fixture(t) + await fx.router.bindShared(fx.key, { accountId: 'vm-gpt-a', vmId: 'vm-gpt-a', sessionId: 'native-panel' }) + const entered = deferred() + const resume = deferred() + const remove = fx.router.unbindByAccountShared.bind(fx.router) + fx.router.unbindByAccountShared = async (options) => { + entered.resolve() + await resume.promise + return remove(options) + } + let settled = false + const pending = Promise.resolve( + clearVmCooldown({ + cfg: { paths: { project: fx.root } }, + id: 'vm-gpt-a', + stickyRouter: fx.router, + accountQuota: { repo: { get: () => ({ unified: {} }) } }, + }), + ).then((result) => { + settled = true + return result + }) + try { + await Promise.race([ + entered.promise, + pending.then(() => assert.fail('panel returned without shared sticky deletion')), + ]) + await setImmediate() + assert.equal(settled, false) + } finally { + resume.resolve() + } + assert.equal((await pending).ok, true) + assert.equal(await fx.peer.persistence.get(fx.key), null) +}) From 04d633f75eaf2b6268a42c5de132e2c2042bc529 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:45:34 +0800 Subject: [PATCH 127/473] fix: coordinate persistent VM operations with execution admission --- .../postgres-migrations/012_vm_operations.sql | 12 ++ .../db/repos/execution-admissions-repo.mjs | 4 + src/lib/vm/shared-vm-operations.mjs | 160 ++++++++++++++++++ src/lib/vm/vm-file.mjs | 44 ++++- 4 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 src/lib/db/postgres-migrations/012_vm_operations.sql create mode 100644 src/lib/vm/shared-vm-operations.mjs diff --git a/src/lib/db/postgres-migrations/012_vm_operations.sql b/src/lib/db/postgres-migrations/012_vm_operations.sql new file mode 100644 index 000000000..9865a39d5 --- /dev/null +++ b/src/lib/db/postgres-migrations/012_vm_operations.sql @@ -0,0 +1,12 @@ +CREATE TABLE vm_operations ( + id text COLLATE "C" PRIMARY KEY, + vm_id text COLLATE "C" NOT NULL, + owner_id text COLLATE "C" NOT NULL, + kind text COLLATE "C" NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + uncertain_at timestamptz, + released_at timestamptz +); + +-- Neither elapsed time nor a controller disconnect proves remote mutation ended. +CREATE UNIQUE INDEX idx_vm_operations_vm_active ON vm_operations (vm_id) WHERE released_at IS NULL; diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 5563697ed..ac1701e4c 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -92,6 +92,10 @@ export class ExecutionAdmissionsRepo { const family = input.family == null ? null : identifier(input.family, 'family') return withTransaction(this.db, async () => { await this.lockResources({ accountId, vmIds: [vmId] }) + const operation = await this.db + .prepare('SELECT id FROM vm_operations WHERE vm_id = $1 AND released_at IS NULL') + .get(vmId) + if (operation) return denied('vm_operation_active', 0, vmLimit) const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() await this.db .prepare(`DELETE FROM execution_admissions WHERE account_id = $1 diff --git a/src/lib/vm/shared-vm-operations.mjs b/src/lib/vm/shared-vm-operations.mjs new file mode 100644 index 000000000..696b4317b --- /dev/null +++ b/src/lib/vm/shared-vm-operations.mjs @@ -0,0 +1,160 @@ +import crypto from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' +import { withTransaction } from '../db/database.mjs' +import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' + +const callbacks = new AsyncLocalStorage() + +function operationError(code, status = 409) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code, status }) +} + +function validateOperationVmId(vmId) { + if (typeof vmId !== 'string' || !/^[A-Za-z0-9_-]+$/.test(vmId)) { + throw operationError('invalid_vm_operation', 400) + } + return vmId +} + +function identifier(value) { + if (typeof value !== 'string' || !value.trim()) throw operationError('invalid_vm_operation', 400) + return value +} + +/** Durable mutual exclusion with execution admission; unknown operations require explicit recovery. */ +export class SharedVmOperations { + constructor(db, { ownerId = crypto.randomUUID() } = {}) { + this.db = db + this.ownerId = identifier(ownerId) + this.executions = new ExecutionAdmissionsRepo(db, { ownerId: this.ownerId }) + } + + async tryAcquire({ vmId, kind = 'mutation' } = {}) { + validateOperationVmId(vmId) + identifier(kind) + // The claim must commit independently before its owner can perform remote effects. + if (this.db.isTransaction) throw operationError('vm_operation_transaction_active', 500) + return withTransaction(this.db, async () => { + // Identical namespace and VM resource to ExecutionAdmissionsRepo.tryReserve. + await this.executions.lockVm(vmId) + const active = await this.db + .prepare('SELECT id FROM vm_operations WHERE vm_id = $1 AND released_at IS NULL') + .get(vmId) + if (active) return { ok: false, code: 'vm_operation_active' } + const execution = await this.db + .prepare('SELECT id FROM execution_admissions WHERE vm_id = $1 AND released_at IS NULL LIMIT 1') + .get(vmId) + if (execution) return { ok: false, code: 'vm_execution_active' } + const operationId = crypto.randomUUID() + await this.db + .prepare('INSERT INTO vm_operations (id, vm_id, owner_id, kind) VALUES ($1, $2, $3, $4)') + .run(operationId, vmId, this.ownerId, kind) + return this._handle({ operationId, vmId, ownerId: this.ownerId, kind }) + }) + } + + _handle(identity) { + let uncertain = false + let marking = null + let releasing = null + return { + ok: true, + ...identity, + get uncertain() { + return uncertain + }, + markUncertain: () => { + uncertain = true + if (!marking) { + const pending = this.markUncertain(identity) + marking = pending + pending.then( + (result) => { + if (!result.uncertain && marking === pending) marking = null + }, + () => { + if (marking === pending) marking = null + }, + ) + } + return marking + }, + release: async () => { + if (uncertain) { + if (marking) await marking + return { released: false } + } + if (!releasing) { + const pending = this.release(identity) + releasing = pending + pending.then( + (result) => { + if (!result.released && releasing === pending) releasing = null + }, + () => { + if (releasing === pending) releasing = null + }, + ) + } + return releasing + }, + } + } + + async markUncertain({ operationId, ownerId }) { + const result = await this.db + .prepare(`UPDATE vm_operations + SET uncertain_at = COALESCE(uncertain_at, clock_timestamp()) + WHERE id = $1 AND owner_id = $2 AND released_at IS NULL`) + .run(operationId, ownerId) + return { uncertain: result.changes > 0 } + } + + async release({ operationId, ownerId }) { + const result = await this.db + .prepare(`UPDATE vm_operations + SET released_at = COALESCE(released_at, clock_timestamp()) + WHERE id = $1 AND owner_id = $2 AND uncertain_at IS NULL`) + .run(operationId, ownerId) + return { released: result.changes > 0 } + } + + async run(vmId, callback, { kind = 'mutation' } = {}) { + validateOperationVmId(vmId) + const parents = callbacks.getStore() || [] + const nested = parents.findLast( + (context) => + context.active && + context.vmId === vmId && + context.db.schema === this.db.schema && + context.db.connectionString === this.db.connectionString, + ) + if (nested) return callback(nested.operation) + const operation = await this.tryAcquire({ vmId, kind }) + if (!operation.ok) throw operationError(operation.code) + const context = { db: this.db, vmId, operation, active: true } + let result + try { + result = await callbacks.run([...parents, context], () => callback(operation)) + } catch (error) { + context.active = false + if (error?.vmOperationNotStarted === true && !operation.uncertain) { + const released = await operation.release() + if (!released.released) throw operationError('vm_operation_release_unconfirmed') + } else { + await operation.markUncertain() + } + throw error + } finally { + // Descendant async work must not reuse the claim after its callback has returned. + context.active = false + } + if (result?.vmOperationUncertain === true || operation.uncertain) { + await operation.markUncertain() + } else { + const released = await operation.release() + if (!released.released) throw operationError('vm_operation_release_unconfirmed') + } + return result + } +} diff --git a/src/lib/vm/vm-file.mjs b/src/lib/vm/vm-file.mjs index 41124e9ec..09e28b443 100644 --- a/src/lib/vm/vm-file.mjs +++ b/src/lib/vm/vm-file.mjs @@ -3,14 +3,17 @@ * * - atomicWriteJson: write to a temp file in the same dir then rename, so a * concurrent reader (or a crash mid-write) never sees a torn/partial file. - * - withVmLock: per-path async mutex to serialize read-modify-write sequences - * that span `await` boundaries (e.g. the admin sessionKey import path). + * - withVmLock: standalone per-path async mutex; Kubernetes uses a durable + * per-VM claim shared with execution admission. * - writeJsonIfChanged: skip the write entirely when the on-disk content is * byte-identical, cutting per-request write amplification. */ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' +import { getDb } from '../db/database.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { SharedVmOperations } from './shared-vm-operations.mjs' /** * Names inside `vms/` that are not slot records: `active.json` is the active @@ -149,6 +152,7 @@ export function invalidateVmSlotReadCache(projectRoot, id) { const locks = new Map() const pendingLocks = new Map() +const sharedOperations = new WeakMap() /** Includes queued operations: connection changes must not interrupt an awaiting VM mutation. */ export function vmLockBusy(key) { @@ -173,7 +177,41 @@ function notifyVmWrite(filePath) { } catch {} } -export function withVmLock(key, fn) { +export function withVmLock(key, fn, { operations, vmId, kind = 'mutation' } = {}) { + if (operations || isKubernetesMode()) { + if (vmId == null) { + const name = path.basename(String(key || '')) + if ( + path.basename(path.dirname(String(key || ''))) !== 'vms' || + !name.endsWith('.json') || + !isValidVmId(name.slice(0, -5)) + ) { + return Promise.reject( + Object.assign(new Error('invalid VM operation key'), { code: 'invalid_vm_operation', status: 400 }), + ) + } + vmId = name.slice(0, -5) + } + try { + if (!isValidVmId(vmId)) { + throw Object.assign(new Error('invalid VM operation ID'), { code: 'invalid_vm_operation', status: 400 }) + } + if (!operations) { + const db = getDb() + if (!sharedOperations.has(db)) sharedOperations.set(db, new SharedVmOperations(db)) + operations = sharedOperations.get(db) + } + const run = operations.run(vmId, fn, { kind }) + pendingLocks.set(key, (pendingLocks.get(key) || 0) + 1) + return run.finally(() => { + const pending = pendingLocks.get(key) - 1 + if (pending) pendingLocks.set(key, pending) + else pendingLocks.delete(key) + }) + } catch (error) { + return Promise.reject(error) + } + } const prev = locks.get(key) || Promise.resolve() pendingLocks.set(key, (pendingLocks.get(key) || 0) + 1) const run = prev.then(fn, fn).finally(() => { From f7eba5d16416b890153340b462f48b52c7e160b5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:47:38 +0800 Subject: [PATCH 128/473] test: retain original quota runtime observations --- test/unit/runtime-quota-producers.test.mjs | 162 +++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 test/unit/runtime-quota-producers.test.mjs diff --git a/test/unit/runtime-quota-producers.test.mjs b/test/unit/runtime-quota-producers.test.mjs new file mode 100644 index 000000000..30cefc5de --- /dev/null +++ b/test/unit/runtime-quota-producers.test.mjs @@ -0,0 +1,162 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setImmediate } from 'node:timers/promises' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SchedulingRuntime } from '../../src/lib/pool/scheduling-runtime.mjs' +import { AccountQuota } from '../../src/lib/pool/account-quota.mjs' +import { RateLimitService } from '../../src/lib/pool/rate-limit-service.mjs' + +async function fixture() { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const left = await new SchedulingRuntime(db).init() + const right = await new SchedulingRuntime(peer).init() + const quota = await new AccountQuota({ db, config: {} }).init() + quota.ensure({ account_id: 'producer-account', vm_id: 'producer-vm' }) + left.upsert({ account_id: 'producer-account', vm_id: 'producer-vm', status: 'ready' }) + await left.flush() + await right.refresh() + quota.attachRuntimeRepo(left) + return { left, right, quota, service: new RateLimitService({ runtimeRepo: left, accountQuota: quota }) } +} + +for (const producer of ['headers', 'oauth-usage']) { + test(`an older AccountQuota ${producer} result cannot overwrite a newer runtime session window`, async () => { + const { left, right, quota } = await fixture() + const older = await quota.beginObservation() + const newer = await right.beginObservation() + const end = Date.now() + 3600_000 + right.updateWindow('producer-account', { + sessionWindowStatus: 'rejected', + sessionWindowEnd: end, + observation: newer, + }) + await right.flush() + if (producer === 'headers') { + quota.ingestHeaders( + 'producer-account', + { + 'anthropic-ratelimit-unified-5h-status': 'allowed', + 'anthropic-ratelimit-unified-5h-reset': new Date(end - 1000).toISOString(), + }, + null, + { observation: older }, + ) + } else { + quota.ingestOAuthUsage( + 'producer-account', + { + ok: true, + usage_status: 200, + limits_present: true, + five_hour: { status: 'allowed', utilization: 10, resets_at: new Date(end - 1000).toISOString() }, + }, + { observation: older }, + ) + } + await left.flush() + const stored = await left.persistence.get('producer-account') + assert.equal(stored.session_window_status, 'rejected') + assert.equal(stored.session_window_end, end) + }) +} + +for (const status of [429, 529]) { + test(`a late ${status} result retains its original observation and cannot extend a newer block`, async () => { + const { left, right, service } = await fixture() + const older = await left.beginObservation() + const newer = await right.beginObservation() + const until = Date.now() + 90 * 60_000 + const field = status === 429 ? 'rate_limit_reset_at' : 'overload_until' + right.updateWindow('producer-account', { + ...(status === 429 ? { rateLimitResetAt: until } : { overloadUntil: until }), + observation: newer, + }) + await right.flush() + await service.handleUpstreamError({ + accountId: 'producer-account', + vmId: 'producer-vm', + result: { status, accountObservation: older, body: { error: { message: 'synthetic limit' } } }, + policy: { scope: 'account', reason: status === 429 ? 'account_quota_exhausted' : 'provider_overloaded' }, + }) + await left.flush() + assert.equal((await left.persistence.get('producer-account'))[field], until) + }) +} + +test('an older allowed response cannot clear a newer runtime rate limit', async () => { + const { left, right, service } = await fixture() + const older = await left.beginObservation() + const newer = await right.beginObservation() + const until = Date.now() + 3600_000 + right.updateWindow('producer-account', { + rateLimitResetAt: until, + sessionWindowStatus: 'rejected', + observation: newer, + }) + await right.flush() + await left.refresh() + await service.updateSessionWindow({ + accountId: 'producer-account', + vmId: 'producer-vm', + observation: older, + headers: { 'anthropic-ratelimit-unified-5h-status': 'allowed' }, + }) + await left.flush() + const stored = await left.persistence.get('producer-account') + assert.equal(stored.rate_limit_reset_at, until) + assert.equal(stored.session_window_status, 'rejected') +}) + +test('quota clearing cannot restore a stale worker snapshot or clear a peer cooldown', async () => { + const { left, right, quota } = await fixture() + left.upsert({ + account_id: 'producer-account', + cooldown_until: Date.now() + 1000, + cooldown_reason: 'quota_guard', + worker_status: { generation: 1 }, + }) + await left.flush() + await right.refresh() + const until = Date.now() + 3600_000 + right.markCooldown('producer-account', { vmId: 'producer-vm', until, reason: 'account_quota_exhausted' }) + right.upsert({ account_id: 'producer-account', worker_status: { generation: 2 } }) + await right.flush() + let callbacks = 0 + quota.onQuotaCooldownCleared = () => { + callbacks++ + } + const cleared = await quota.clearQuotaExhaustedCooldown('producer-account') + await left.flush() + assert.equal(cleared, false) + assert.equal(callbacks, 0) + const stored = await left.persistence.get('producer-account') + assert.equal(stored.cooldown_until, until) + assert.equal(stored.worker_status.generation, 2) +}) + +test('quota-cleared callbacks wait until the queued PG clearing has committed', async () => { + const { left, quota } = await fixture() + left.upsert({ account_id: 'producer-account', cooldown_until: Date.now() + 1000, cooldown_reason: 'quota_guard' }) + await left.flush() + let release + const paused = new Promise((resolve) => { + release = resolve + }) + left.writes.enqueue(() => paused) + let callbacks = 0 + quota.onQuotaCooldownCleared = () => { + callbacks++ + } + const clearing = quota.clearQuotaExhaustedCooldown('producer-account') + try { + await setImmediate() + assert.equal(callbacks, 0, 'cache eligibility is not proof that PG accepted the clearing') + } finally { + release() + await clearing + await left.flush() + } + assert.equal(callbacks, 1) + assert.equal((await left.persistence.get('producer-account')).cooldown_until, null) +}) From 93ba379c5c82dfa4c839f9759886cabaf44c20c5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:47:47 +0800 Subject: [PATCH 129/473] test: cover shared sticky alias and reconciliation races --- test/unit/shared-sticky-runner.test.mjs | 36 ++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/unit/shared-sticky-runner.test.mjs b/test/unit/shared-sticky-runner.test.mjs index 00035a283..280ad34d2 100644 --- a/test/unit/shared-sticky-runner.test.mjs +++ b/test/unit/shared-sticky-runner.test.mjs @@ -27,7 +27,7 @@ async function fixture(t) { } function success() { - return { ok: true, status: 200, terminalState: 'verified', body: { content: [{ type: 'text', text: 'ok' }] } } + return { ok: true, status: 200, terminalState: 'verified', body: { type: 'message', id: 'msg_synthetic', role: 'assistant', content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', usage: { input_tokens: 1, output_tokens: 1 } } } } for (const initial of ['absent', 'tombstone']) { @@ -98,3 +98,37 @@ test('runner late success preserves shared pin moved during execution', async (t assert.equal(stored.vm_id, 'vm-b') assert.equal(stored.session_id, null) }) + +test('scheduler shared alias release uses captured versions and preserves a newer alias', async (t) => { + const { PoolScheduler } = await import('../../src/lib/pool/pool-scheduler.mjs') + const fx = await fixture(t) + const keys = ['runner-alias-one', 'runner-alias-two'] + for (const key of keys) await fx.router.bindShared(key, { accountId: 'account-a', vmId: 'vm-a' }) + const versions = new Map(keys.map((key) => [key, fx.router.versionOf(key)])) + await fx.peer.refresh() + await fx.peer.rebindFamilyShared(keys[1], { accountId: 'account-b', vmId: 'vm-b' }) + const pool = new PoolScheduler({ stickyRouter: fx.router }) + await pool.releaseSticky({ accountId: 'account-a', vmId: 'vm-a' }, keys[0], keys, versions) + assert.equal(await fx.peer.persistence.get(keys[0]), null) + assert.equal((await fx.peer.persistence.get(keys[1])).vm_id, 'vm-b') +}) + +test('scheduler shared reconciliation preserves a pin moved while clearing the old account', async (t) => { + const { PoolScheduler } = await import('../../src/lib/pool/pool-scheduler.mjs') + const fx = await fixture(t) + const key = 'runner-reconcile-conflict' + await fx.router.bindShared(key, { accountId: 'account-a', vmId: 'vm-a' }) + const clear = fx.router.unbindShared.bind(fx.router) + fx.router.unbindShared = async (...args) => { + await fx.peer.refresh() + await fx.peer.rebindFamilyShared(key, { accountId: 'account-b', vmId: 'vm-b' }) + return clear(...args) + } + const pool = new PoolScheduler({ stickyRouter: fx.router }) + const candidate = { accountId: 'account-b', vmId: 'vm-b' } + const result = await pool.reconcileStickyShared({ stickyKey: key, eligible: [candidate] }) + assert.equal(result.cleared, false) + assert.equal(result.vmId, 'vm-b') + assert.deepEqual(result.match, candidate) + assert.equal((await fx.peer.persistence.get(key)).vm_id, 'vm-b') +}) From 9f2607c8da6f3c8e488dcfb01c6289531235860a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:48:59 +0800 Subject: [PATCH 130/473] fix(k8s): confirm shared sticky bindings before Codex submission --- src/lib/admin/panel-api.mjs | 9 ++-- src/lib/protocol/handle-codex.mjs | 61 ++++++++++++++++++---- src/lib/protocol/user-count-tokens.mjs | 24 +++++++-- test/unit/clear-vm-cooldown.test.mjs | 12 ++--- test/unit/shared-sticky-producers.test.mjs | 47 +++++++++++++++-- 5 files changed, 124 insertions(+), 29 deletions(-) diff --git a/src/lib/admin/panel-api.mjs b/src/lib/admin/panel-api.mjs index 151ef11cb..4c9b71537 100644 --- a/src/lib/admin/panel-api.mjs +++ b/src/lib/admin/panel-api.mjs @@ -286,7 +286,7 @@ export function publicPhysicalFeatures(vm) { } /** Clear runtime cooldown, leftover /usage 429 flag, and sticky pins for a slot. */ -export function clearVmCooldown({ cfg, accountQuota, stickyRouter = null, poolScheduler = null, id } = {}) { +export async function clearVmCooldown({ cfg, accountQuota, stickyRouter = null, poolScheduler = null, id } = {}) { const vm = getVm(cfg?.paths?.project, id) if (!vm) { return fail( @@ -308,6 +308,10 @@ export function clearVmCooldown({ cfg, accountQuota, stickyRouter = null, poolSc try { if (repo?.clearAccountCooldown?.(key, { vmId: vm.id })) runtimeCleared += 1 } catch {} + try { + if (stickyRouter?.shared) await stickyRouter.unbindByAccountShared({ accountId: key, vmId: vm.id }) + else stickyRouter?.unbindByAccount?.({ accountId: key, vmId: vm.id }) + } catch {} try { const row = accountQuota?.repo?.get?.(key) if (!row?.unified) continue @@ -326,9 +330,6 @@ export function clearVmCooldown({ cfg, accountQuota, stickyRouter = null, poolSc } if (changed) accountQuota.repo.save(row) } catch {} - try { - stickyRouter?.unbindByAccount?.({ accountId: key, vmId: vm.id }) - } catch {} } const vmPath = path.join(cfg.paths.project, 'vms', `${vm.id}.json`) try { diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index 9b604d25c..c5bf06bbc 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -234,8 +234,10 @@ export async function pickCodexCandidates( policy: routingPolicy, }) } + if (stickyRouter?.shared) await stickyRouter.refresh() const stickyKeys = stickyRouter?.collectPoolKeys?.(req, body || {}, { platform: 'openai' }) || [] const sessionKey = stickyRouter?.extractPoolKey?.(req, body || {}, { platform: 'openai' }) || stickyKeys[0] || null + const stickyVersions = new Map(stickyKeys.map((key) => [key, stickyRouter?.shared ? stickyRouter.versionOf(key) : 0])) const bound = sessionKey ? stickyRouter?.resolve?.(sessionKey) : null const vms = listVms(projectRoot, { codex: { quota: routingPolicy } }).filter((vm) => keyAllowsVm(scope, vm)) const continuesResponse = !!body?.previous_response_id && !!bound?.vmId @@ -253,7 +255,15 @@ export async function pickCodexCandidates( // A resting home migrates the conversation like a gone one: the next slot rebinds it. const boundLeft = ordered.boundState === 'gone' || ordered.boundState === 'resting' if (bound?.vmId && sessionKey && boundLeft) { - for (const key of stickyKeys.length ? stickyKeys : [sessionKey]) stickyRouter?.unbind?.(key) + for (const key of stickyKeys.length ? stickyKeys : [sessionKey]) { + if (stickyRouter?.shared) { + const result = await stickyRouter.unbindShared(key, { + ifGeneration: stickyVersions.get(key) ?? stickyRouter.versionOf(key), + }) + if (!result.applied) return { error: 'sticky_binding_conflict', ids: [], candidates: [] } + stickyVersions.set(key, result.generation) + } else stickyRouter?.unbind?.(key) + } try { sessions?.drop?.(bound.vmId, sessionKey) } catch {} @@ -261,7 +271,7 @@ export async function pickCodexCandidates( if (continuesResponse && (boundLeft || ordered.error === 'candidates_exhausted')) { return { error: 'response_not_portable', ids: [], sessionKey, stickyKeys, boundVmId: bound.vmId } } - return { ...ordered, sessionKey, stickyKeys } + return { ...ordered, sessionKey, stickyKeys, ...(stickyRouter?.shared ? { stickyVersions } : {}) } } function ingestCodexHop(projectRoot, vmId, result, now = Date.now(), policy = null) { @@ -654,19 +664,31 @@ export async function handleCodexProtocol({ windowGen = sessions?.touch?.(vm.id, picked.sessionKey) ?? null } catch {} } - const ownsPin = !!stickyBound?.vmId && stickyBound.vmId === vm.id - const bindSticky = (sessionId = null) => { - if (!picked.sessionKey || !stickyRouter?.bind) return + let ownsPin = !!stickyBound?.vmId && stickyBound.vmId === vm.id + const stickyVersions = picked.stickyVersions || new Map() + const bindSticky = async (sessionId = null) => { + if (!picked.sessionKey || !stickyRouter) return true const payload = { accountId: vm.id, vmId: vm.id } if (sessionId) payload.sessionId = sessionId - for (const key of stickyKeys) stickyRouter.bind(key, payload) + for (const key of stickyKeys) { + if (stickyRouter.shared) { + const result = await stickyRouter.bindShared(key, payload, { ifGeneration: stickyVersions.get(key) ?? 0 }) + if (!result.applied || result.record?.vmId !== vm.id || result.nativeSessionId !== sessionId) return false + stickyVersions.set(key, result.generation) + ownsPin = true + } else stickyRouter.bind?.(key, payload) + } + return true } - const leaveSticky = () => { + const leaveSticky = async () => { if (!ownsPin || !picked.sessionKey) return try { sessions?.drop?.(vm.id, picked.sessionKey) } catch {} - for (const key of stickyKeys) stickyRouter?.unbind?.(key) + for (const key of stickyKeys) { + if (stickyRouter?.shared) await stickyRouter.unbindShared(key, { ifGeneration: stickyVersions.get(key) ?? 0 }) + else stickyRouter?.unbind?.(key) + } } let attemptKind = 'failed' try { @@ -741,7 +763,18 @@ export async function handleCodexProtocol({ let hopEmitted = false const runHop = async () => await runCodexKernelHop({ - hop, + hop: stickyRouter?.shared + ? async (args) => { + // Confirm each actual submission, including hidden retries. A CAS + // conflict must never send a body carrying a different native session. + if (!(await bindSticky(outboundSessionId))) { + throw Object.assign(new Error('Shared session binding changed before Codex submission'), { + code: 'sticky_binding_conflict', + }) + } + return hop(args) + } + : hop, args: { exec: execFor(projectRoot, vm), signal: gone.signal, @@ -839,7 +872,7 @@ export async function handleCodexProtocol({ const delivered = !conversionError && (result?.ok || (Number(result?.status) === 200 && usageTokens(usage) > 0)) if (delivered) { attemptKind = 'succeeded' - bindSticky(outboundSessionId) + await bindSticky(outboundSessionId) const extracted = extractOpenaiUsage(usage) const serviceTier = responseServiceTier || converted.body?.service_tier || usage?.service_tier || null const disjoint = extracted ? openaiAnthropicUsageFromExtract(extracted) : null @@ -896,12 +929,18 @@ export async function handleCodexProtocol({ } if (isCodexFailoverError(result)) { pickOpts.excluded.add(vm.id) - leaveSticky() + await leaveSticky() switches += 1 if (switches > failoverBudget.maxSwitches || attempts >= failoverBudget.maxAttempts) break continue } break + } catch (error) { + if (error?.code !== 'sticky_binding_conflict') throw error + stats.errors++ + logBag.error_code = error.code + logBag.final_state = 'rejected' + return json(res, 503, { error: { type: 'api_error', code: error.code, message: '会话绑定已变更，请重试请求' } }) } finally { lease.release() if (claimsWindow) { diff --git a/src/lib/protocol/user-count-tokens.mjs b/src/lib/protocol/user-count-tokens.mjs index 774299df8..77b174310 100644 --- a/src/lib/protocol/user-count-tokens.mjs +++ b/src/lib/protocol/user-count-tokens.mjs @@ -104,14 +104,28 @@ export async function peekCurrentAccount({ const detected = detectInboundPlatform(model) const platform = detected.ok ? detected.platform : undefined const identity = resolveInboundIdentity({ inbound, body: inbound, headers: req?.headers || {} }) - const stickyKey = - (platform === 'anthropic' && identity.sessionId && stickyRouter?.sessionPoolKeys - ? stickyRouter.sessionPoolKeys(req, inbound, { + let stickyKey + if ( + platform === 'anthropic' && + (identity.sessionId || stickyRouter?.shared) && + (stickyRouter?.shared || stickyRouter?.sessionPoolKeys) + ) { + const keys = stickyRouter.shared + ? await stickyRouter.sessionPoolKeysShared(req, inbound, { + sessionId: identity.sessionId, + deviceId: identity.deviceId, + migrate: false, + }) + : stickyRouter.sessionPoolKeys(req, inbound, { sessionId: identity.sessionId, deviceId: identity.deviceId, migrate: false, - }).stickyKey - : stickyRouter?.extractPoolKey?.(req, inbound, { platform })) || null + }) + stickyKey = keys.stickyKey || null + } else { + if (stickyRouter?.shared) await stickyRouter.refresh() + stickyKey = stickyRouter?.extractPoolKey?.(req, inbound, { platform }) || null + } if (!poolScheduler?.peekAccount) { return { ok: false, code: 'no_eligible_accounts' } } diff --git a/test/unit/clear-vm-cooldown.test.mjs b/test/unit/clear-vm-cooldown.test.mjs index 002e96688..7c74c0ee5 100644 --- a/test/unit/clear-vm-cooldown.test.mjs +++ b/test/unit/clear-vm-cooldown.test.mjs @@ -19,14 +19,14 @@ function seedVm(project, id = 'vm-11') { return rec } -test('clearVmCooldown 404s unknown slots', () => { +test('clearVmCooldown 404s unknown slots', async () => { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cool-')) - const out = clearVmCooldown({ cfg: { paths: { project } }, id: 'vm-missing' }) + const out = await clearVmCooldown({ cfg: { paths: { project } }, id: 'vm-missing' }) assert.equal(out.status, 404) assert.equal(out.body.ok, false) }) -test('clearVmCooldown drops runtime leftover and usage 429 flag', () => { +test('clearVmCooldown drops runtime leftover and usage 429 flag', async () => { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cool-')) seedVm(project) const runtime = new Map() @@ -49,7 +49,7 @@ test('clearVmCooldown drops runtime leftover and usage 429 flag', () => { }) const unbound = [] let woken = 0 - const out = clearVmCooldown({ + const out = await clearVmCooldown({ cfg: { paths: { project } }, id: 'vm-11', accountQuota: { @@ -98,7 +98,7 @@ test('clearVmCooldown drops runtime leftover and usage 429 flag', () => { assert.equal(woken, 1) }) -test('clearVmCooldown drops the on-disk park and refreshes a rejected 5h window', () => { +test('clearVmCooldown drops the on-disk park and refreshes a rejected 5h window', async () => { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cool-')) const rec = seedVm(project) rec.claude.temp_unschedulable_until = Date.now() + 60_000 @@ -116,7 +116,7 @@ test('clearVmCooldown drops the on-disk park and refreshes a rejected 5h window' }, }, }) - const out = clearVmCooldown({ + const out = await clearVmCooldown({ cfg: { paths: { project } }, id: 'vm-11', accountQuota: { diff --git a/test/unit/shared-sticky-producers.test.mjs b/test/unit/shared-sticky-producers.test.mjs index b350d7dd0..841a83e33 100644 --- a/test/unit/shared-sticky-producers.test.mjs +++ b/test/unit/shared-sticky-producers.test.mjs @@ -72,7 +72,7 @@ async function fixture(t) { key, response, jobs, - run({ ensure, hop } = {}) { + run({ ensure, hop, routing = {} } = {}) { return handleCodexProtocol({ req, res: response, @@ -83,6 +83,7 @@ async function fixture(t) { stats: { errors: 0, requests: 0, by_route: {} }, projectRoot: root, stickyRouter: router, + routing, json(_res, status, value) { response.status = status response.body = value @@ -96,7 +97,10 @@ async function fixture(t) { ensureCodexKernel: ensure || (async () => ({ ok: true })), streamCodexKernel: async (args) => { jobs.push(args) - if (hop) await hop(args) + if (hop) { + const result = await hop(args) + if (result) return result + } return { ok: true, status: 200, terminalState: 'verified', body: { id: 'resp-synthetic' } } }, }, @@ -157,6 +161,44 @@ test('a late Codex success cannot restore a pin moved by another controller', as assert.equal(stored.session_id, null) }) +test('a shared native-session conflict stops an implicit Codex transport retry', async (t) => { + const fx = await fixture(t) + await fx.run({ + hop: async () => { + await fx.peer.refresh() + await fx.peer.rebindFamilyShared( + fx.key, + { accountId: 'vm-gpt-b', vmId: 'vm-gpt-b' }, + { ifGeneration: fx.peer.versionOf(fx.key) }, + ) + return { ok: false, status: 502, body: { error: { code: 'upstream_transport' } } } + }, + }) + assert.equal(fx.jobs.length, 1, 'each actual retry must confirm the captured binding generation again') + assert.equal(fx.response.status, 503) + assert.equal(fx.response.body.error.code, 'sticky_binding_conflict') + assert.equal((await fx.peer.persistence.get(fx.key)).vm_id, 'vm-gpt-b') +}) + +test('a late Codex failure cannot unbind a pin moved by another controller', async (t) => { + const fx = await fixture(t) + await fx.run({ + routing: { failover: { max_total_attempts: 1, max_account_switches: 0 } }, + hop: async () => { + await fx.peer.refresh() + await fx.peer.rebindFamilyShared( + fx.key, + { accountId: 'vm-gpt-b', vmId: 'vm-gpt-b' }, + { ifGeneration: fx.peer.versionOf(fx.key) }, + ) + return { ok: false, status: 401, body: { error: { code: 'oauth_revoked' } } } + }, + }) + assert.equal(fx.jobs.length, 1) + assert.equal(fx.response.status, 401) + assert.equal((await fx.peer.persistence.get(fx.key)).vm_id, 'vm-gpt-b') +}) + test('count_tokens awaits the shared read-only session helper with migrate disabled', async () => { const entered = deferred() const resume = deferred() @@ -214,7 +256,6 @@ test('panel cooldown clearing awaits shared sticky deletion before returning', a cfg: { paths: { project: fx.root } }, id: 'vm-gpt-a', stickyRouter: fx.router, - accountQuota: { repo: { get: () => ({ unified: {} }) } }, }), ).then((result) => { settled = true From 2e940209fd80c977902529840e88f66c10512565 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:51:34 +0800 Subject: [PATCH 131/473] test: require committed scheduler quota cleanup --- test/unit/runtime-producers.test.mjs | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/unit/runtime-producers.test.mjs b/test/unit/runtime-producers.test.mjs index 32cd8707c..487ac31e0 100644 --- a/test/unit/runtime-producers.test.mjs +++ b/test/unit/runtime-producers.test.mjs @@ -65,3 +65,43 @@ test('runtime observation tickets retain local invocation order while allocation first.resolve(1) assert.deepEqual(await Promise.all([a, b]), [1, 2]) }) + +test('shared scheduler waits for committed quota cleanup and preserves VM restriction when CAS loses', async (t) => { + const fs = await import('node:fs') + const os = await import('node:os') + const path = await import('node:path') + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-cleanup-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const file = path.join(root, 'vm-cleanup.json') + const vm = { id: 'vm-cleanup', claude: { temp_unschedulable_reason: 'quota_exhausted', temp_unschedulable_until: 1 } } + fs.writeFileSync(file, JSON.stringify(vm)) + const entered = deferred() + const resume = deferred() + const pool = new PoolScheduler({ + projectRoot: root, + runtimeRepo: { + get: () => ({ cooldown_reason: 'quota_exhausted', cooldown_until: 1 }), + upsert() {}, + async clearQuotaCooldownCommitted() { + entered.resolve() + await resume.promise + return false + }, + }, + }) + pool.shared = true + let settled = false + const result = Promise.resolve(pool.clearQuotaRestriction(vm, 'account-cleanup', file)).then((value) => { + settled = true + return value + }) + try { + await Promise.race([entered.promise, result.then(() => assert.fail('cleanup returned before PostgreSQL confirmation'))]) + assert.equal(settled, false) + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).claude.temp_unschedulable_reason, 'quota_exhausted') + } finally { + resume.resolve() + } + assert.equal(await result, false) + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).claude.temp_unschedulable_reason, 'quota_exhausted') +}) From e6e794754f9bfa86b5d3fb0c454f4a1bc2be87d0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:53:57 +0800 Subject: [PATCH 132/473] test(codex): reproduce unsafe shared retries and reused job IDs --- test/unit/handle-codex.test.mjs | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/unit/handle-codex.test.mjs b/test/unit/handle-codex.test.mjs index 42f0d7ceb..4bc84314c 100644 --- a/test/unit/handle-codex.test.mjs +++ b/test/unit/handle-codex.test.mjs @@ -71,6 +71,44 @@ test('runCodexKernelHop does not retry after SSE has started', async () => { assert.equal(result.transport_retried, undefined) }) +for (const execution of [ + undefined, + { requestId: 'pending-codex-job', started: true, ended: false, reason: 'transport_error' }, +]) { + test(`shared Codex ${execution ? 'unfinished' : 'missing'} execution metadata prevents a transport retry`, async () => { + let jobs = 0 + const result = await runCodexKernelHop({ + sharedExecution: true, + hop: async () => { + jobs++ + return { ok: false, status: 502, body: { error: { code: 'upstream_transport' } }, execution } + }, + }) + assert.equal(jobs, 1, 'an unconfirmed execution must not be submitted twice') + assert.equal(result.transport_retried, undefined) + }) +} + +test('shared Codex retries use distinct durable job IDs after confirmed EOF', async () => { + const ids = [] + await runCodexKernelHop({ + sharedExecution: true, + args: { requestId: 'fixed-first-codex-job' }, + hop: async ({ requestId }) => { + ids.push(requestId) + return { + ok: ids.length > 1, + status: ids.length > 1 ? 200 : 502, + body: { error: { code: 'upstream_transport' } }, + execution: { requestId, started: true, ended: true, reason: 'eof' }, + } + }, + }) + assert.equal(ids.length, 2) + assert.equal(ids[0], 'fixed-first-codex-job') + assert.notEqual(ids[1], ids[0]) +}) + function writeGptVm(root, id) { const dir = path.join(root, 'vms') fs.mkdirSync(path.join(dir, id), { recursive: true }) From a17f53e36286ac9756b3c2fb9bb9ff2dba02e85c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:55:08 +0800 Subject: [PATCH 133/473] test(codex): reproduce missing PostgreSQL execution admission --- test/unit/shared-codex-execution.test.mjs | 165 ++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 test/unit/shared-codex-execution.test.mjs diff --git a/test/unit/shared-codex-execution.test.mjs b/test/unit/shared-codex-execution.test.mjs new file mode 100644 index 000000000..3041b0ab0 --- /dev/null +++ b/test/unit/shared-codex-execution.test.mjs @@ -0,0 +1,165 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' +import { handleCodexProtocol } from '../../src/lib/protocol/handle-codex.mjs' +import { resetOpenAIAccountRuntime } from '../../src/lib/pool/openai-account-runtime.mjs' + +const unixTest = process.platform === 'win32' ? test.skip : test +const accountId = 'codex:synthetic-shared-account' +const vmId = 'vm-shared-codex' + +function success(res) { + const usage = { input_tokens: 1, output_tokens: 1 } + res.writeHead(200, { 'content-type': 'text/event-stream', 'x-usage': JSON.stringify(usage) }) + res.end( + `data: ${JSON.stringify({ type: 'response.completed', response: { id: 'resp-shared', status: 'completed', output: [], usage } })}\n\n`, + ) +} + +async function fixture(t, handle = (_job, _req, res) => success(res)) { + resetOpenAIAccountRuntime() + const env = ['VM2API_CONTROL_PLANE_MODE', 'KIN_CRS_MOCK'] + const previous = Object.fromEntries(env.map((key) => [key, process.env[key]])) + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + process.env.KIN_CRS_MOCK = '0' + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-codex-')) + const runDir = path.join(root, 'vms', vmId, 'run') + fs.mkdirSync(runDir, { recursive: true }) + fs.mkdirSync(path.join(root, 'vms', vmId, 'cli-home')) + const socket = path.join(runDir, 'codex-kernel.sock') + const token = path.join(runDir, 'internal.token') + fs.writeFileSync(token, 'synthetic-internal-token\n', { mode: 0o600 }) + fs.writeFileSync( + path.join(root, 'vms', `${vmId}.json`), + JSON.stringify({ + id: vmId, + platform: 'openai', + family: 'codex', + schedulable: true, + status: 'running', + has_token: true, + runtime: { codex_kernel_socket: socket, worker_run_dir: runDir, worker_token_file: token }, + policy: { maxConcurrency: 1, concurrencyOverride: true }, + }), + ) + fs.writeFileSync( + path.join(root, 'vms', vmId, 'codex-credentials.json'), + JSON.stringify({ + accounts: [ + { + id: 'synthetic-id', + access_token: 'synthetic-access', + refresh_token: 'synthetic-refresh', + chatgpt_account_id: 'synthetic-shared-account', + }, + ], + }), + ) + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const messages = [] + const bindingsAtSend = [] + const server = http.createServer((req, res) => { + void (async () => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + const job = JSON.parse(Buffer.concat(chunks).toString()) + if (req.url.endsWith('/cancel')) { + res.end('{"ok":true,"cancelled":true}') + return + } + messages.push(job) + bindingsAtSend.push(await other.prepare('SELECT * FROM execution_jobs WHERE request_id = $1').get(job.request_id)) + await handle(job, req, res) + })().catch((error) => res.destroy(error)) + }) + t.after(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + await other.close() + fs.rmSync(root, { recursive: true, force: true }) + for (const key of env) { + if (previous[key] == null) delete process.env[key] + else process.env[key] = previous[key] + } + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(socket, resolve) + }) + return { + root, + db, + other, + messages, + bindingsAtSend, + async run({ sharedRuntime, sessionId = null } = {}) { + const response = { headersSent: false, write() {}, end() {} } + const body = { model: 'gpt-5.4', input: 'synthetic request', stream: false } + await handleCodexProtocol({ + req: { headers: sessionId ? { 'x-session-id': sessionId } : {}, apiKeyKind: 'user' }, + res: response, + protocol: 'openai.responses', + ctx: { body }, + inbound: body, + projectRoot: root, + logBag: {}, + stats: { requests: 0, errors: 0, by_route: {} }, + routing: { + codex: { quota: { max_concurrency: 1 } }, + failover: { fallback_wait_timeout_ms: 1000, max_total_attempts: 2, max_same_account_retries: 0 }, + }, + json(_res, status, payload) { + response.status = status + response.body = payload + return payload + }, + writeSSEHeaders() { + response.headersSent = true + }, + ops: { sharedRuntime, writeCodexKernelConfig() {}, ensureCodexKernel: async () => ({ ok: true }) }, + }) + return response + }, + } +} + +unixTest('Kubernetes Codex persists its actual kernel job before submission and releases only after EOF', async (t) => { + const fx = await fixture(t) + const response = await fx.run() + assert.equal(response.status, 200) + assert.equal(fx.messages.length, 1) + assert.ok(fx.bindingsAtSend[0], 'the kernel job must already exist on the independent database connection') + const row = await fx.other + .prepare(`SELECT j.request_id, j.ended_at, a.account_id, a.released_at FROM execution_jobs j + JOIN execution_admissions a ON a.id = j.admission_id`) + .get() + assert.equal(row.account_id, accountId) + assert.equal(row.request_id, fx.messages[0].request_id) + assert.ok(row.ended_at && row.released_at >= row.ended_at) +}) + +unixTest('Codex cannot bypass account capacity held by another database connection', async (t) => { + const fx = await fixture(t) + const held = await new SharedExecutionAdmissions(fx.other).tryReserve({ + accountId, + vmId, + accountLimit: 1, + vmLimit: 1, + fableLimit: 0, + rpmLimit: 0, + }) + try { + const response = await fx.run() + assert.equal(fx.messages.length, 0, 'PostgreSQL is the final capacity authority') + assert.equal(response.status, 429) + assert.equal(response.body.error.code, 'pool_overloaded') + } finally { + await held.release() + } +}) From 674c5182fc0c1e1aec45e93b09119963fa91662d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:55:38 +0800 Subject: [PATCH 134/473] test: verify twelve shared-state migrations restore correctly --- test/unit/backup-service.test.mjs | 13 ++++++++++--- test/unit/database.test.mjs | 7 ++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 9bf4b55eb..a6ec474e6 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,9 +79,12 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; + ALTER TABLE sticky_sessions DROP COLUMN deleted_at; + ALTER TABLE sticky_sessions ALTER COLUMN generation DROP DEFAULT; + DROP SEQUENCE sticky_generation_seq; DROP SEQUENCE IF EXISTS account_observation_seq; DELETE FROM schema_migrations WHERE version <> '001'`, ) @@ -102,18 +105,22 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'execution_seats', 'execution_jobs', 'session_turns', + 'vm_operations', ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 10) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 12) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; + ALTER TABLE sticky_sessions DROP COLUMN deleted_at; + ALTER TABLE sticky_sessions ALTER COLUMN generation DROP DEFAULT; + DROP SEQUENCE sticky_generation_seq; DROP SEQUENCE IF EXISTS account_observation_seq; DELETE FROM schema_migrations WHERE version <> '001'`, ) diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index c758d9bdb..f0a06ccd7 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 37) + assert.equal(rows.length, 38) for (const name of [ 'settings', 'users', @@ -52,6 +52,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'execution_seats', 'execution_jobs', 'session_turns', + 'vm_operations', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -112,7 +113,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 10) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 12) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -128,7 +129,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 10) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 12) fs.rmSync(dir, { recursive: true, force: true }) }) From e2e77804f987248e3e558d921a09ea108a6edb20 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:58:31 +0800 Subject: [PATCH 135/473] fix: commit quota runtime observations before side effects --- src/lib/db/repos/account-runtime-repo.mjs | 71 +++++++++++++++++++--- src/lib/pool/account-quota.mjs | 65 ++++++++------------ src/lib/pool/rate-limit-service.mjs | 49 ++++++++------- src/lib/pool/scheduling-runtime.mjs | 46 +++++++++++++- test/unit/account-quota.test.mjs | 26 ++++++-- test/unit/rate-limit-service.test.mjs | 24 ++++---- test/unit/runtime-quota-producers.test.mjs | 37 +++++++++++ 7 files changed, 231 insertions(+), 87 deletions(-) diff --git a/src/lib/db/repos/account-runtime-repo.mjs b/src/lib/db/repos/account-runtime-repo.mjs index a95d2cd03..d3d0276c2 100644 --- a/src/lib/db/repos/account-runtime-repo.mjs +++ b/src/lib/db/repos/account-runtime-repo.mjs @@ -152,18 +152,42 @@ export class AccountRuntimeRepo { return account } - async patch(accountId, fields, { vmId, observation, modelPatch, clearModels = false, expected, condition } = {}) { + async patch( + accountId, + fields, + { + vmId, + observation, + modelPatch, + clearModels = false, + expected, + condition, + returnApplied = false, + atomic = false, + } = {}, + ) { const sequence = BigInt(await (observation ?? this.beginObservation())) return withTransaction(this.db, async () => { const account = await this.lockAccount(accountId, vmId) - if (!account) return null + if (!account) return returnApplied ? { applied: false, state: null } : null const runtime = await this.db.prepare('SELECT * FROM account_runtime WHERE account_id = $1').get(accountId) const current = mergeState(runtime, account) - if (!current.vm_id && !vmId && !fields.vm_id) return null - if (condition && !condition(current)) return null + const rejected = () => (returnApplied ? { applied: false, state: current } : null) + if (!current.vm_id && !vmId && !fields.vm_id) return rejected() + if (condition && !condition(current)) return rejected() if (expected && Object.entries(expected).some(([key, value]) => !isDeepStrictEqual(current[key], value))) - return null + return rejected() const versions = account.observation_versions || {} + if ( + atomic && + Object.entries(fields).some( + ([field, value]) => + value !== undefined && + (Object.hasOwn(ACCOUNT_FIELDS, field) || Object.hasOwn(RUNTIME_FIELDS, field)) && + !accepts(versions, ['runtime', field], sequence), + ) + ) + return rejected() const acc = {} const rt = {} let accepted = false @@ -197,7 +221,7 @@ export class AccountRuntimeRepo { rt.model_states_json = JSON.stringify(models) accepted = true } - if (!accepted) return current + if (!accepted) return returnApplied ? { applied: false, state: current } : current const now = new Date().toISOString() await this.db .prepare(`INSERT INTO account_runtime (account_id, vm_id, updated_at) @@ -216,7 +240,8 @@ export class AccountRuntimeRepo { const assignments = Object.keys(acc).map((key, i) => `${key} = $${i + 1}`) assignments.push(`observation_versions = $${values.length - 2}::jsonb`, `updated_at = $${values.length - 1}`) await this.db.prepare(`UPDATE accounts SET ${assignments.join(', ')} WHERE id = $${values.length}`).run(...values) - return this.get(accountId) + const stored = await this.get(accountId) + return returnApplied ? { applied: true, state: stored } : stored }) } @@ -245,6 +270,21 @@ export class AccountRuntimeRepo { return this.patch(accountId, fields, { vmId, observation }) } + async updateWindowCommitted(accountId, { vmId = null, observation, ...input } = {}) { + const fields = {} + for (const [key, field] of Object.entries({ + rateLimitedAt: 'rate_limited_at', + rateLimitResetAt: 'rate_limit_reset_at', + overloadUntil: 'overload_until', + sessionWindowStart: 'session_window_start', + sessionWindowEnd: 'session_window_end', + sessionWindowStatus: 'session_window_status', + })) { + if (input[key] !== undefined) fields[field] = input[key] + } + return this.patch(accountId, fields, { vmId, observation, returnApplied: true, atomic: true }) + } + async markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown', observation } = {}) { const cooldown = Number(until) || null if (model) @@ -316,11 +356,22 @@ export class AccountRuntimeRepo { } async clearQuotaCooldown(accountId, { vmId, observation, expected } = {}) { - return !!(await this.patch( + return (await this.clearQuotaCooldownCommitted(accountId, { vmId, observation, expected })).applied + } + + async clearQuotaCooldownCommitted(accountId, { vmId, observation, expected } = {}) { + return this.patch( accountId, { status: 'ready', cooldown_until: null, cooldown_reason: null, rate_limit_reset_at: null }, - { vmId, observation, expected, condition: (state) => this.isQuotaCooldown(state) }, - )) + { + vmId, + observation, + expected, + condition: (state) => this.isQuotaCooldown(state), + returnApplied: true, + atomic: true, + }, + ) } async clearExpired(now = Date.now()) { diff --git a/src/lib/pool/account-quota.mjs b/src/lib/pool/account-quota.mjs index 0a6d5c7b0..9e321305b 100644 --- a/src/lib/pool/account-quota.mjs +++ b/src/lib/pool/account-quota.mjs @@ -208,7 +208,7 @@ export class AccountQuota { this.runtimeRepo = runtimeRepo || null } - _writeSessionWindow(acc) { + _writeSessionWindow(acc, { observation } = {}) { if (!this.runtimeRepo?.updateWindow || !acc) return const extra = headerWindow(acc.unified, '5h') const w5 = extra.utilization != null || extra.status || extra.reset ? extra : officialWindow(acc.unified, '5h') @@ -217,6 +217,7 @@ export class AccountQuota { try { this.runtimeRepo.updateWindow(acc.account_id, { vmId: acc.vm_id || null, + observation, sessionWindowEnd: Number.isFinite(endMs) ? endMs : Number.isFinite(parseResetMs(endIso)) @@ -275,6 +276,7 @@ export class AccountQuota { * headers: Headers | plain object (lower-cased keys preferred) */ ingestHeaders(accountId, headers, usageBody = null, extra = {}) { + const observation = extra.observation ?? this.beginObservation() const acc = this.ensure({ account_id: accountId }) const h = normalizeHeaders(headers) acc.unified.headers = acc.unified.headers || {} @@ -382,8 +384,8 @@ export class AccountQuota { }) } - const saved = this.repo.observe(acc, { observation: extra.observation, paths }) - this._writeSessionWindow(saved) + const saved = this.repo.observe(acc, { observation, paths }) + this._writeSessionWindow(saved, { observation }) if (sampled) { const observed5 = headerWindowOrEmpty(saved.unified, '5h') const observed7 = headerWindowOrEmpty(saved.unified, '7d') @@ -424,6 +426,7 @@ export class AccountQuota { * Fable weekly limit is stored separately and does not unschedulable the account. */ ingestOAuthUsage(accountId, probe = {}, { observation } = {}) { + observation ??= this.beginObservation() const acc = this.ensure({ account_id: accountId }) const nowIso = probe.probed_at || new Date().toISOString() if (isOfficialUsageRateLimited(probe)) { @@ -568,7 +571,7 @@ export class AccountQuota { clearHeaderExhaustIfOfficialOpen(acc) } const saved = this.repo.observe(acc, { observation, paths }) - this._writeSessionWindow(saved) + this._writeSessionWindow(saved, { observation }) if (usageOk) { const observed5 = headerWindowOrEmpty(saved.unified, '5h') const observed7 = headerWindowOrEmpty(saved.unified, '7d') @@ -579,7 +582,7 @@ export class AccountQuota { }) } if (usageOk && this.headerUnderSafety(saved)) { - this.clearQuotaExhaustedCooldown(accountId) + this.clearQuotaExhaustedCooldown(accountId, { observation }) } return saved } @@ -995,7 +998,8 @@ export class AccountQuota { * Write effective 5h/7d windows back to SQLite when a reset has elapsed * or last_used proves a probe-source 100% is leftover from the previous window. */ - persistEffectiveWindows(accountId, { now = Date.now() } = {}) { + persistEffectiveWindows(accountId, { now = Date.now(), observation } = {}) { + observation ??= this.beginObservation() if (!accountId) return null const acc = this.repo.get(accountId) if (!acc?.unified) return acc || null @@ -1030,7 +1034,7 @@ export class AccountQuota { const healed = healOfficialOverwriteExtra(acc, accountId, this.repo) if (!healed) { if (backoffExpired) this.repo.save(acc) - if (this.headerUnderSafety(acc)) this.clearQuotaExhaustedCooldown(accountId) + if (this.headerUnderSafety(acc)) this.clearQuotaExhaustedCooldown(accountId, { observation }) return acc } } else { @@ -1047,8 +1051,8 @@ export class AccountQuota { } acc.unified.updated_at = new Date().toISOString() const saved = this.repo.save(acc) - this._writeSessionWindow(saved) - if (this.headerUnderSafety(saved)) this.clearQuotaExhaustedCooldown(accountId) + this._writeSessionWindow(saved, { observation }) + if (this.headerUnderSafety(saved)) this.clearQuotaExhaustedCooldown(accountId, { observation }) return saved } @@ -1071,36 +1075,19 @@ export class AccountQuota { return true } - clearQuotaExhaustedCooldown(accountId) { - if (!accountId || !this.runtimeRepo) return false - let state = null - try { - state = this.runtimeRepo.get?.(accountId) - } catch { - return false - } - if (!state) return false - // Live 429 / overload columns belong to RateLimitService; passive Extra never lifts them. - if (Number(state.rate_limit_reset_at) > Date.now() || Number(state.overload_until) > Date.now()) return false - const reason = String(state.cooldown_reason || '') - const quotaCool = !reason || /quota|account_quota_exhausted|rate_limited/i.test(reason) - if (!quotaCool) return false - if (!state.cooldown_until && !reason) return false - try { - this.runtimeRepo.upsert?.({ - ...state, - status: 'ready', - cooldown_until: null, - cooldown_reason: null, - rate_limit_reset_at: null, - }) - } catch { - return false - } - try { - this.onQuotaCooldownCleared?.(accountId) - } catch {} - return true + clearQuotaExhaustedCooldown(accountId, { observation } = {}) { + const clearing = (async () => { + if (!accountId || !this.runtimeRepo?.clearQuotaCooldownCommitted) return false + const cleared = await this.runtimeRepo.clearQuotaCooldownCommitted(accountId, { observation }) + if (!cleared) return false + try { + this.onQuotaCooldownCleared?.(accountId) + } catch {} + return true + })() + // Synchronous quota ingestion schedules this work; flush exposes queued DB failures. + clearing.catch(() => {}) + return clearing } _lastUsedAt(accountId) { diff --git a/src/lib/pool/rate-limit-service.mjs b/src/lib/pool/rate-limit-service.mjs index 1a19df66a..acddf6aa0 100755 --- a/src/lib/pool/rate-limit-service.mjs +++ b/src/lib/pool/rate-limit-service.mjs @@ -100,7 +100,7 @@ export class RateLimitService { * An unknown bare 429 writes no hard block: the scope is unproven, so one * bounded usage probe decides whether the account window is really spent. */ - handleUpstreamError({ accountId, vmId, result = {}, policy = null, now = Date.now() } = {}) { + async handleUpstreamError({ accountId, vmId, result = {}, policy = null, now = Date.now() } = {}) { if (!accountId || !this.runtimeRepo || result?.committed) return null const reason = String(policy?.reason || '') if (policy?.scope === 'account' && reason === 'account_quota_exhausted') { @@ -111,7 +111,7 @@ export class RateLimitService { return null } if (reason === 'provider_overloaded' && Number(result?.status) === 529) { - return this.handle529({ accountId, vmId, now }) + return this.handle529({ accountId, vmId, result, now }) } return null } @@ -123,46 +123,53 @@ export class RateLimitService { } catch {} } - handle429({ accountId, vmId, result = {}, now = Date.now() }) { + async handle429({ accountId, vmId, result = {}, now = Date.now() }) { + const observation = result.accountObservation ?? this.runtimeRepo.beginObservation() const headers = extraHeadersFromLimitError(bodyMessage(result.body), result.headers || {}, now) const parsed = anthropic429Reset(headers, now) const fallback = !parsed const resetAt = parsed?.resetAt || now + this.config.fallback_cooldown_min * 60_000 const windowEnd = parsed?.fiveHourReset || (parsed?.window === '7d' ? null : resetAt) - this.runtimeRepo.updateWindow?.(accountId, { + const committed = await this.runtimeRepo.updateWindowCommitted(accountId, { vmId, + observation, rateLimitedAt: now, rateLimitResetAt: resetAt, ...(windowEnd ? { sessionWindowStart: windowEnd - WINDOW_5H_MS, sessionWindowEnd: windowEnd } : {}), sessionWindowStatus: 'rejected', }) - try { - this.accountQuota?.ingestHeaders?.(accountId, headers, null, { - exhausted: true, - status: 429, - countRequest: false, - observation: result.accountObservation, - }) - } catch {} - if (fallback) this.requestUsageProbe({ accountId, vmId }) - return { kind: 'rate_limited', until: resetAt, window: parsed?.window || null, fallback } + this.accountQuota?.ingestHeaders?.(accountId, headers, null, { + exhausted: true, + status: 429, + countRequest: false, + observation, + }) + await this.accountQuota?.flush?.() + if (fallback && committed.applied) this.requestUsageProbe({ accountId, vmId }) + const until = Number(committed.state?.rate_limit_reset_at) + return until > now ? { kind: 'rate_limited', until, window: parsed?.window || null, fallback } : null } - handle529({ accountId, vmId, now = Date.now() }) { + async handle529({ accountId, vmId, result = {}, now = Date.now() }) { const until = now + this.config.overload_cooldown_min * 60_000 - this.runtimeRepo.updateWindow?.(accountId, { vmId, overloadUntil: until }) - return { kind: 'overloaded', until } + const committed = await this.runtimeRepo.updateWindowCommitted(accountId, { + vmId, + overloadUntil: until, + observation: result.accountObservation, + }) + const stored = Number(committed.state?.overload_until) + return stored > now ? { kind: 'overloaded', until: stored } : null } /** * sub2api UpdateSessionWindow: every response with live headers. * An out-of-range reset is ignored. `allowed` is the only early unblock. */ - updateSessionWindow({ accountId, vmId = null, headers = {}, now = Date.now() } = {}) { + async updateSessionWindow({ accountId, vmId = null, headers = {}, now = Date.now(), observation } = {}) { if (!accountId || !this.runtimeRepo) return false const status = String(header(headers, 'anthropic-ratelimit-unified-5h-status') || '').toLowerCase() if (!status) return false - const patch = { vmId, sessionWindowStatus: status } + const patch = { vmId, sessionWindowStatus: status, observation } const resetMs = parseResetMs(header(headers, 'anthropic-ratelimit-unified-5h-reset')) if ( Number.isFinite(resetMs) && @@ -178,8 +185,8 @@ export class RateLimitService { if (status === 'allowed' && sevenDayOpen && limited) { patch.rateLimitResetAt = null } - this.runtimeRepo.updateWindow?.(accountId, patch) - return patch.rateLimitResetAt === null + const committed = await this.runtimeRepo.updateWindowCommitted(accountId, patch) + return committed.applied && patch.rateLimitResetAt === null && committed.state?.rate_limit_reset_at == null } } diff --git a/src/lib/pool/scheduling-runtime.mjs b/src/lib/pool/scheduling-runtime.mjs index 878ffd801..40c7cd814 100644 --- a/src/lib/pool/scheduling-runtime.mjs +++ b/src/lib/pool/scheduling-runtime.mjs @@ -44,6 +44,7 @@ export class SchedulingRuntime { this.revisions = new Map() this.refreshStarted = 0 this.refreshPublished = 0 + this.observationTail = Promise.resolve() this.writes = this.persistence.db.schedulingWriteQueue ||= new WriteQueue(this.persistence.db) } @@ -55,7 +56,9 @@ export class SchedulingRuntime { return this.writes.flush() } beginObservation() { - return this.persistence.beginObservation() + const observation = this.observationTail.then(() => this.persistence.beginObservation()) + this.observationTail = observation.catch(() => {}) + return observation } get(id) { return structuredClone(this.rows.get(id) || null) @@ -149,6 +152,29 @@ export class SchedulingRuntime { ) } + commit(accountId, write) { + const revision = (this.revisions.get(accountId) || 0) + 1 + this.revisions.set(accountId, revision) + let result + const queued = this.writes.enqueue(async () => { + result = await write() + if (result.state && this.revisions.get(accountId) === revision) this.rows.set(accountId, result.state) + }) + const committed = queued.then(() => { + if (this.writes.failure) throw this.writes.failure + return result + }) + committed.catch(() => {}) + return committed + } + + updateWindowCommitted(accountId, { observation, ...input } = {}) { + const ticket = this.ticket(observation) + return this.commit(accountId, () => + this.persistence.updateWindowCommitted(accountId, { ...input, observation: ticket }), + ) + } + markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown', observation } = {}) { const current = this.get(accountId) || { account_id: accountId, vm_id: vmId } current.vm_id ||= vmId @@ -201,6 +227,24 @@ export class SchedulingRuntime { return this.clearCooldown('clearQuotaCooldown', accountId, options) } + clearQuotaCooldownCommitted(accountId, { vmId = null, observation, expected } = {}) { + const current = this.get(accountId) + if (!current || !this.persistence.isQuotaCooldown(current)) return Promise.resolve(false) + const ticket = this.ticket(observation) + return this.commit(accountId, () => + this.persistence.clearQuotaCooldownCommitted(accountId, { + vmId, + observation: ticket, + expected: expected || { + cooldown_until: current.cooldown_until, + cooldown_reason: current.cooldown_reason, + rate_limit_reset_at: current.rate_limit_reset_at, + overload_until: current.overload_until, + }, + }), + ).then((result) => result.applied) + } + clearCooldown(method, accountId, { vmId = null, observation } = {}) { const current = this.get(accountId) if (!current) return false diff --git a/test/unit/account-quota.test.mjs b/test/unit/account-quota.test.mjs index c6c5b6e98..0b35d9e50 100644 --- a/test/unit/account-quota.test.mjs +++ b/test/unit/account-quota.test.mjs @@ -1020,8 +1020,17 @@ test('persistEffectiveWindows clears leftover quota cooldown when official is un const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, - upsert: (row) => { - runtime.set(row.account_id, { ...runtime.get(row.account_id), ...row }) + clearQuotaCooldownCommitted: async (id) => { + const row = runtime.get(id) + if (!row) return false + runtime.set(id, { + ...row, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + rate_limit_reset_at: null, + }) + return true }, updateWindow: () => {}, }) @@ -1057,8 +1066,17 @@ test('official 0.44 allowed overwrites local 0.9 and clears quota cooldown', asy const runtime = new Map() q.attachRuntimeRepo({ get: (id) => runtime.get(id) || null, - upsert: (row) => { - runtime.set(row.account_id, { ...runtime.get(row.account_id), ...row }) + clearQuotaCooldownCommitted: async (id) => { + const row = runtime.get(id) + if (!row) return false + runtime.set(id, { + ...row, + status: 'ready', + cooldown_until: null, + cooldown_reason: null, + rate_limit_reset_at: null, + }) + return true }, updateWindow: () => {}, }) diff --git a/test/unit/rate-limit-service.test.mjs b/test/unit/rate-limit-service.test.mjs index e0539e891..ac020cd00 100755 --- a/test/unit/rate-limit-service.test.mjs +++ b/test/unit/rate-limit-service.test.mjs @@ -45,7 +45,7 @@ test('plan limit writes rate_limit_reset_at from the text and survives a passive const { service, runtimeRepo, accountQuota, probes, close } = await setup() try { const now = Date.now() - const block = service.handleUpstreamError({ + const block = await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: limitResult("provider error: You've hit your limit · resets 11am (America/New_York)"), @@ -61,7 +61,7 @@ test('plan limit writes rate_limit_reset_at from the text and survives a passive assert.deepEqual(hardBlockOf(state, now), { reason: 'rate_limited', until: state.rate_limit_reset_at }) assert.equal(probes.length, 0) // Passive Extra saying "under safety" must not lift the 429 block. - accountQuota.clearQuotaExhaustedCooldown('acc-1') + await accountQuota.clearQuotaExhaustedCooldown('acc-1') assert.equal(runtimeRepo.get('acc-1').rate_limit_reset_at, state.rate_limit_reset_at) } finally { await close() @@ -72,7 +72,7 @@ test('plan limit without any reset falls back to the configured cooldown and ask const { service, runtimeRepo, probes, close } = await setup({ fallback_cooldown_min: 30 }) try { const now = Date.now() - const block = service.handleUpstreamError({ + const block = await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: limitResult("You've hit your limit"), @@ -91,7 +91,7 @@ test('plan limit without any reset falls back to the configured cooldown and ask test('model-scoped 429 does not write an account hard block', async () => { const { service, runtimeRepo, close } = await setup() try { - const block = service.handleUpstreamError({ + const block = await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: limitResult('rate limited'), @@ -107,7 +107,7 @@ test('model-scoped 429 does not write an account hard block', async () => { test('#163: an unknown bare 429 asks for one usage probe and fabricates no 5h block', async () => { const { service, runtimeRepo, probes, close } = await setup() try { - const block = service.handleUpstreamError({ + const block = await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: limitResult('Rate limited'), @@ -125,7 +125,7 @@ test('529 writes overload_until for the configured minutes', async () => { const { service, runtimeRepo, close } = await setup({ overload_cooldown_min: 10 }) try { const now = Date.now() - const block = service.handleUpstreamError({ + const block = await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: { status: 529, body: { type: 'error', error: { type: 'overloaded_error' } } }, @@ -143,7 +143,7 @@ test('only a live allowed header lifts a rate limit early', async () => { const { service, runtimeRepo, close } = await setup() try { const now = Date.now() - service.handleUpstreamError({ + await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: limitResult("You've hit your limit · resets 11pm (UTC)"), @@ -151,13 +151,13 @@ test('only a live allowed header lifts a rate limit early', async () => { now, }) const reset = String(Math.floor((now + 4 * 3600_000) / 1000)) - service.updateSessionWindow({ + await service.updateSessionWindow({ accountId: 'acc-1', headers: { 'anthropic-ratelimit-unified-5h-status': 'allowed_warning' }, now, }) assert.ok(hardBlockOf(runtimeRepo.get('acc-1'), now)) - const lifted = service.updateSessionWindow({ + const lifted = await service.updateSessionWindow({ accountId: 'acc-1', headers: { 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -178,7 +178,7 @@ test('an out-of-range 5h reset header is ignored for the session window', async const { service, runtimeRepo, close } = await setup() try { const now = Date.now() - service.updateSessionWindow({ + await service.updateSessionWindow({ accountId: 'acc-1', headers: { 'anthropic-ratelimit-unified-5h-status': 'allowed', @@ -209,7 +209,7 @@ test('clearExpired drops elapsed rate limit and overload columns', async () => { test('an incomplete hop does not write an empty_response cooldown', async () => { const { service, runtimeRepo, close } = await setup({ empty_response_cooldown_sec: 60 }) try { - const block = service.handleUpstreamError({ + const block = await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: { status: 200, committed: false, terminalState: 'incomplete' }, @@ -246,7 +246,7 @@ test('429 headers retain the ticket captured before the attempt began', async () options = value } try { - service.handleUpstreamError({ + await service.handleUpstreamError({ accountId: 'acc-1', vmId: 'vm-01', result: { ...limitResult('rate limited'), accountObservation: 41 }, diff --git a/test/unit/runtime-quota-producers.test.mjs b/test/unit/runtime-quota-producers.test.mjs index 30cefc5de..65906ab6f 100644 --- a/test/unit/runtime-quota-producers.test.mjs +++ b/test/unit/runtime-quota-producers.test.mjs @@ -160,3 +160,40 @@ test('quota-cleared callbacks wait until the queued PG clearing has committed', assert.equal(callbacks, 1) assert.equal((await left.persistence.get('producer-account')).cooldown_until, null) }) + +test('an older clearing reports false when newer identical cooldown fields have claimed their observation versions', async () => { + const { left, right } = await fixture() + const until = Date.now() + 60_000 + const fields = { account_id: 'producer-account', cooldown_until: until, cooldown_reason: 'quota_guard' } + left.upsert(fields) + await left.flush() + const older = await left.beginObservation() + const newer = await right.beginObservation() + right.upsert(fields, { observation: newer }) + await right.flush() + assert.equal(await left.clearQuotaCooldownCommitted('producer-account', { observation: older }), false) + assert.equal((await left.persistence.get('producer-account')).cooldown_until, until) +}) + +test('a failed PG rate-limit write rejects without publishing cache or firing the usage callback', async () => { + const { left, service } = await fixture() + let callbacks = 0 + service.onUsageProbe = () => { + callbacks++ + } + await left.persistence.db.exec(`CREATE FUNCTION reject_limit_write() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic runtime write failure' USING ERRCODE = 'P0001'; END $$; + CREATE TRIGGER reject_limit_write BEFORE UPDATE ON accounts FOR EACH ROW EXECUTE FUNCTION reject_limit_write()`) + try { + await assert.rejects(service.handle429({ accountId: 'producer-account', vmId: 'producer-vm', result: {} }), { + code: 'P0001', + }) + assert.equal(callbacks, 0) + assert.equal(left.get('producer-account').rate_limit_reset_at, null) + assert.equal((await left.persistence.get('producer-account')).rate_limit_reset_at, null) + } finally { + await left.persistence.db.exec('DROP TRIGGER reject_limit_write ON accounts') + // This isolated test deliberately faulted the queue; allow its fixture to drain. + left.writes.failure = null + } +}) From b8d68cdc5b28f5afaae2d5fbca3a7e67d1343e9e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:58:46 +0800 Subject: [PATCH 136/473] fix(k8s): use committed sticky and runtime observations in scheduling --- src/lib/pool/failover-runner.mjs | 149 +++++++++++++---- src/lib/pool/pool-scheduler.mjs | 207 +++++++++++++++++------- src/lib/protocol/handle-protocol.mjs | 27 +++- src/server.mjs | 2 + test/unit/runtime-producers.test.mjs | 16 +- test/unit/shared-sticky-runner.test.mjs | 62 ++++++- 6 files changed, 354 insertions(+), 109 deletions(-) diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 713185734..49e9cd1bd 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -413,19 +413,20 @@ function selectionFailure(selected, { excluded, lastPolicy, lastResult, hops }) return failed } -function applyCooldown(scheduler, selected, policy, model, { diagnosticPin = false } = {}) { +async function applyCooldown(scheduler, selected, policy, model, { diagnosticPin = false, observation } = {}) { if (policy?.action !== 'continue-and-cooldown' && policy?.action !== 'disable' && policy?.action !== 'pause') return // VM / master pin is a diagnostic. A 401 from the wrong inbound class // must not forever-park a Setup Token that has no refresh by design. if (diagnosticPin && (policy.reason === 'oauth_no_refresh' || policy.reason === 'oauth_revoked')) { return } - scheduler.markCooldown(selected, { + await scheduler.markCooldown(selected, { until: policy.action === 'disable' || policy.reason === 'oauth_no_refresh' || policy.reason === 'oauth_revoked' ? Number.MAX_SAFE_INTEGER : policy.cooldownUntil, reason: policy.reason, + observation, model: policy.scope === 'model' ? policy.model || model : null, status: policy.action === 'disable' || policy.reason === 'oauth_no_refresh' || policy.reason === 'oauth_revoked' @@ -436,7 +437,7 @@ function applyCooldown(scheduler, selected, policy, model, { diagnosticPin = fal // conversation pin: other seats absorb this turn. Quota and auth move // every conversation off the unit, windows included (#163). if (policy.scope === 'account' && policy.action !== 'pause' && !SOFT_COOLDOWN_REASONS.has(policy.reason)) { - scheduler.releaseAccountSessions?.({ accountId: selected?.accountId, vmId: selected?.vmId }) + await scheduler.releaseAccountSessions?.({ accountId: selected?.accountId, vmId: selected?.vmId }) } } @@ -485,16 +486,23 @@ export class FailoverRunner { * Client cancellation only ends its job; kernel cancel_ack fences slot reuse. * Planner seat indexes are not kernel native slots. */ - retireOutboundSession(selected, bindKeys) { + async retireOutboundSession(selected, bindKeys, versions = null) { if (!selected?.vmId) return for (const key of bindKeys || []) { const bound = this.stickyRouter?.resolve?.(key) if (bound?.accountId !== selected.accountId) continue - this.stickyRouter?.bind?.( + const changed = await (this.stickyRouter.shared ? this.stickyRouter.bindShared : this.stickyRouter.bind)?.call( + this.stickyRouter, key, { accountId: selected.accountId, vmId: selected.vmId }, - { countHit: false, replaceSession: true, clearSlot: true }, + { + countHit: false, + replaceSession: true, + clearSlot: true, + ...(this.stickyRouter.shared ? { ifGeneration: versions?.get(key) ?? this.stickyRouter.versionOf(key) } : {}), + }, ) + if (this.stickyRouter.shared && changed.applied) versions?.set(key, changed.generation) } } @@ -527,15 +535,15 @@ export class FailoverRunner { } } - forgetCredential(selected, policy, { familyKey = null, sessionKeys = [] } = {}) { + async forgetCredential(selected, policy, { familyKey = null, sessionKeys = [], versions = null } = {}) { if (familyKey) { - this.scheduler.releaseSticky?.(selected, null, sessionKeys) + await this.scheduler.releaseSticky?.(selected, null, sessionKeys, versions) } else { - this.scheduler.releaseAccountSessions?.({ accountId: selected?.accountId, vmId: selected?.vmId }) + await this.scheduler.releaseAccountSessions?.({ accountId: selected?.accountId, vmId: selected?.vmId }) } if (typeof this.onCredentialFailure === 'function') { try { - this.onCredentialFailure({ selected, policy }) + await this.onCredentialFailure({ selected, policy }) } catch {} } } @@ -638,6 +646,7 @@ export class FailoverRunner { } = {}) { if (!this.scheduler) throw new Error('FailoverRunner requires a scheduler') if (typeof callAttempt !== 'function') throw new Error('FailoverRunner requires callAttempt') + if (this.stickyRouter?.shared) await this.stickyRouter.refresh() const startedAt = Date.now() const budget = new AttemptCoordinator({ maxSameUnitRetries: Number(this.config.max_same_account_retries ?? 1), @@ -651,15 +660,21 @@ export class FailoverRunner { // Accounts left only because the kernel had no free slot; their pins stay. const spilled = budget.spilled const bindKeys = uniqueStickyKeys(stickyKey, stickyKeys) - const pinKeys = familyKey ? [...bindKeys, familyKey] : bindKeys + const pinKeys = [...bindKeys, familyKey, deviceKey].filter(Boolean) // Pins as this request last saw them. A pin moved by someone else since // (quota migration, credential death) is never written back by a late hop. const pinBaseline = new Map() + const pinVersions = new Map() const snapshotPins = () => { - for (const key of pinKeys) pinBaseline.set(key, this.stickyRouter?.resolve?.(key) || null) + for (const key of pinKeys) { + if (this.stickyRouter?.shared && pinVersions.has(key)) continue + pinBaseline.set(key, this.stickyRouter?.resolve?.(key) || null) + pinVersions.set(key, this.stickyRouter?.versionOf?.(key) || 0) + } } const pinMoved = (key) => { const base = pinBaseline.get(key) + if (this.stickyRouter?.shared) return this.stickyRouter.versionOf(key) !== pinVersions.get(key) if (!base) return false const now = this.stickyRouter?.resolve?.(key) return !now || now.accountId !== base.accountId || (now.generation || 0) !== (base.generation || 0) @@ -677,8 +692,9 @@ export class FailoverRunner { vmId: lastSelected?.vmId, attemptCount: budget.hops, }) - const bindAll = (account, opts) => { - if (!this.stickyRouter || !account) return + const bindAll = async (account, opts, { required = false } = {}) => { + if (!this.stickyRouter || !account) return true + let accepted = true const sessionId = account.sessionId || (account.accountId === outboundSessionAccountId ? outboundSessionId : '') const payload = { accountId: account.accountId, vmId: account.vmId } const slotIndex = account.slotIndex != null ? account.slotIndex : pinnedSlot @@ -687,21 +703,49 @@ export class FailoverRunner { if (stickyDeviceId) payload.deviceId = stickyDeviceId if (this.stickyRouter.bind) { for (const key of bindKeys) { - if (pinMoved(key)) continue + if (pinMoved(key)) { + accepted = false + continue + } const prev = this.stickyRouter.resolve?.(key) // A live pin on another account means this request only borrowed // capacity. Rewriting it would move the whole session off its home. if (prev?.accountId && prev.accountId !== account.accountId) continue let guard = opts - if (prev) guard = { ...opts, ifGeneration: prev.generation || 0 } + if (this.stickyRouter.shared) guard = { ...opts, ifGeneration: pinVersions.get(key) } + else if (prev) guard = { ...opts, ifGeneration: prev.generation || 0 } if (freshSlot || opts?.replaceSession) guard = { ...guard, replaceSession: true } - this.stickyRouter.bind(key, payload, guard) + const bound = await (this.stickyRouter.shared ? this.stickyRouter.bindShared : this.stickyRouter.bind).call( + this.stickyRouter, + key, + payload, + guard, + ) + if ( + this.stickyRouter.shared && + (!bound.applied || (required && sessionId && bound.nativeSessionId !== sessionId)) + ) { + accepted = false + continue + } pinBaseline.set(key, this.stickyRouter.resolve?.(key) || null) + pinVersions.set(key, this.stickyRouter.versionOf?.(key) || 0) } } if (familyKey && account.vmId && !pinMoved(familyKey)) { - this.stickyRouter.bind?.(familyKey, { accountId: account.accountId, vmId: account.vmId }, { countHit: false }) - pinBaseline.set(familyKey, this.stickyRouter.resolve?.(familyKey) || null) + const result = await (this.stickyRouter.shared + ? this.stickyRouter.rebindFamilyShared + : this.stickyRouter.bind + )?.call( + this.stickyRouter, + familyKey, + { accountId: account.accountId, vmId: account.vmId }, + { countHit: false, ifGeneration: pinVersions.get(familyKey) }, + ) + if (!this.stickyRouter.shared || result.applied) { + pinBaseline.set(familyKey, this.stickyRouter.resolve?.(familyKey) || null) + pinVersions.set(familyKey, this.stickyRouter.versionOf?.(familyKey) || 0) + } else accepted = false } if ( deviceKey && @@ -710,12 +754,22 @@ export class FailoverRunner { ) { const devicePayload = { accountId: account.accountId, vmId: account.vmId } if (this.stickyRouter.bindDeviceAffinity) { - this.stickyRouter.bindDeviceAffinity(deviceKey, devicePayload, { countHit: false }) + if (!this.stickyRouter.shared) + this.stickyRouter.bindDeviceAffinity(deviceKey, devicePayload, { countHit: false }) + else if (!pinMoved(deviceKey)) { + const bound = await this.stickyRouter.bindDeviceAffinityShared(deviceKey, devicePayload, { + countHit: false, + ifGeneration: pinVersions.get(deviceKey), + }) + if (bound.applied) pinVersions.set(deviceKey, bound.generation) + } } else { this.stickyRouter.bind?.(deviceKey, devicePayload, { countHit: false }) } } + return accepted } + const stickyConflict = () => poolError('sticky_binding_conflict', 'Conversation binding changed before execution') let lastResult = null let lastPolicy = null let repaired = false @@ -806,7 +860,11 @@ export class FailoverRunner { const home = this.stickyRouter?.resolve?.(familyKey) const ownExclusion = !!home && (budget.tried.has(home.accountId) || budget.tried.has(home.vmId) || spilled.has(home.accountId)) - if (!ownExclusion) this.stickyRouter?.unbind?.(familyKey) + if (!ownExclusion) { + if (this.stickyRouter?.shared) + await this.stickyRouter.unbindShared(familyKey, { ifGeneration: this.stickyRouter.versionOf(familyKey) }) + else this.stickyRouter?.unbind?.(familyKey) + } familyVmId = null } if (!selected?.ok && selected?.reason === 'fable_requires_max') { @@ -830,7 +888,14 @@ export class FailoverRunner { lastSelected = selected pinnedSlot = selected.slotIndex ?? null if (selected.seatMoved) seatMovedTo = selected.vmId - bindAll({ accountId: selected.accountId, vmId: selected.vmId, slotIndex: pinnedSlot }, { countHit: false }) + if ( + !(await bindAll( + { accountId: selected.accountId, vmId: selected.vmId, slotIndex: pinnedSlot }, + { countHit: false }, + { required: true }, + )) + ) + return stickyConflict() const attemptStarted = Date.now() await this.attemptsRepo?.begin?.({ requestId, @@ -880,14 +945,16 @@ export class FailoverRunner { if (attemptMeta?.sessionId) { outboundSessionId = String(attemptMeta.sessionId) outboundSessionAccountId = selected.accountId - bindAll( + const bound = await bindAll( { accountId: selected.accountId, vmId: selected.vmId, sessionId: outboundSessionId }, { countHit: false, // Prebinding moved the VM already; replace the previous VM's session. replaceSession: !!attemptStickyBound?.vmId && attemptStickyBound.vmId !== selected.vmId, }, + { required: true }, ) + if (!bound) return stickyConflict() } callEntered = true result = await callAttempt({ @@ -940,12 +1007,12 @@ export class FailoverRunner { // One writer for rate_limit_reset_at / overload_until (sub2api HandleUpstreamError). const hardBlock = !result?.committed && !pinVmId - ? this.rateLimitService?.handleUpstreamError?.({ + ? (await this.rateLimitService?.handleUpstreamError?.({ accountId: selected.accountId, vmId: selected.vmId, result, policy, - }) || null + })) || null : null if (hardBlock?.until && policy.action === 'continue-and-cooldown' && policy.scope === 'account') { policy = { ...policy, cooldownUntil: hardBlock.until } @@ -978,8 +1045,12 @@ export class FailoverRunner { if (unknownExecution) return incompleteHopResult(result, policy, { ...attribution(), executionUncertain: true }) if (policy.reason === 'content_filter_refusal' || policy.reason === 'usage_policy_refusal') { - this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, countUsage }) - bindAll({ accountId: selected.accountId, vmId: selected.vmId }) + await this.scheduler.markSuccess(selected, { + workerStatus: result.workerStatus || null, + countUsage, + observation: result.accountObservation, + }) + await bindAll({ accountId: selected.accountId, vmId: selected.vmId }) return { ...result, ...attribution(), @@ -998,13 +1069,17 @@ export class FailoverRunner { expiresAt: Date.now() + SEARCH_REPAIR_IDLE_MS, }) } - this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, countUsage }) + await this.scheduler.markSuccess(selected, { + workerStatus: result.workerStatus || null, + countUsage, + observation: result.accountObservation, + }) if (isFableModel(model) && typeof this.onFableSuccess === 'function') { try { this.onFableSuccess({ selected, model }) } catch {} } - bindAll({ accountId: selected.accountId, vmId: selected.vmId }) + await bindAll({ accountId: selected.accountId, vmId: selected.vmId }) return { ...result, ...attribution(), finalState: 'verified' } } if (policy.action === 'repair-and-retry' && !repaired && !result?.committed) { @@ -1016,9 +1091,12 @@ export class FailoverRunner { requestBody = repairAnthropicRequest(requestBody, policy) continue } - applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId }) + await applyCooldown(this.scheduler, selected, policy, model, { + diagnosticPin: !!pinVmId, + observation: result?.accountObservation, + }) if (!pinVmId && isCredentialDeath(policy)) { - this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys }) + await this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys, versions: pinVersions }) if (familyKey) { return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } } @@ -1030,7 +1108,7 @@ export class FailoverRunner { const hopMs = Date.now() - attemptStarted if (isRetryableEmptyHop(policy)) { if (!emptyHopReleased(result) || pinVmId) return incompleteHopResult(result, policy, attribution()) - this.retireOutboundSession(selected, bindKeys) + await this.retireOutboundSession(selected, bindKeys, pinVersions) freshSlot = true } if (budget.allowSameUnit(selected.accountId, policy, hopMs, this.config.same_account_retry_max_hop_ms)) { @@ -1148,9 +1226,12 @@ export class FailoverRunner { if (unknownExecution || committed || !shouldContinue(policy)) { return { ...result, ...attribution(), finalState: result.terminalState, policy } } - applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId }) + await applyCooldown(this.scheduler, selected, policy, model, { + diagnosticPin: !!pinVmId, + observation: result?.accountObservation, + }) if (!pinVmId && isCredentialDeath(policy)) { - this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys }) + await this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys, versions: pinVersions }) if (familyKey) { return { ...result, ...attribution(), finalState: result.terminalState, policy } } diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 8c460c95c..419c5514a 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -454,8 +454,18 @@ export class PoolScheduler extends EventEmitter { const preferred = avoid?.size ? available.filter((candidate) => !avoid.has(candidate.accountId) && !avoid.has(candidate.vmId)) : available + const stickySnapshot = this.stickyRouter?.shared + ? await this.reconcileStickyShared({ + stickyKey, + stickyKeys, + eligible: candidates, + spilled, + familyVmId: familyGated ? null : familyVm, + }) + : null let selected = this.pick(preferred, { stickyKey, + stickySnapshot, eligible: candidates, spilled, stickyKeys, @@ -621,13 +631,16 @@ export class PoolScheduler extends EventEmitter { // A grant can land while eligibility was being re-read. if (ticket?.granted) return finish(ticket.granted) if (familyVm && !candidates.some((candidate) => candidate.vmId === familyVm)) familyGated = true - const sticky = this.reconcileSticky({ - stickyKey, - stickyKeys, - eligible: candidates, - spilled, - familyVmId: familyGated ? null : familyVm, - }) + const sticky = await (this.stickyRouter?.shared ? this.reconcileStickyShared : this.reconcileSticky).call( + this, + { + stickyKey, + stickyKeys, + eligible: candidates, + spilled, + familyVmId: familyGated ? null : familyVm, + }, + ) if (sticky.cleared) stickyCleared = true const byVm = new Map(candidates.map((candidate) => [candidate.vmId, candidate])) if (ticket) ticket.candidates = byVm @@ -954,6 +967,10 @@ export class PoolScheduler extends EventEmitter { * `spilled` homes keep their pin. Returns the VM the pin still points at. */ reconcileSticky({ stickyKey = null, stickyKeys = null, eligible = [], spilled = null, familyVmId = null } = {}) { + if (stickyKey && this.stickyRouter?.shared) + throw Object.assign(new Error('Shared sticky reconciliation requires the asynchronous API'), { + code: 'shared_sticky_async_required', + }) const bound = stickyKey ? this.stickyRouter?.resolve?.(stickyKey) : null if (!bound) return { cleared: false, vmId: null, match: null } const familyVm = familyVmId ? String(familyVmId).trim() : '' @@ -975,6 +992,34 @@ export class PoolScheduler extends EventEmitter { return { cleared: false, vmId: bound.vmId, match } } + async reconcileStickyShared({ + stickyKey = null, + stickyKeys = null, + eligible = [], + spilled = null, + familyVmId = null, + } = {}) { + if (!stickyKey || !this.stickyRouter) return { cleared: false, vmId: null, match: null } + await this.stickyRouter.refresh() + const bound = this.stickyRouter.resolve(stickyKey) + if (!bound) return { cleared: false, vmId: null, match: null } + const matchFor = (pin) => + eligible.find((candidate) => candidate.vmId === pin?.vmId && candidate.accountId === pin?.accountId) || null + const match = matchFor(bound) + const familyMoved = + familyVmId && bound.vmId !== familyVmId && eligible.some((candidate) => candidate.vmId === familyVmId) + const clear = + familyMoved || + (!match && !spilled?.has(bound.accountId)) || + (match?.busy && !stickyShouldWait(match.waitReason, match.cooldownReason)) + if (!clear) return { cleared: !match, vmId: match ? bound.vmId : null, match } + const results = await this.releaseSticky(bound, stickyKey, stickyKeys) + const primary = results.find((result) => result.key === stickyKey) + if (primary?.applied) return { cleared: true, vmId: null, match: null } + const current = this.stickyRouter.resolve(stickyKey) + return { cleared: false, vmId: current?.vmId || null, match: matchFor(current) } + } + async eligibleCandidates({ model, excluded = new Set(), @@ -1107,7 +1152,7 @@ export class PoolScheduler extends EventEmitter { const modelGate = slotAllowsModel({ vm, account, model }) if (!modelGate.ok) return modelGate if (account) { - this.syncQuotaSchedule(vm, account) + await this.syncQuotaSchedule(vm, account) if (this.projectRoot) { const live = getVm(this.projectRoot, vm.id) if (live) { @@ -1305,6 +1350,7 @@ export class PoolScheduler extends EventEmitter { if (cached && now - cached.at < this.config.worker_health_ttl_ms) { return cached.value } + const observation = await this.runtimeRepo?.beginObservation?.() let value try { value = await this.workerHealth(exec, { signal }) @@ -1317,15 +1363,18 @@ export class PoolScheduler extends EventEmitter { const prevGen = Number(prev?.credential_generation) || 0 const nextGen = Number(value?.credential?.generation) || 0 const effectiveGen = Math.max(prevGen, nextGen) - this.runtimeRepo.upsert({ - account_id: exec.accountId, - vm_id: exec.vmId, - status: runtimeHealthStatus(value), - worker_heartbeat_at: now, - worker_status: value, - credential_generation: effectiveGen, - refresh_status: value?.credential?.credential_state || (value?.credential?.needs_refresh ? 'needed' : 'fresh'), - }) + this.runtimeRepo.upsert( + { + account_id: exec.accountId, + status: runtimeHealthStatus(value), + worker_heartbeat_at: now, + worker_status: value, + credential_generation: effectiveGen, + refresh_status: + value?.credential?.credential_state || (value?.credential?.needs_refresh ? 'needed' : 'fresh'), + }, + { vmId: exec.vmId, observation }, + ) const liveMs = expiresAtToMs(value?.credential?.expires_at) const prevMs = expiresAtToMs(prev?.worker_status?.credential?.expires_at) if ( @@ -1399,8 +1448,22 @@ export class PoolScheduler extends EventEmitter { } /** Session leaves this VM: drop its alias keys. A device seat lives in the planner, not here. */ - releaseSticky(bound, stickyKey, stickyKeys) { - const keys = Array.isArray(stickyKeys) && stickyKeys.length ? stickyKeys : stickyKey ? [stickyKey] : [] + releaseSticky(bound, stickyKey, stickyKeys, versions = null) { + const keys = [...new Set([stickyKey, ...(Array.isArray(stickyKeys) ? stickyKeys : [])].filter(Boolean))] + if (this.stickyRouter?.shared) { + const snapshots = keys.map((key) => ({ + key, + record: this.stickyRouter.resolve(key), + generation: versions?.get(key) ?? this.stickyRouter.versionOf(key), + })) + return Promise.all( + snapshots.map(async ({ key, record, generation }) => { + if (record && bound?.accountId && (record.accountId !== bound.accountId || record.vmId !== bound.vmId)) + return { key, applied: false } + return { key, ...(await this.stickyRouter.unbindShared(key, { ifGeneration: generation })) } + }), + ) + } for (const key of keys) this.stickyRouter?.unbind?.(key) } @@ -1411,6 +1474,7 @@ export class PoolScheduler extends EventEmitter { */ releaseAccountSessions({ accountId = null, vmId = null } = {}) { if (!accountId && !vmId) return 0 + if (this.stickyRouter?.shared) return this.stickyRouter.unbindByAccountShared({ accountId, vmId }) return this.stickyRouter?.unbindByAccount?.({ accountId, vmId }) || 0 } @@ -1423,17 +1487,27 @@ export class PoolScheduler extends EventEmitter { */ pick( candidates, - { stickyKey, eligible = candidates, spilled = null, stickyKeys = null, deviceVmId = null, familyVmId = null } = {}, + { + stickyKey, + stickySnapshot = null, + eligible = candidates, + spilled = null, + stickyKeys = null, + deviceVmId = null, + familyVmId = null, + } = {}, ) { this.lastStickyCleared = false if (!candidates.length && !eligible?.length) return null - const sticky = this.reconcileSticky({ - stickyKey, - stickyKeys, - eligible: eligible || candidates, - spilled, - familyVmId, - }) + const sticky = + stickySnapshot || + this.reconcileSticky({ + stickyKey, + stickyKeys, + eligible: eligible || candidates, + spilled, + familyVmId, + }) this.lastStickyCleared = sticky.cleared let borrowed = false if (sticky.match) { @@ -1733,37 +1807,42 @@ export class PoolScheduler extends EventEmitter { this.emit('change') } - markSuccess(candidate, { workerStatus = null, countUsage = true } = {}) { + markSuccess(candidate, { workerStatus = null, countUsage = true, observation } = {}) { const now = Date.now() if (countUsage !== false) this.accountQuota?.noteBandSettled?.(candidate.accountId) - const prev = this.runtimeRepo?.get?.(candidate.accountId) if (countUsage !== false) this.lastUsed.set(candidate.accountId, now) - this.runtimeRepo?.upsert?.({ - account_id: candidate.accountId, - vm_id: candidate.vmId, - status: 'ready', - priority: candidate.priority, - weight: candidate.weight, - cooldown_until: null, - cooldown_reason: null, - last_used_at: countUsage === false ? prev?.last_used_at || this.lastUsed.get(candidate.accountId) || null : now, - worker_heartbeat_at: workerStatus ? now : candidate.state?.worker_heartbeat_at, - worker_status: workerStatus || candidate.state?.worker_status || null, - }) - if (this.projectRoot && candidate.vmId) { + this.runtimeRepo?.upsert?.( + { + account_id: candidate.accountId, + ...(!this.shared + ? { + status: 'ready', + priority: candidate.priority, + weight: candidate.weight, + cooldown_until: null, + cooldown_reason: null, + } + : {}), + ...(countUsage === false ? {} : { last_used_at: now }), + ...(workerStatus ? { worker_heartbeat_at: now, worker_status: workerStatus } : {}), + }, + { vmId: candidate.vmId, observation }, + ) + if (!this.shared && this.projectRoot && candidate.vmId) { try { clearVmAuthCooldown(vmJsonPath(this.projectRoot, candidate.vmId)) } catch {} } } - markCooldown(candidate, { until, reason, model = null, status = 'cooldown' } = {}) { + markCooldown(candidate, { until, reason, model = null, status = 'cooldown', observation } = {}) { this.runtimeRepo?.markCooldown?.(candidate.accountId, { vmId: candidate.vmId, until, reason, model: model ? normalizeModel(model) : null, status, + observation, }) if (isAuthCooldownReason(reason) && this.projectRoot && candidate.vmId) { try { @@ -2015,14 +2094,18 @@ export class PoolScheduler extends EventEmitter { return { action: 'restrict', reason: ev.reason, until } } if (ev.accept) { - this.clearQuotaRestriction(vm, accountId, file) - if (leftover) { - setVmSchedulable(this.projectRoot, vm.id, true, null, { preserveStatus: true, source: 'force' }) - vm.schedulable = true - vm.schedule_disabled_reason = null - return { action: 'enable', reason: null } + const finish = (cleared) => { + if (cleared === false) return { action: 'keep', reason: 'quota_state_changed' } + if (leftover) { + setVmSchedulable(this.projectRoot, vm.id, true, null, { preserveStatus: true, source: 'force' }) + vm.schedulable = true + vm.schedule_disabled_reason = null + return { action: 'enable', reason: null } + } + return { action: 'clear', reason: null } } - return { action: 'clear', reason: null } + const cleared = this.clearQuotaRestriction(vm, accountId, file) + return cleared?.then ? cleared.then(finish) : finish(cleared) } if (leftover) { setVmSchedulable(this.projectRoot, vm.id, true, null, { preserveStatus: true, source: 'force' }) @@ -2057,7 +2140,21 @@ export class PoolScheduler extends EventEmitter { const reason = state?.cooldown_reason || vm.claude?.temp_unschedulable_reason || vm.temp_unschedulable_reason if (reason && !isAccountRestrictionReason(reason) && !isQuotaWindowReason(reason)) return false if (isAuthCooldownReason(reason)) return false - if (state && isAccountRestrictionReason(state.cooldown_reason)) { + const apply = () => { + const next = clearVmQuotaRestriction(file) + if (next) { + vm.claude = next.claude || vm.claude + delete vm.temp_unschedulable_until + delete vm.temp_unschedulable_reason + } + return true + } + if (state && (isAccountRestrictionReason(state.cooldown_reason) || isQuotaWindowReason(state.cooldown_reason))) { + if (this.shared) { + return this.runtimeRepo + .clearQuotaCooldownCommitted(accountId, { vmId: vm.id }) + .then((applied) => (applied ? apply() : false)) + } this.runtimeRepo?.upsert?.({ account_id: accountId, vm_id: vm.id, @@ -2066,13 +2163,7 @@ export class PoolScheduler extends EventEmitter { status: 'ready', }) } - const next = clearVmQuotaRestriction(file) - if (next) { - vm.claude = next.claude || vm.claude - delete vm.temp_unschedulable_until - delete vm.temp_unschedulable_reason - } - return true + return apply() } snapshot() { diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index c9330873d..b3aff4448 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -995,11 +995,16 @@ export function createHandleProtocol(deps) { ? null : resolveSeatIdentity({ inbound, body: ctx.body, headers: req.headers, clientIp: clientIp(req) }).key const sessionKeys = stickyRouter?.sessionPoolKeys - ? stickyRouter.sessionPoolKeys(req, inbound, { - sessionId: inboundIdentity.sessionId, - deviceId: stickyDeviceId, - migrate: !isProbe, - }) + ? await (stickyRouter.shared ? stickyRouter.sessionPoolKeysShared : stickyRouter.sessionPoolKeys).call( + stickyRouter, + req, + inbound, + { + sessionId: inboundIdentity.sessionId, + deviceId: stickyDeviceId, + migrate: !isProbe, + }, + ) : { stickyKey: stickyRouter?.extractPoolKey?.(req, inbound, { platform: 'anthropic' }) || null, stickyKeys: stickyRouter?.collectPoolKeys?.(req, inbound, { platform: 'anthropic' }) || [], @@ -1014,7 +1019,12 @@ export function createHandleProtocol(deps) { const familyKey = !familySession ? null : stickyRouter?.familyPoolKey - ? stickyRouter.familyPoolKey(req, familySession, { trusted: familyTrusted }) + ? await (stickyRouter.shared ? stickyRouter.familyPoolKeyShared : stickyRouter.familyPoolKey).call( + stickyRouter, + req, + familySession, + { trusted: familyTrusted }, + ) : stickyRouter?.familyKey?.(req, familySession, 'anthropic') || null const familyVmId = familyKey ? stickyRouter?.resolve?.(familyKey)?.vmId || null : null const stickyBound = @@ -1688,15 +1698,16 @@ export function createHandleProtocol(deps) { const pool = typeof deps.getPoolScheduler === 'function' ? deps.getPoolScheduler() : deps.poolScheduler // Live `5h-status` on every response; `allowed` is the only early unblock. if (!exhausted) { - pool?.rateLimitService?.updateSessionWindow?.({ + await pool?.rateLimitService?.updateSessionWindow?.({ accountId: result.accountId, vmId: result.vmId || null, headers: result.headers || {}, + observation: result?.accountObservation, }) } if (pool?.syncQuotaSchedule && result.vmId && cfg?.paths?.project) { const vm = getVm(cfg.paths.project, result.vmId) - if (vm) pool.syncQuotaSchedule(vm) + if (vm) await pool.syncQuotaSchedule(vm) } } catch {} await attachPricing(logBag, { diff --git a/src/server.mjs b/src/server.mjs index 6cf24f374..ac10f6c55 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -819,6 +819,8 @@ async function requireAuth(req, res) { invalidatePriceCatalog() await accountQuota.refresh?.() await proxyPool.refresh?.() + await runtimeRepo?.refresh?.() + await stickyRouter?.refresh?.() } // Panel session const session = await verifyPanelSession(token) diff --git a/test/unit/runtime-producers.test.mjs b/test/unit/runtime-producers.test.mjs index 487ac31e0..2efeba211 100644 --- a/test/unit/runtime-producers.test.mjs +++ b/test/unit/runtime-producers.test.mjs @@ -73,14 +73,17 @@ test('shared scheduler waits for committed quota cleanup and preserves VM restri const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-runtime-cleanup-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) const file = path.join(root, 'vm-cleanup.json') - const vm = { id: 'vm-cleanup', claude: { temp_unschedulable_reason: 'quota_exhausted', temp_unschedulable_until: 1 } } + const vm = { + id: 'vm-cleanup', + claude: { temp_unschedulable_reason: 'account_quota_exhausted', temp_unschedulable_until: 1 }, + } fs.writeFileSync(file, JSON.stringify(vm)) const entered = deferred() const resume = deferred() const pool = new PoolScheduler({ projectRoot: root, runtimeRepo: { - get: () => ({ cooldown_reason: 'quota_exhausted', cooldown_until: 1 }), + get: () => ({ cooldown_reason: 'account_quota_exhausted', cooldown_until: 1 }), upsert() {}, async clearQuotaCooldownCommitted() { entered.resolve() @@ -96,12 +99,15 @@ test('shared scheduler waits for committed quota cleanup and preserves VM restri return value }) try { - await Promise.race([entered.promise, result.then(() => assert.fail('cleanup returned before PostgreSQL confirmation'))]) + await Promise.race([ + entered.promise, + result.then(() => assert.fail('cleanup returned before PostgreSQL confirmation')), + ]) assert.equal(settled, false) - assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).claude.temp_unschedulable_reason, 'quota_exhausted') + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).claude.temp_unschedulable_reason, 'account_quota_exhausted') } finally { resume.resolve() } assert.equal(await result, false) - assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).claude.temp_unschedulable_reason, 'quota_exhausted') + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).claude.temp_unschedulable_reason, 'account_quota_exhausted') }) diff --git a/test/unit/shared-sticky-runner.test.mjs b/test/unit/shared-sticky-runner.test.mjs index 280ad34d2..03321f15b 100644 --- a/test/unit/shared-sticky-runner.test.mjs +++ b/test/unit/shared-sticky-runner.test.mjs @@ -27,7 +27,19 @@ async function fixture(t) { } function success() { - return { ok: true, status: 200, terminalState: 'verified', body: { type: 'message', id: 'msg_synthetic', role: 'assistant', content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', usage: { input_tokens: 1, output_tokens: 1 } } } + return { + ok: true, + status: 200, + terminalState: 'verified', + body: { + type: 'message', + id: 'msg_synthetic', + role: 'assistant', + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + usage: { input_tokens: 1, output_tokens: 1 }, + }, + } } for (const initial of ['absent', 'tombstone']) { @@ -67,10 +79,17 @@ test('runner rejects shared native-session CAS conflict before submission', asyn canonicalBody: { model: 'claude-test' }, applyAttempt: async () => { await fx.peer.refresh() - await fx.peer.rebindFamilyShared(key, { accountId: 'account-b', vmId: 'vm-b' }, { ifGeneration: fx.peer.versionOf(key) }) + await fx.peer.rebindFamilyShared( + key, + { accountId: 'account-b', vmId: 'vm-b' }, + { ifGeneration: fx.peer.versionOf(key) }, + ) return { body: {}, meta: { sessionId: 'native-a' } } }, - callAttempt: async () => { calls++; return success() }, + callAttempt: async () => { + calls++ + return success() + }, }) assert.equal(calls, 0) assert.equal(result.status, 503) @@ -88,7 +107,11 @@ test('runner late success preserves shared pin moved during execution', async (t applyAttempt: () => ({ body: {}, meta: { sessionId: 'native-a' } }), callAttempt: async () => { await fx.peer.refresh() - await fx.peer.rebindFamilyShared(key, { accountId: 'account-b', vmId: 'vm-b' }, { ifGeneration: fx.peer.versionOf(key) }) + await fx.peer.rebindFamilyShared( + key, + { accountId: 'account-b', vmId: 'vm-b' }, + { ifGeneration: fx.peer.versionOf(key) }, + ) await fx.router.refresh() return success() }, @@ -132,3 +155,34 @@ test('scheduler shared reconciliation preserves a pin moved while clearing the o assert.deepEqual(result.match, candidate) assert.equal((await fx.peer.persistence.get(key)).vm_id, 'vm-b') }) + +test('runner retains its own generation changes when retiring an empty native session', async (t) => { + const fx = await fixture(t) + fx.runner.reloadConfig({ same_account_retry_delay_ms: 0 }) + let calls = 0 + const result = await fx.runner.run({ + requestId: 'runner-empty-retire', + stickyKey: 'runner-empty-retire', + canonicalBody: { model: 'claude-test' }, + applyAttempt: (_body, _selected, { attemptNo }) => ({ body: {}, meta: { sessionId: `native-${attemptNo}` } }), + callAttempt: async () => { + calls++ + if (calls === 1) + return { + ok: true, + status: 200, + terminalState: 'verified', + body: { + type: 'message', + role: 'assistant', + content: [{ type: 'thinking', thinking: 'draft', signature: 'sig' }], + stop_reason: null, + }, + } + return success() + }, + }) + assert.equal(result.ok, true) + assert.equal(calls, 2) + assert.equal((await fx.peer.persistence.get('runner-empty-retire')).session_id, 'native-2') +}) From 5227aa194eb56ab2ac7a9b26bba522a0a263ec13 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:59:45 +0800 Subject: [PATCH 137/473] test: require committed cooldown admission results --- test/unit/runtime-cooldown-committed.test.mjs | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 test/unit/runtime-cooldown-committed.test.mjs diff --git a/test/unit/runtime-cooldown-committed.test.mjs b/test/unit/runtime-cooldown-committed.test.mjs new file mode 100644 index 000000000..7462529a2 --- /dev/null +++ b/test/unit/runtime-cooldown-committed.test.mjs @@ -0,0 +1,97 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setImmediate } from 'node:timers/promises' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SchedulingRuntime } from '../../src/lib/pool/scheduling-runtime.mjs' + +async function fixture() { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const left = await new SchedulingRuntime(db).init() + const right = await new SchedulingRuntime(other).init() + left.upsert({ account_id: 'cooldown-account', vm_id: 'cooldown-vm', status: 'ready' }) + await left.flush() + await right.refresh() + return { left, right } +} + +async function mark(runtime, options) { + if (runtime.markCooldownCommitted) return runtime.markCooldownCommitted('cooldown-account', options) + const value = runtime.markCooldown('cooldown-account', options) + await runtime.flush() + return { applied: !!value, state: runtime.get('cooldown-account') } +} + +for (const model of [null, 'claude-test']) { + test(`an old ${model ? 'model' : 'account'} cooldown reports not applied after a newer observation`, async () => { + const { left, right } = await fixture() + const older = await left.beginObservation() + const newer = await right.beginObservation() + const until = Date.now() + 90_000 + right.markCooldown('cooldown-account', { + vmId: 'cooldown-vm', + model, + until, + reason: 'newer-block', + observation: newer, + }) + await right.flush() + const result = await mark(left, { + vmId: 'cooldown-vm', + model, + until: until - 1000, + reason: 'older-block', + observation: older, + }) + assert.equal(result.applied, false) + assert.equal(model ? result.state.model_states[model].cooldown_until : result.state.cooldown_until, until) + assert.equal(model ? result.state.model_states[model].reason : result.state.cooldown_reason, 'newer-block') + }) +} + +test('committed cooldowns publish cache only after PG accepts the queued mutation', async () => { + const { left } = await fixture() + let release + const paused = new Promise((resolve) => { + release = resolve + }) + left.writes.enqueue(() => paused) + const until = Date.now() + 60_000 + const pending = mark(left, { vmId: 'cooldown-vm', until, reason: 'synthetic-block' }) + try { + await setImmediate() + assert.equal(left.get('cooldown-account').cooldown_until, null) + } finally { + release() + await pending + } + const result = await pending + assert.equal(result.applied, true) + assert.equal(result.state.cooldown_until, until) + assert.equal((await left.persistence.get('cooldown-account')).cooldown_until, until) +}) + +test('independent model versions preserve peer cooldowns while accepting an older different-model observation', async () => { + const { left, right } = await fixture() + const older = await left.beginObservation() + const newer = await right.beginObservation() + const until = Date.now() + 60_000 + right.markCooldown('cooldown-account', { + vmId: 'cooldown-vm', + model: 'model-b', + until, + reason: 'peer', + observation: newer, + }) + await right.flush() + const result = await mark(left, { + vmId: 'cooldown-vm', + model: 'model-a', + until, + reason: 'older-independent', + observation: older, + }) + assert.equal(result.applied, true) + assert.equal(result.state.model_states['model-a'].reason, 'older-independent') + assert.equal(result.state.model_states['model-b'].reason, 'peer') +}) From 3f349d0305f762d4c16cd0ec4fe7e65c00300bd8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 19:59:47 +0800 Subject: [PATCH 138/473] test: reject cooldown side effects before committed observation --- test/unit/scheduler-cooldown-commit.test.mjs | 88 ++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 test/unit/scheduler-cooldown-commit.test.mjs diff --git a/test/unit/scheduler-cooldown-commit.test.mjs b/test/unit/scheduler-cooldown-commit.test.mjs new file mode 100644 index 000000000..34ab219a1 --- /dev/null +++ b/test/unit/scheduler-cooldown-commit.test.mjs @@ -0,0 +1,88 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +test('shared cooldown side effects await PostgreSQL acceptance and skip a rejected observation', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cooldown-commit-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + fs.mkdirSync(path.join(root, 'vms')) + const file = path.join(root, 'vms', 'vm-cooldown.json') + fs.writeFileSync(file, JSON.stringify({ id: 'vm-cooldown', claude: {} })) + const entered = deferred() + const resume = deferred() + const pool = new PoolScheduler({ + projectRoot: root, + runtimeRepo: { + markCooldown() {}, + async markCooldownCommitted() { + entered.resolve() + await resume.promise + return { applied: false, state: { cooldown_reason: null } } + }, + }, + }) + pool.shared = true + const result = Promise.resolve( + pool.markCooldown( + { vmId: 'vm-cooldown', accountId: 'account-cooldown' }, + { + until: Date.now() + 60_000, + reason: 'account_quota_exhausted', + observation: 1, + }, + ), + ) + try { + await Promise.race([ + entered.promise, + result.then(() => assert.fail('VM cooldown changed before PostgreSQL accepted the observation')), + ]) + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).temp_unschedulable_reason, undefined) + } finally { + resume.resolve() + } + assert.equal(await result, false) + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).temp_unschedulable_reason, undefined) +}) + +test('a rejected cooldown observation cannot invalidate the current shared account sessions', async () => { + let releases = 0 + const runner = new FailoverRunner({ + scheduler: { + async selectAndReserve() { + return { ok: true, accountId: 'account-cooldown', vmId: 'vm-cooldown', release() {} } + }, + async markCooldown() { + return false + }, + async releaseAccountSessions() { + releases++ + }, + }, + config: { max_total_attempts: 1, max_same_account_retries: 0, same_account_retry_delay_ms: 0 }, + }) + await runner.run({ + requestId: 'rejected-cooldown', + canonicalBody: { model: 'claude-test' }, + callAttempt: async () => ({ + ok: false, + status: 429, + terminalState: 'rejected', + accountObservation: 1, + body: { type: 'error', error: { type: 'api_error', message: "You've hit your limit · resets 11am (UTC)" } }, + }), + }) + assert.equal(releases, 0) +}) From 0da3bae36021fd0a41f82f165262dc4d7b6551c2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:00:30 +0800 Subject: [PATCH 139/473] test: honor excluded accounts in cooldown regression fixture --- test/unit/scheduler-cooldown-commit.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/scheduler-cooldown-commit.test.mjs b/test/unit/scheduler-cooldown-commit.test.mjs index 34ab219a1..2b697b8ed 100644 --- a/test/unit/scheduler-cooldown-commit.test.mjs +++ b/test/unit/scheduler-cooldown-commit.test.mjs @@ -61,7 +61,8 @@ test('a rejected cooldown observation cannot invalidate the current shared accou let releases = 0 const runner = new FailoverRunner({ scheduler: { - async selectAndReserve() { + async selectAndReserve({ excluded }) { + if (excluded.has('account-cooldown')) return { ok: false, reason: 'no_eligible_accounts' } return { ok: true, accountId: 'account-cooldown', vmId: 'vm-cooldown', release() {} } }, async markCooldown() { From c16a914a3b016ac18327ffbfd753844faaf4dd3e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:00:37 +0800 Subject: [PATCH 140/473] fix: expose durable cooldown admission results --- src/lib/db/repos/account-runtime-repo.mjs | 21 +++++++++++++++++---- src/lib/pool/scheduling-runtime.mjs | 7 +++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/lib/db/repos/account-runtime-repo.mjs b/src/lib/db/repos/account-runtime-repo.mjs index d3d0276c2..6495d8739 100644 --- a/src/lib/db/repos/account-runtime-repo.mjs +++ b/src/lib/db/repos/account-runtime-repo.mjs @@ -180,12 +180,16 @@ export class AccountRuntimeRepo { const versions = account.observation_versions || {} if ( atomic && - Object.entries(fields).some( + (Object.entries(fields).some( ([field, value]) => value !== undefined && (Object.hasOwn(ACCOUNT_FIELDS, field) || Object.hasOwn(RUNTIME_FIELDS, field)) && !accepts(versions, ['runtime', field], sequence), - ) + ) || + (clearModels && !accepts(versions, ['runtime', 'model_states'], sequence)) || + Object.keys(modelPatch || {}).some( + (model) => !accepts(versions, ['runtime', 'model_states', model], sequence), + )) ) return rejected() const acc = {} @@ -285,7 +289,10 @@ export class AccountRuntimeRepo { return this.patch(accountId, fields, { vmId, observation, returnApplied: true, atomic: true }) } - async markCooldown(accountId, { vmId, until, reason, model = null, status = 'cooldown', observation } = {}) { + async markCooldown( + accountId, + { vmId, until, reason, model = null, status = 'cooldown', observation, committed = false } = {}, + ) { const cooldown = Number(until) || null if (model) return this.patch( @@ -294,6 +301,8 @@ export class AccountRuntimeRepo { { vmId, observation, + returnApplied: committed, + atomic: committed, modelPatch: { [model]: { status, cooldown_until: cooldown, reason: reason || null, updated_at: new Date().toISOString() }, }, @@ -309,10 +318,14 @@ export class AccountRuntimeRepo { ? { rate_limited_at: Date.now(), rate_limit_reset_at: cooldown } : {}), }, - { vmId, observation }, + { vmId, observation, returnApplied: committed, atomic: committed }, ) } + markCooldownCommitted(accountId, options = {}) { + return this.markCooldown(accountId, { ...options, committed: true }) + } + static AUTH_COOLDOWN = /authentication_failed_after_refresh|permission_denied/i static GRANT_REVOKE_COOLDOWN = /oauth_revoked|oauth_invalid_grant|invalid_grant|token has been revoked|oauth_no_refresh/i diff --git a/src/lib/pool/scheduling-runtime.mjs b/src/lib/pool/scheduling-runtime.mjs index 40c7cd814..2f568c1d8 100644 --- a/src/lib/pool/scheduling-runtime.mjs +++ b/src/lib/pool/scheduling-runtime.mjs @@ -205,6 +205,13 @@ export class SchedulingRuntime { ) } + markCooldownCommitted(accountId, { observation, ...options } = {}) { + const ticket = this.ticket(observation) + return this.commit(accountId, () => + this.persistence.markCooldownCommitted(accountId, { ...options, observation: ticket }), + ) + } + static AUTH_COOLDOWN = AccountRuntimeRepo.AUTH_COOLDOWN static GRANT_REVOKE_COOLDOWN = AccountRuntimeRepo.GRANT_REVOKE_COOLDOWN isAuthCooldown(current) { From 5f94e31f143ee1839c54a6d058e7cd6f0c687679 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:00:37 +0800 Subject: [PATCH 141/473] test: reproduce unsafe VM mutation producer boundaries --- test/unit/vm-operation-producers.test.mjs | 257 ++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 test/unit/vm-operation-producers.test.mjs diff --git a/test/unit/vm-operation-producers.test.mjs b/test/unit/vm-operation-producers.test.mjs new file mode 100644 index 000000000..968246a08 --- /dev/null +++ b/test/unit/vm-operation-producers.test.mjs @@ -0,0 +1,257 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { createImportCommit } from '../../src/lib/oauth/import-commit.mjs' +import { queryClaudeResetCredits, redeemClaudeResetCredit } from '../../src/lib/oauth/claude-reset-credits.mjs' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' +import { vmLockBusy } from '../../src/lib/vm/vm-file.mjs' + +async function fixture(t, { codex = false } = {}) { + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'vm-operation-producers-')) + t.after(() => fs.rmSync(project, { recursive: true, force: true })) + const original = { + VM2API_CONTROL_PLANE_MODE: process.env.VM2API_CONTROL_PLANE_MODE, + KIN_CRS_MOCK: process.env.KIN_CRS_MOCK, + } + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + process.env.KIN_CRS_MOCK = '1' + t.after(() => { + for (const [key, value] of Object.entries(original)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }) + const vm = codex + ? { id: 'vm-1', platform: 'openai', family: 'codex', codex: {}, status: 'stopped' } + : { + id: 'vm-1', + status: 'stopped', + claude: { mode: 'oauth', has_access: true, scope: 'user:profile user:inference' }, + } + fs.mkdirSync(path.join(project, 'vms'), { recursive: true }) + const vmPath = path.join(project, 'vms', 'vm-1.json') + fs.writeFileSync(vmPath, JSON.stringify(vm)) + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + const active = async () => + ( + await other + .prepare('SELECT count(*) AS n FROM vm_operations WHERE vm_id = $1 AND released_at IS NULL') + .get('vm-1') + ).n + const input = { + vmId: 'vm-1', + vmPath, + existing: vm, + oauth: { access_token: 'synthetic-import-access', account_uuid: 'synthetic-account' }, + skipOfficialCc: true, + } + const ctx = { cfg: { paths: { project } }, accountQuota: { rebindToVm() {}, clearGrantRevokeLeftover() {} } } + return { project, vmPath, vm, other, active, input, ctx } +} + +function admission(other) { + return new ExecutionAdmissionsRepo(other).tryReserve({ + accountId: 'account-a', + vmId: 'vm-1', + accountLimit: 2, + vmLimit: 2, + rpmLimit: 0, + fableLimit: 0, + }) +} + +test('credential import does not send or write before the VM claim rejects an active execution', async (t) => { + const f = await fixture(t) + const executing = await admission(f.other) + let calls = 0 + const commit = createImportCommit({ + ...f.ctx, + importWorkerCredential: async () => { + calls++ + return { ok: true } + }, + }).commitImportedOauth + try { + await assert.rejects(commit(f.input), { code: 'vm_execution_active' }) + assert.equal(calls, 0) + } finally { + await executing.release() + } +}) + +test('credential import and post-import file writes share the complete VM operation scope', async (t) => { + const f = await fixture(t) + let sawImport = false + let sawFinalFiles = false + const commit = createImportCommit({ + ...f.ctx, + importWorkerCredential: async () => { + sawImport = true + assert.equal(await f.active(), 1) + return { ok: true } + }, + getRoutingConfig: () => { + sawFinalFiles = true + assert.equal(vmLockBusy(f.vmPath), true) + assert.equal(fs.existsSync(path.join(f.project, 'vms/vm-1/cli-home/.claude/credentials.json')), true) + return {} + }, + }).commitImportedOauth + const result = await commit(f.input) + assert.equal(result.ok, true) + assert.equal(sawImport, true) + assert.equal(sawFinalFiles, true) + assert.equal(await f.active(), 0) +}) + +test('a nonthrowing unknown credential import retains the operation and denies execution', async (t) => { + const f = await fixture(t) + let calls = 0 + const commit = createImportCommit({ + ...f.ctx, + importWorkerCredential: async () => { + calls++ + return { ok: false, status: 0, error: { code: 'synthetic_remote_timeout' } } + }, + }).commitImportedOauth + const result = await commit(f.input) + assert.equal(result.ok, false) + assert.equal(calls, 1) + assert.equal(await f.active(), 1) + assert.equal((await admission(f.other)).code, 'vm_operation_active') +}) + +test('credit query holds the VM claim during its remote operation', async (t) => { + const f = await fixture(t) + const result = await queryClaudeResetCredits({ + projectRoot: f.project, + vmId: 'vm-1', + transport: async () => { + assert.equal(await f.active(), 1) + return { ok: true, status: 200, body: { cedar_ember: null } } + }, + }) + assert.equal(result.ok, true) + assert.equal(await f.active(), 0) +}) + +test('a credit query timeout returned as an ordinary error retains the VM operation', async (t) => { + const f = await fixture(t) + const result = await queryClaudeResetCredits({ + projectRoot: f.project, + vmId: 'vm-1', + transport: async () => ({ ok: false, status: 0, transportError: true }), + }) + assert.equal(result.ok, false) + assert.equal(await f.active(), 1) +}) + +test('an unknown credit redemption retains its VM operation as well as the organization file fence', async (t) => { + const f = await fixture(t) + const result = await redeemClaudeResetCredit({ + projectRoot: f.project, + vmId: 'vm-1', + idempotencyKey: 'synthetic-redemption', + transport: async (_exec, op) => { + if (op === 'profile') + return { ok: true, status: 200, body: { organization: { uuid: '11111111-1111-4111-8111-111111111111' } } } + if (op === 'reset-status') + return { + ok: true, + status: 200, + body: { + cedar_ember: { + eligible: true, + at_limit: true, + next_grant_id: 'synthetic', + grants: [ + { id: 'synthetic', resets_left: 1, usable_now: true, use_requires_limit: false, clears: ['five_hour'] }, + ], + }, + }, + } + return { ok: false, status: 0, transportError: true } + }, + }) + assert.equal(result.outcome, 'unknown') + assert.equal(await f.active(), 1) + assert.equal((await admission(f.other)).code, 'vm_operation_active') +}) + +function panel(f, overrides = {}) { + const response = {} + const handler = createPanelHandler({ + ...f.ctx, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + readBody: async () => ({}), + json(_res, status, body) { + response.status = status + response.body = body + return true + }, + ...overrides, + }) + return { response, handler } +} + +test('a reset destroy timeout returned as a normal HTTP error keeps its operation', async (t) => { + const f = await fixture(t) + // Protect the failing baseline too: its un-injected docker calls reach only this synthetic executable. + const bin = path.join(f.project, 'fake-bin') + fs.mkdirSync(bin) + fs.writeFileSync( + path.join(bin, 'docker'), + '#!/bin/sh\nif [ "$1" = inspect ]; then printf "true|123|bridge|now|synthetic|synthetic|docker||false\\n"; exit 0; fi\nexit 1\n', + { mode: 0o700 }, + ) + const previousPath = process.env.PATH + process.env.PATH = bin + path.delimiter + previousPath + t.after(() => { + process.env.PATH = previousPath + }) + const p = panel(f, { destroySlot: async () => ({ ok: false, error: 'synthetic timeout' }) }) + await p.handler({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/vm-1/reset')) + assert.equal(p.response.status, 500) + assert.equal(await f.active(), 1) +}) + +test('credential clear cannot report SIGTERM delivery as confirmed kernel exit', async (t) => { + const f = await fixture(t, { codex: true }) + // The old implementation ignores ctx mocks; remove all binary fallbacks before executing it. + const cwd = process.cwd() + const keys = ['KIN_CODEX_KERNEL_BIN', 'KIN_KERNEL_BIN', 'KIN_API_KERNEL_BIN'] + const previous = Object.fromEntries(keys.map((key) => [key, process.env[key]])) + for (const key of keys) process.env[key] = '' + process.chdir(f.project) + t.after(() => { + process.chdir(cwd) + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }) + let starts = 0 + const p = panel(f, { + stopCodexKernel: () => ({ exitCode: null, signalCode: null }), + finishCodexRecycle: async () => ({ exited: false }), + ensureCodexKernel: async () => { + starts++ + return { ok: true } + }, + }) + await p.handler({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/vm-1/credential/clear')) + assert.equal(p.response.status, 200) + assert.equal(starts, 0) + assert.equal(await f.active(), 1) + assert.equal(p.response.body.data.cleared.kernel.stopped, false) +}) From 12f19167ae5b2544add8e2a719993b5c972c644e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:07:03 +0800 Subject: [PATCH 142/473] fix(k8s): apply cooldown effects only after PostgreSQL accepts observation --- src/lib/pool/failover-runner.mjs | 12 +++++++----- src/lib/pool/pool-scheduler.mjs | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 49e9cd1bd..2e8d18c60 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -420,7 +420,7 @@ async function applyCooldown(scheduler, selected, policy, model, { diagnosticPin if (diagnosticPin && (policy.reason === 'oauth_no_refresh' || policy.reason === 'oauth_revoked')) { return } - await scheduler.markCooldown(selected, { + const applied = await scheduler.markCooldown(selected, { until: policy.action === 'disable' || policy.reason === 'oauth_no_refresh' || policy.reason === 'oauth_revoked' ? Number.MAX_SAFE_INTEGER @@ -433,12 +433,14 @@ async function applyCooldown(scheduler, selected, policy, model, { diagnosticPin ? 'disabled' : 'cooldown', }) + if (applied === false) return false // A 5xx pause, an RPM cooldown, or an unproven bare 429 keeps the // conversation pin: other seats absorb this turn. Quota and auth move // every conversation off the unit, windows included (#163). if (policy.scope === 'account' && policy.action !== 'pause' && !SOFT_COOLDOWN_REASONS.has(policy.reason)) { await scheduler.releaseAccountSessions?.({ accountId: selected?.accountId, vmId: selected?.vmId }) } + return true } export class FailoverRunner { @@ -1091,11 +1093,11 @@ export class FailoverRunner { requestBody = repairAnthropicRequest(requestBody, policy) continue } - await applyCooldown(this.scheduler, selected, policy, model, { + const cooldownApplied = await applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId, observation: result?.accountObservation, }) - if (!pinVmId && isCredentialDeath(policy)) { + if (cooldownApplied !== false && !pinVmId && isCredentialDeath(policy)) { await this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys, versions: pinVersions }) if (familyKey) { return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } @@ -1226,11 +1228,11 @@ export class FailoverRunner { if (unknownExecution || committed || !shouldContinue(policy)) { return { ...result, ...attribution(), finalState: result.terminalState, policy } } - await applyCooldown(this.scheduler, selected, policy, model, { + const cooldownApplied = await applyCooldown(this.scheduler, selected, policy, model, { diagnosticPin: !!pinVmId, observation: result?.accountObservation, }) - if (!pinVmId && isCredentialDeath(policy)) { + if (cooldownApplied !== false && !pinVmId && isCredentialDeath(policy)) { await this.forgetCredential(selected, policy, { familyKey, sessionKeys: bindKeys, versions: pinVersions }) if (familyKey) { return { ...result, ...attribution(), finalState: result.terminalState, policy } diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 419c5514a..ad5183339 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -1836,14 +1836,27 @@ export class PoolScheduler extends EventEmitter { } markCooldown(candidate, { until, reason, model = null, status = 'cooldown', observation } = {}) { - this.runtimeRepo?.markCooldown?.(candidate.accountId, { + const options = { vmId: candidate.vmId, until, reason, model: model ? normalizeModel(model) : null, status, observation, - }) + } + if (this.shared) { + return this.runtimeRepo.markCooldownCommitted(candidate.accountId, options).then(({ applied }) => { + if (!applied) return false + this.applyCooldownEffects(candidate, { until, reason, model }) + return true + }) + } + this.runtimeRepo?.markCooldown?.(candidate.accountId, options) + this.applyCooldownEffects(candidate, { until, reason, model }) + return true + } + + applyCooldownEffects(candidate, { until, reason, model }) { if (isAuthCooldownReason(reason) && this.projectRoot && candidate.vmId) { try { markVmAuthCooldown(vmJsonPath(this.projectRoot, candidate.vmId), { From 7f068745caec81cef17f9392ec01e06780763ab9 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:09:12 +0800 Subject: [PATCH 143/473] test: require PG authoritative shared routing configuration --- test/unit/shared-routing-config.test.mjs | 93 ++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 test/unit/shared-routing-config.test.mjs diff --git a/test/unit/shared-routing-config.test.mjs b/test/unit/shared-routing-config.test.mjs new file mode 100644 index 000000000..0ff41f20a --- /dev/null +++ b/test/unit/shared-routing-config.test.mjs @@ -0,0 +1,93 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createRoutingRuntime } from '../../src/lib/admin/routing-runtime.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-routing-')) + const seed = { sticky: { enabled: true, ttl_seconds: 60 }, codex: { quota: {} }, pool: { queue_max: 10 } } + const apps = [] + for (const [index, connection] of [db, peer].entries()) { + const project = path.join(dir, String(index)) + fs.mkdirSync(path.join(project, 'vms'), { recursive: true }) + const file = path.join(project, 'routing.json') + fs.writeFileSync(file, JSON.stringify(seed)) + let config = structuredClone(seed) + const calls = [] + const runtime = createRoutingRuntime({ + db: connection, + shared: true, + cfg: { paths: { project } }, + routingConfigPath: file, + getRoutingConfig: () => config, + setRoutingConfig: (next) => { + config = next + }, + stickyRouter: { reloadConfig() {} }, + accountQuota: { reloadConfig() {}, repo: { list: () => [] } }, + healthMonitor: { + setConfig(value) { + calls.push(value) + }, + }, + }) + config = await (runtime.initRoutingConfig ? runtime.initRoutingConfig() : runtime.loadRoutingConfig()) + apps.push({ runtime, file, calls, get: () => config }) + } + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + return { db, peer, apps, seed } +} + +test('routing patches merge into committed PG configuration across independent replicas', async (t) => { + const { + apps: [first, second], + } = await fixture(t) + await first.runtime.persistRoutingPatch({ sticky: { ttl_seconds: 300 } }) + await second.runtime.persistRoutingPatch({ pool: { queue_max: 20 } }) + assert.equal(second.get().sticky.ttl_seconds, 300) + assert.equal(second.get().pool.queue_max, 20) +}) + +test('a shared routing update treats its seed file as read-only input', async (t) => { + const { + apps: [first], + } = await fixture(t) + const before = fs.readFileSync(first.file, 'utf8') + fs.chmodSync(first.file, 0o444) + await first.runtime.persistRoutingPatch({ sticky: { ttl_seconds: 300 } }) + assert.equal(first.get().sticky.ttl_seconds, 300) + assert.equal(fs.readFileSync(first.file, 'utf8'), before) +}) + +test('PG routing commit failure does not publish memory, monitor changes or seed-file writes', async (t) => { + const { + db, + apps: [first], + } = await fixture(t) + const before = structuredClone(first.get()) + const file = fs.readFileSync(first.file, 'utf8') + first.calls.length = 0 + await db.exec(`CREATE FUNCTION reject_routing_write() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic routing commit failure' USING ERRCODE = 'P0001'; END $$; + CREATE TRIGGER reject_routing_write BEFORE UPDATE ON settings FOR EACH ROW EXECUTE FUNCTION reject_routing_write()`) + await assert.rejects(first.runtime.persistRoutingPatch({ health_probe: { enabled: false } }), { code: 'P0001' }) + assert.deepEqual(first.get(), before) + assert.deepEqual(first.calls, []) + assert.equal(fs.readFileSync(first.file, 'utf8'), file) +}) + +test('an explicitly stale routing revision is rejected instead of overwriting newer settings', async (t) => { + const { + apps: [first, second], + } = await fixture(t) + const revision = second.runtime.routingState?.currentRevision || 1 + await first.runtime.persistRoutingPatch({ sticky: { ttl_seconds: 300 } }) + await assert.rejects(second.runtime.persistRoutingPatch({ sticky: { ttl_seconds: 999 } }, { ifRevision: revision }), { + code: 'routing_config_stale', + }) +}) From 5b62786c4b7b633e04a3f08f7aefd09d9115c946 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:09:30 +0800 Subject: [PATCH 144/473] test: reject stale routing limits in shared admission --- test/unit/routing-admission-revision.test.mjs | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 test/unit/routing-admission-revision.test.mjs diff --git a/test/unit/routing-admission-revision.test.mjs b/test/unit/routing-admission-revision.test.mjs new file mode 100644 index 000000000..b004ccd25 --- /dev/null +++ b/test/unit/routing-admission-revision.test.mjs @@ -0,0 +1,100 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' + +const input = { + accountId: 'routing-account', + vmId: 'routing-vm', + accountLimit: 1, + vmLimit: 1, + rpmLimit: 0, + fableLimit: 0, +} +const stale = () => Object.assign(new Error('routing changed'), { code: 'routing_config_stale' }) + +test('execution admission rejects an outdated routing revision inside its transaction', async () => { + const db = getDb() + const repo = new ExecutionAdmissionsRepo(db, { + assertRoutingRevision: async (revision) => { + assert.equal(db.isTransaction, true) + if (revision !== 7) throw stale() + }, + }) + await assert.rejects(repo.tryReserve({ ...input, routingRevision: 6 }), { code: 'routing_config_stale' }) + assert.equal((await repo.getCounts(input)).vmInflight, 0) + const granted = await repo.tryReserve({ ...input, routingRevision: 7 }) + assert.equal(granted.ok, true) + await granted.release() +}) + +test('shared device admission checks routing before every resource lock', async () => { + const events = [] + const db = getDb() + const admissions = new SharedExecutionAdmissions(db, { + assertRoutingRevision: async (revision) => { + assert.equal(db.isTransaction, true) + assert.equal(revision, 7) + events.push('revision') + }, + }) + const lock = admissions.executions.lockResources.bind(admissions.executions) + admissions.executions.lockResources = async (...args) => { + events.push('resources') + return lock(...args) + } + const device = admissions.lockDevice.bind(admissions) + admissions.lockDevice = async (...args) => { + events.push('device') + return device(...args) + } + const granted = await admissions.tryReserve({ + ...input, + routingRevision: 7, + seatKey: 'routing-device', + seatCap: 2, + seatGraceMs: 0, + seatReservePct: 0, + seatHeadroom: 1, + }) + assert.equal(granted.ok, true) + assert.equal(events[0], 'revision') + assert.ok(events.indexOf('revision') < events.indexOf('device')) + await granted.release() +}) + +test('scheduler rejects its old candidate after a routing change and refreshes committed configuration', async () => { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + let refreshed = 0 + const routingState = { + currentRevision: 8, + async assertRevision(revision) { + if (revision !== 8) throw stale() + }, + async refresh() { + refreshed++ + }, + } + let pool + try { + pool = new PoolScheduler({ routingState }) + } finally { + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + } + const candidate = { + accountId: 'routing-pool-account', + vmId: 'routing-pool-vm', + routingRevision: 7, + maxConcurrency: 1, + vm: { policy: { maxConcurrency: 1 } }, + } + const lease = await pool.reserveShared(candidate, {}) + if (lease) await lease.release() + assert.equal(lease, null) + assert.equal(refreshed, 1) + assert.equal((await pool.executionAdmissions.getCounts(candidate.accountId, candidate.vmId)).vmInflight, 0) +}) From 66e8444f407c9d9130c52d91c4cc2ced31db84fe Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:11:46 +0800 Subject: [PATCH 145/473] feat(codex): coordinate shared execution admission and session turns --- src/lib/pool/codex-slot-pool.mjs | 7 +- src/lib/pool/shared-codex-runtime.mjs | 126 ++++++++ src/lib/protocol/handle-codex.mjs | 175 +++++++++-- src/lib/transport/codex-kernel-client.mjs | 45 ++- test/unit/shared-codex-execution.test.mjs | 340 +++++++++++++++++++--- 5 files changed, 599 insertions(+), 94 deletions(-) create mode 100644 src/lib/pool/shared-codex-runtime.mjs diff --git a/src/lib/pool/codex-slot-pool.mjs b/src/lib/pool/codex-slot-pool.mjs index 7a11db493..27f791afb 100644 --- a/src/lib/pool/codex-slot-pool.mjs +++ b/src/lib/pool/codex-slot-pool.mjs @@ -197,7 +197,7 @@ export function codexAccountCandidate(vm, now = Date.now(), signals = null, poli id: vm.id, weight: Number(vm?.policy?.weight ?? vm?.weight ?? 1) || 1, concurrency: effective.max_concurrency, - maxRpm: effective.max_rpm, + maxRpm: runtime.maxRpm ?? effective.max_rpm, maxSessions: effective.max_sessions, rpmCount: runtime.rpmCount || 0, rpmResetAt: runtime.rpmResetAt ?? null, @@ -239,6 +239,7 @@ export function orderCodexSessionSlots( roundRobinCursor = null, model = null, excluded = null, + runtimeSignals = null, } = {}, ) { const list = Array.isArray(vms) ? vms : [] @@ -258,7 +259,7 @@ export function orderCodexSessionSlots( ...picked, ids: pool, sticky: false, - candidates: pool.map((id) => codexAccountCandidate(byId.get(id), now, null, policy)), + candidates: pool.map((id) => codexAccountCandidate(byId.get(id), now, runtimeSignals?.get(id), policy)), } } const home = @@ -286,7 +287,7 @@ export function orderCodexSessionSlots( return { error: 'session_window_full', ids: [], ready: [], parked: [], sticky: false, boundState, candidates: [] } } const cursor = roundRobinCursor == null ? readOpenAICursor() : roundRobinCursor - const candidates = ids.map((id) => codexAccountCandidate(byId.get(id), now, null, policy)) + const candidates = ids.map((id) => codexAccountCandidate(byId.get(id), now, runtimeSignals?.get(id), policy)) const ordered = orderOpenAIAccounts(candidates, { strategy, now, diff --git a/src/lib/pool/shared-codex-runtime.mjs b/src/lib/pool/shared-codex-runtime.mjs new file mode 100644 index 000000000..6268b1f73 --- /dev/null +++ b/src/lib/pool/shared-codex-runtime.mjs @@ -0,0 +1,126 @@ +import { getDb } from '../db/database.mjs' +import { SharedExecutionAdmissions } from './shared-execution-admissions.mjs' +import { SharedSessionTurns } from './shared-session-turns.mjs' +import { readCodexAccounts } from '../vm/codex-slot.mjs' +import { normalizeOpenAIQuotaPolicy } from './openai-quota-policy.mjs' + +function identityError() { + return Object.assign(new Error('Shared Codex execution requires exactly one account with chatgpt_account_id'), { + code: 'codex_account_identity_unavailable', + }) +} + +/** Reuses the durable Claude execution ledger and FIFO without process-local capacity authority. */ +export class SharedCodexRuntime { + constructor(db = getDb(), options = {}) { + this.db = db + this.admissions = new SharedExecutionAdmissions(db, options) + this.turns = new SharedSessionTurns(db, options) + } + + accountId(projectRoot, vm) { + const accounts = readCodexAccounts(projectRoot, vm.id) + const id = accounts.length === 1 ? String(accounts[0]?.chatgpt_account_id || '').trim() : '' + if (!id) throw identityError() + return `codex:${id}` + } + + async signals(projectRoot, vms, policy) { + const signals = new Map() + for (const vm of vms) { + let accountId + try { + accountId = this.accountId(projectRoot, vm) + } catch (error) { + if (error.code === 'codex_account_identity_unavailable') continue + throw error + } + const counts = await this.admissions.getCounts(accountId, vm.id) + signals.set(vm.id, { + inFlight: counts.vmInflight, + rpmCount: counts.rpmCount, + rpmResetAt: counts.rpmResetAt, + maxRpm: normalizeOpenAIQuotaPolicy(policy).max_rpm, + }) + } + return signals + } + + async tryReserve(projectRoot, vm, candidate, policy) { + if (!vm) return null + const limits = normalizeOpenAIQuotaPolicy(policy) + const gate = await this.admissions.tryReserve({ + accountId: this.accountId(projectRoot, vm), + vmId: vm.id, + accountLimit: limits.max_concurrency, + vmLimit: candidate.concurrency, + rpmLimit: limits.max_rpm, + fableLimit: 0, + family: 'codex', + }) + if (!gate.ok) return null + let releasing = null + const lease = { + ...gate, + uncertain: false, + released: false, + markUncertain: async () => { + if (lease.released) return { uncertain: false } + lease.uncertain = true + return gate.markUncertain() + }, + release: async () => { + if (lease.released) return { released: true } + if (lease.uncertain) { + await lease.markUncertain() + return { released: false } + } + if (releasing) return releasing + releasing = (async () => { + try { + const result = await gate.release() + if (result.released) lease.released = true + else await lease.markUncertain() + return result + } catch (error) { + lease.uncertain = true + throw error + } finally { + releasing = null + } + })() + return releasing + }, + } + return lease + } + + async invoke(lease, hop, args) { + let bound = false + let completed = false + try { + const result = await hop({ + ...args, + onExecutionStart: async (execution) => { + await lease.bindExecution(execution) + bound = true + }, + onExecutionEnd: async (execution) => { + await lease.completeExecution(execution) + completed = execution?.ended === true + if (!completed) await lease.markUncertain() + }, + }) + const notStarted = result?.execution?.started === false && result?.execution?.ended === true && !bound + if ((!bound || !completed || result?.execution?.ended !== true) && !notStarted) { + await lease.markUncertain() + if (result) result.executionUncertain = true + } + return result + } catch (error) { + await lease.markUncertain() + if (error && typeof error === 'object') error.executionUncertain = true + throw error + } + } +} diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index c5bf06bbc..808a7b4af 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -2,6 +2,9 @@ * Codex hop from handle-protocol. Claude convert/pool/CRS never runs here. */ import path from 'node:path' +import { randomUUID } from 'node:crypto' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { SharedCodexRuntime } from '../pool/shared-codex-runtime.mjs' import { getVm, listVms, persistCodexUsage, syncCodexQuotaSchedule } from '../vm/vm-registry.mjs' import { isValidVmId } from '../vm/vm-file.mjs' import { isCodexProtocolAllowed, isCodexVm, normalizeCodexRouting } from './codex-route.mjs' @@ -207,7 +210,7 @@ function pinnedVmId(req) { export async function pickCodexCandidates( projectRoot, req, - { stickyRouter = null, sessions = null, body = null, excluded = null, routing = {} } = {}, + { stickyRouter = null, sessions = null, body = null, excluded = null, routing = {}, sharedRuntime = null } = {}, ) { const pin = pinnedVmId(req) const model = body?.model || null @@ -221,10 +224,14 @@ export async function pickCodexCandidates( if (!vm || !isCodexVm(vm)) return { error: 'platform_mismatch', pin, ids: [] } if (!keyAllowsVm(scope, vm)) return { error: 'key_group_mismatch', pin, ids: [] } if (!codexSlotAllowsModel(vm, model)) return { error: 'model_not_allowed', pin, ids: [] } + const runtimeSignals = sharedRuntime ? await sharedRuntime.signals(projectRoot, [vm], routingPolicy) : null + if (runtimeSignals && !runtimeSignals.has(vm.id)) + return { error: 'codex_account_identity_unavailable', ids: [], candidates: [] } const ordered = orderCodexSessionSlots([vm], { pin, excluded, quotaPolicy: routingPolicy, + runtimeSignals, }) return { ...ordered, pin, sticky: false, sessionKey: null, stickyKeys: [] } } @@ -239,7 +246,14 @@ export async function pickCodexCandidates( const sessionKey = stickyRouter?.extractPoolKey?.(req, body || {}, { platform: 'openai' }) || stickyKeys[0] || null const stickyVersions = new Map(stickyKeys.map((key) => [key, stickyRouter?.shared ? stickyRouter.versionOf(key) : 0])) const bound = sessionKey ? stickyRouter?.resolve?.(sessionKey) : null - const vms = listVms(projectRoot, { codex: { quota: routingPolicy } }).filter((vm) => keyAllowsVm(scope, vm)) + let vms = listVms(projectRoot, { codex: { quota: routingPolicy } }).filter((vm) => keyAllowsVm(scope, vm)) + if (sharedRuntime) vms = vms.filter(isCodexVm) + const runtimeSignals = sharedRuntime ? await sharedRuntime.signals(projectRoot, vms, routingPolicy) : null + if (runtimeSignals) { + const hadCandidates = vms.length > 0 + vms = vms.filter((vm) => runtimeSignals.has(vm.id)) + if (hadCandidates && !vms.length) return { error: 'codex_account_identity_unavailable', ids: [], candidates: [] } + } const continuesResponse = !!body?.previous_response_id && !!bound?.vmId if (scope.vm_pool_id && vms.length === 0 && !continuesResponse) { return { error: 'vm_pool_unavailable', message: '账号池没有可调度的 GPT 槽位', ids: [], candidates: [] } @@ -251,6 +265,7 @@ export async function pickCodexCandidates( model, excluded, quotaPolicy: routingPolicy, + runtimeSignals, }) // A resting home migrates the conversation like a gone one: the next slot rebinds it. const boundLeft = ordered.boundState === 'gone' || ordered.boundState === 'resting' @@ -416,11 +431,13 @@ async function admitCodexCandidate(projectRoot, req, opts, { deadline, signal }) const livePolicy = normalizeCodexRouting(opts.routing?.codex || opts.routing).quota if (picked.error && picked.error !== 'capacity_unavailable') return { picked } for (const candidate of picked.candidates || []) { - const lease = tryAcquireOpenAISlot(candidate.id, { - concurrency: candidate.concurrency, - maxRpm: candidate.maxRpm, - quotaPolicy: livePolicy, - }) + const lease = opts.sharedRuntime + ? await opts.sharedRuntime.tryReserve(projectRoot, getVm(projectRoot, candidate.id), candidate, livePolicy) + : tryAcquireOpenAISlot(candidate.id, { + concurrency: candidate.concurrency, + maxRpm: candidate.maxRpm, + quotaPolicy: livePolicy, + }) if (lease) return { picked, vmId: candidate.id, lease } } // A woken waiter that still missed hands the wake to the next in line. @@ -428,7 +445,11 @@ async function admitCodexCandidate(projectRoot, req, opts, { deadline, signal }) const now = Date.now() if (now >= deadline) return { picked: { ...picked, error: 'capacity_unavailable', ids: [] } } const retryAt = Number(picked.retryAt) - const sliceDeadline = Number.isFinite(retryAt) && retryAt > now ? Math.min(deadline, retryAt) : deadline + const sliceDeadline = opts.sharedRuntime + ? Math.min(deadline, now + 250) + : Number.isFinite(retryAt) && retryAt > now + ? Math.min(deadline, retryAt) + : deadline let waited try { waited = await waitForOpenAICapacity({ deadline: sliceDeadline, signal }) @@ -483,23 +504,42 @@ export function reasoningHistoryFallback(args, result) { } } -export async function runCodexKernelHop({ hop, args = {}, onEvent, onReasoningFallback } = {}) { +export async function runCodexKernelHop({ + hop, + args = {}, + onEvent, + onReasoningFallback, + sharedExecution = false, +} = {}) { let emitted = false const wrapped = async (line) => { emitted = true if (onEvent) await onEvent(line) } - let result = await hop({ ...args, onEvent: wrapped }) - if (!result?.ok && !emitted && isRetryableCodexTransport(result)) { - result = { ...(await hop({ ...args, onEvent: wrapped })), transport_retried: true } + let sends = 0 + const send = (input) => + hop({ + ...input, + ...(sharedExecution + ? { + requestId: sends++ === 0 && args.requestId != null ? args.requestId : randomUUID(), + } + : {}), + onEvent: wrapped, + }) + const canRetry = (value) => !sharedExecution || (value?.execution?.ended === true && !value?.executionUncertain) + let result = await send(args) + if (!result?.ok && !emitted && canRetry(result) && isRetryableCodexTransport(result)) { + result = { ...(await send(args)), transport_retried: true } } - const fallback = !emitted && !result?.transport_retried ? reasoningHistoryFallback(args, result) : null + const fallback = + !emitted && !result?.transport_retried && canRetry(result) ? reasoningHistoryFallback(args, result) : null if (fallback) { if (onReasoningFallback) await onReasoningFallback({ originalResult: result, body: fallback.args.body, ...fallback.info }) if (args.signal?.aborted) return result result = { - ...(await hop({ ...fallback.args, onEvent: wrapped })), + ...(await send(fallback.args)), reasoning_history_retried: true, reasoning_history_fallback: fallback.info, } @@ -585,6 +625,7 @@ export async function handleCodexProtocol({ // Chat→Responses conversion can surface a session key the raw inbound lacked. if (!logBag.session_id) logBag.session_id = sessionIdForLog(callerSession) const firstUserIdentity = firstUserIdentityFromCodex(converted.body) + const sharedRuntime = ops.sharedRuntime || (isKubernetesMode() ? new SharedCodexRuntime() : null) const hop = ops.streamCodexKernel || streamCodexKernel const writeCfg = ops.writeCodexKernelConfig || writeCodexKernelConfig const ensure = ops.ensureCodexKernel || ensureCodexKernel @@ -599,8 +640,9 @@ export async function handleCodexProtocol({ body: body || converted.body || inbound, excluded: new Set(), routing, + sharedRuntime, } - const deadline = Date.now() + codexWaitTimeoutMs(routing) + let deadline = Date.now() + codexWaitTimeoutMs(routing) const gone = clientGoneSignal(req, res) const failoverBudget = codexFailoverBudget(routing) const sameRetryDelay = Number(routing?.failover?.same_account_retry_delay_ms) @@ -613,8 +655,26 @@ export async function handleCodexProtocol({ let attempts = 0 let switches = 0 let counted = false + let turn = null + let activeLease = null + let executionUncertain = false try { + if (sharedRuntime) { + if (stickyRouter?.shared) await stickyRouter.refresh() + const turnKey = + stickyRouter?.extractPoolKey?.(req, pickOpts.body, { platform: 'openai' }) || + (callerSession ? `codex:${req.apiKeyRecord?.id || ''}:${callerSession}` : null) + if (turnKey) + turn = await sharedRuntime.turns.acquire(turnKey, { + signal: gone.signal, + requestId: logBag.request_id, + timeoutMs: routing?.failover?.session_queue_timeout_ms ?? 900_000, + }) + deadline = Date.now() + codexWaitTimeoutMs(routing) + } for (;;) { + if (executionUncertain) + throw Object.assign(new Error('Codex execution termination is unknown'), { code: 'codex_execution_uncertain' }) const admitted = await admitCodexCandidate(projectRoot, req, pickOpts, { deadline, signal: gone.signal }) const picked = admitted.picked if (!admitted.lease) { @@ -628,14 +688,15 @@ export async function handleCodexProtocol({ } const vm = getVm(projectRoot, admitted.vmId) const lease = admitted.lease + activeLease = lease if (!vm || !isCodexVm(vm)) { - lease.release() + await lease.release() pickOpts.excluded.add(admitted.vmId) continue } attempts += 1 if (attempts > failoverBudget.maxAttempts) { - lease.release() + await lease.release() break } if (!counted) { @@ -649,7 +710,7 @@ export async function handleCodexProtocol({ logBag.vm_id = vm.id const blocked = await accountGate(scopeAccountFromVm(vm)) if (blocked) { - lease.release() + await lease.release() return } } @@ -763,18 +824,20 @@ export async function handleCodexProtocol({ let hopEmitted = false const runHop = async () => await runCodexKernelHop({ - hop: stickyRouter?.shared - ? async (args) => { - // Confirm each actual submission, including hidden retries. A CAS - // conflict must never send a body carrying a different native session. - if (!(await bindSticky(outboundSessionId))) { - throw Object.assign(new Error('Shared session binding changed before Codex submission'), { - code: 'sticky_binding_conflict', - }) + sharedExecution: !!sharedRuntime, + hop: + sharedRuntime || stickyRouter?.shared + ? async (args) => { + // Confirm each actual submission, including hidden retries. A CAS + // conflict must never send a body carrying a different native session. + if (stickyRouter?.shared && !(await bindSticky(outboundSessionId))) { + throw Object.assign(new Error('Shared session binding changed before Codex submission'), { + code: 'sticky_binding_conflict', + }) + } + return sharedRuntime ? sharedRuntime.invoke(lease, hop, args) : hop(args) } - return hop(args) - } - : hop, + : hop, args: { exec: execFor(projectRoot, vm), signal: gone.signal, @@ -837,7 +900,8 @@ export async function handleCodexProtocol({ headersSent: res.headersSent || hopEmitted, conversionError, signal: gone.signal, - }) + }) && + (!sharedRuntime || (result?.execution?.ended === true && !result?.executionUncertain)) ) { sameAccountRetries += 1 attempts += 1 @@ -858,6 +922,21 @@ export async function handleCodexProtocol({ result.terminalState = 'incomplete' result.body = { error: conversionError } } + if (sharedRuntime && (result?.executionUncertain || result?.execution?.ended !== true)) { + executionUncertain = true + logBag.final_state = gone.signal.aborted ? 'cancelled' : 'incomplete' + logBag.error_code = 'codex_execution_uncertain' + if (gone.signal.aborted) return + stats.errors++ + if (res.headersSent || hopEmitted) return res.end() + return json(res, 502, { + error: { + type: 'api_error', + code: 'codex_execution_uncertain', + message: 'Codex execution termination is unknown', + }, + }) + } ingestCodexHop(projectRoot, vm.id, result, Date.now(), normalizeCodexRouting(routing.codex).quota) if (result?.transport_retried) logBag.transport_retried = true if (result?.reasoning_history_retried) { @@ -942,7 +1021,11 @@ export async function handleCodexProtocol({ logBag.final_state = 'rejected' return json(res, 503, { error: { type: 'api_error', code: error.code, message: '会话绑定已变更，请重试请求' } }) } finally { - lease.release() + try { + await lease.release() + } finally { + if (sharedRuntime && (lease.uncertain || !lease.released)) executionUncertain = true + } if (claimsWindow) { try { sessions?.release?.(vm.id, picked.sessionKey, { gen: windowGen }) @@ -950,8 +1033,36 @@ export async function handleCodexProtocol({ } } } + } catch (error) { + if (error?.code === 'request_cancelled') { + logBag.final_state = 'cancelled' + return + } + if (error?.code === 'session_queue_timeout') { + return rejectCodexAdmission({ + res, + json, + stats, + logBag, + picked: { error: 'capacity_unavailable' }, + model: converted.body.model, + }) + } + throw error } finally { - gone.settle() + try { + try { + if (sharedRuntime && activeLease && !activeLease.released) await activeLease.release() + } finally { + if (turn) { + if (executionUncertain || activeLease?.uncertain || (activeLease && !activeLease.released)) + await turn.markUncertain() + else await turn.release() + } + } + } finally { + gone.settle() + } } stats.errors++ logBag.error_code = last?.body?.error?.code || 'codex_upstream' diff --git a/src/lib/transport/codex-kernel-client.mjs b/src/lib/transport/codex-kernel-client.mjs index 303a6f423..973233d97 100644 --- a/src/lib/transport/codex-kernel-client.mjs +++ b/src/lib/transport/codex-kernel-client.mjs @@ -41,24 +41,41 @@ function remapVia(result) { } } +async function dispatchCodex(opts, send) { + let bound = false + let startFailed = false + let startError + const result = await send({ + ...opts, + exec: withCodexKernelExec(opts.exec), + requestPath: '/internal/v1/codex/responses', + onExecutionStart: + typeof opts.onExecutionStart === 'function' + ? async (execution) => { + try { + await opts.onExecutionStart(execution) + bound = true + } catch (error) { + startFailed = true + startError = error + throw error + } + } + : opts.onExecutionStart, + }) + if (startFailed) throw startError + if (typeof opts.onExecutionEnd === 'function' && (bound || typeof opts.onExecutionStart !== 'function')) { + await opts.onExecutionEnd(result?.execution) + } + return remapVia(result) +} + export async function callCodexKernel(opts = {}) { - return remapVia( - await callGoWorker({ - ...opts, - exec: withCodexKernelExec(opts.exec), - requestPath: '/internal/v1/codex/responses', - }), - ) + return dispatchCodex(opts, callGoWorker) } export async function streamCodexKernel(opts = {}) { - return remapVia( - await streamGoWorker({ - ...opts, - exec: withCodexKernelExec(opts.exec), - requestPath: '/internal/v1/codex/responses', - }), - ) + return dispatchCodex(opts, streamGoWorker) } export async function codexKernelHealth(exec, opts = {}) { diff --git a/test/unit/shared-codex-execution.test.mjs b/test/unit/shared-codex-execution.test.mjs index 3041b0ab0..c18b3908e 100644 --- a/test/unit/shared-codex-execution.test.mjs +++ b/test/unit/shared-codex-execution.test.mjs @@ -4,15 +4,34 @@ import http from 'node:http' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' +import { EventEmitter } from 'node:events' +import { setTimeout as delay } from 'node:timers/promises' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' import { handleCodexProtocol } from '../../src/lib/protocol/handle-codex.mjs' import { resetOpenAIAccountRuntime } from '../../src/lib/pool/openai-account-runtime.mjs' +import { SharedCodexRuntime } from '../../src/lib/pool/shared-codex-runtime.mjs' const unixTest = process.platform === 'win32' ? test.skip : test const accountId = 'codex:synthetic-shared-account' const vmId = 'vm-shared-codex' +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function until(check) { + const deadline = Date.now() + 2000 + while (!(await check())) { + if (Date.now() >= deadline) assert.fail('condition did not become true') + await delay(5) + } +} + function success(res) { const usage = { input_tokens: 1, output_tokens: 1 } res.writeHead(200, { 'content-type': 'text/event-stream', 'x-usage': JSON.stringify(usage) }) @@ -21,7 +40,11 @@ function success(res) { ) } -async function fixture(t, handle = (_job, _req, res) => success(res)) { +async function fixture( + t, + handle = (_job, _req, res) => success(res), + { concurrency = 1, onCancel = (_job, _req, res) => res.end('{"ok":true,"cancelled":true}') } = {}, +) { resetOpenAIAccountRuntime() const env = ['VM2API_CONTROL_PLANE_MODE', 'KIN_CRS_MOCK'] const previous = Object.fromEntries(env.map((key) => [key, process.env[key]])) @@ -44,7 +67,7 @@ async function fixture(t, handle = (_job, _req, res) => success(res)) { status: 'running', has_token: true, runtime: { codex_kernel_socket: socket, worker_run_dir: runDir, worker_token_file: token }, - policy: { maxConcurrency: 1, concurrencyOverride: true }, + policy: { maxConcurrency: concurrency, concurrencyOverride: true }, }), ) fs.writeFileSync( @@ -63,14 +86,19 @@ async function fixture(t, handle = (_job, _req, res) => success(res)) { const db = getDb() const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) const messages = [] + const cancellations = [] const bindingsAtSend = [] + const runtimes = [db, other].map( + (connection, index) => new SharedCodexRuntime(connection, { ownerId: `codex-controller-${index}` }), + ) const server = http.createServer((req, res) => { void (async () => { const chunks = [] for await (const chunk of req) chunks.push(chunk) const job = JSON.parse(Buffer.concat(chunks).toString()) if (req.url.endsWith('/cancel')) { - res.end('{"ok":true,"cancelled":true}') + cancellations.push(job) + await onCancel(job, req, res) return } messages.push(job) @@ -98,32 +126,47 @@ async function fixture(t, handle = (_job, _req, res) => success(res)) { other, messages, bindingsAtSend, - async run({ sharedRuntime, sessionId = null } = {}) { - const response = { headersSent: false, write() {}, end() {} } - const body = { model: 'gpt-5.4', input: 'synthetic request', stream: false } - await handleCodexProtocol({ - req: { headers: sessionId ? { 'x-session-id': sessionId } : {}, apiKeyKind: 'user' }, - res: response, - protocol: 'openai.responses', - ctx: { body }, - inbound: body, - projectRoot: root, - logBag: {}, - stats: { requests: 0, errors: 0, by_route: {} }, - routing: { - codex: { quota: { max_concurrency: 1 } }, - failover: { fallback_wait_timeout_ms: 1000, max_total_attempts: 2, max_same_account_retries: 0 }, - }, - json(_res, status, payload) { - response.status = status - response.body = payload - return payload - }, - writeSSEHeaders() { - response.headersSent = true - }, - ops: { sharedRuntime, writeCodexKernelConfig() {}, ensureCodexKernel: async () => ({ ok: true }) }, + cancellations, + runtimes, + counts: () => runtimes[1].admissions.getCounts(accountId, vmId), + async run({ sharedRuntime, runtimeIndex, sessionId = null, signal, quota = {}, sessionTimeout = 100 } = {}) { + if (runtimeIndex != null) sharedRuntime = runtimes[runtimeIndex] + const response = Object.assign(new EventEmitter(), { headersSent: false, write() {}, end() {} }) + const req = Object.assign(new EventEmitter(), { + headers: sessionId ? { 'x-session-id': sessionId } : {}, + apiKeyKind: 'user', }) + const abort = () => req.emit('aborted') + signal?.addEventListener('abort', abort, { once: true }) + const body = { model: 'gpt-5.4', input: 'synthetic request', stream: false } + try { + await handleCodexProtocol({ + req, + res: response, + protocol: 'openai.responses', + ctx: { body }, + inbound: body, + projectRoot: root, + logBag: {}, + stats: { requests: 0, errors: 0, by_route: {} }, + routing: { + codex: { quota: { max_concurrency: concurrency, ...quota } }, + pool: { fallback_wait_timeout_ms: 1000 }, + failover: { session_queue_timeout_ms: sessionTimeout, max_total_attempts: 2, max_same_account_retries: 0 }, + }, + json(_res, status, payload) { + response.status = status + response.body = payload + return payload + }, + writeSSEHeaders() { + response.headersSent = true + }, + ops: { sharedRuntime, writeCodexKernelConfig() {}, ensureCodexKernel: async () => ({ ok: true }) }, + }) + } finally { + signal?.removeEventListener('abort', abort) + } return response }, } @@ -144,22 +187,229 @@ unixTest('Kubernetes Codex persists its actual kernel job before submission and assert.ok(row.ended_at && row.released_at >= row.ended_at) }) -unixTest('Codex cannot bypass account capacity held by another database connection', async (t) => { - const fx = await fixture(t) - const held = await new SharedExecutionAdmissions(fx.other).tryReserve({ - accountId, - vmId, - accountLimit: 1, - vmLimit: 1, - fableLimit: 0, - rpmLimit: 0, +for (const scope of ['account', 'VM']) { + unixTest(`Codex cannot bypass ${scope} capacity held by another database connection`, async (t) => { + const fx = await fixture(t) + const held = await new SharedExecutionAdmissions(fx.other).tryReserve({ + accountId: scope === 'account' ? accountId : 'codex:another-account', + vmId: scope === 'VM' ? vmId : 'vm-another-codex', + accountLimit: 1, + vmLimit: 1, + fableLimit: 0, + rpmLimit: 0, + }) + try { + const response = await fx.run() + assert.equal(fx.messages.length, 0, 'PostgreSQL is the final capacity authority') + assert.equal(response.status, 429) + assert.equal(response.body.error.code, 'pool_overloaded') + } finally { + await held.release() + } }) - try { - const response = await fx.run() - assert.equal(fx.messages.length, 0, 'PostgreSQL is the final capacity authority') - assert.equal(response.status, 429) - assert.equal(response.body.error.code, 'pool_overloaded') - } finally { - await held.release() - } +} + +unixTest('a truncated Codex socket retains its admission and session turn without retry or failover', async (t) => { + const fx = await fixture(t, (_job, _req, res) => res.destroy()) + const response = await fx.run({ runtimeIndex: 0, sessionId: 'unknown-conversation' }) + assert.equal(response.status, 502) + assert.equal(response.body.error.code, 'codex_execution_uncertain') + assert.equal(fx.messages.length, 1) + assert.equal((await fx.counts()).vmInflight, 1) + const job = await fx.other.prepare('SELECT * FROM execution_jobs').get() + assert.equal(job.ended_at, null) + const turn = await fx.other.prepare('SELECT * FROM session_turns').get() + assert.equal(turn.state, 'active') + assert.ok(turn.uncertain_at) + const blocked = await fx.run({ runtimeIndex: 1, sessionId: 'unknown-conversation' }) + assert.equal(blocked.status, 429) + assert.equal(fx.messages.length, 1) +}) + +unixTest('Codex EOF permits an implicit retry only with a new persisted job ID', async (t) => { + let executions = 0 + const fx = await fixture(t, (_job, _req, res) => { + if (++executions === 1) { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end('{"error":{"code":"upstream_transport"}}') + } else success(res) + }) + const response = await fx.run({ runtimeIndex: 0, sessionId: 'retry-conversation' }) + assert.equal(response.status, 200) + assert.equal(fx.messages.length, 2) + assert.notEqual(fx.messages[0].request_id, fx.messages[1].request_id) + assert.ok(fx.bindingsAtSend.every(Boolean)) + const rows = await fx.other.prepare('SELECT * FROM execution_jobs').all() + assert.equal(rows.length, 2) + assert.ok(rows.every((row) => row.ended_at)) + assert.equal((await fx.counts()).vmInflight, 0) + assert.equal((await fx.other.prepare('SELECT count(*) AS count FROM session_turns').get()).count, 0) +}) + +for (const [phase, trigger, code, expectedJobs] of [ + ['binding', 'INSERT', 'P6481', 0], + ['completion', 'UPDATE OF ended_at', 'P6482', 1], +]) { + unixTest(`a Codex PostgreSQL ${phase} failure stops jobs and retains capacity and its turn`, async (t) => { + const fx = await fixture(t) + await fx.db.exec(`CREATE FUNCTION reject_codex_job() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic codex ${phase}' USING ERRCODE = '${code}'; END; + $$; CREATE TRIGGER reject_codex_job BEFORE ${trigger} ON execution_jobs FOR EACH ROW EXECUTE FUNCTION reject_codex_job();`) + await assert.rejects(fx.run({ runtimeIndex: 0, sessionId: 'fault-conversation' }), { code }) + assert.equal(fx.messages.length, expectedJobs) + assert.equal((await fx.counts()).vmInflight, 1) + const turn = await fx.other.prepare('SELECT * FROM session_turns').get() + assert.equal(turn.state, 'active') + assert.ok(turn.uncertain_at) + }) +} + +unixTest('a Codex admission release failure cannot release its shared session turn', async (t) => { + const fx = await fixture(t) + await fx.db.exec(`CREATE FUNCTION reject_codex_release() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic codex release' USING ERRCODE = 'P6483'; END; + $$; CREATE TRIGGER reject_codex_release BEFORE UPDATE OF released_at ON execution_admissions + FOR EACH ROW EXECUTE FUNCTION reject_codex_release();`) + await assert.rejects(fx.run({ runtimeIndex: 0, sessionId: 'release-conversation' }), { code: 'P6483' }) + assert.equal(fx.messages.length, 1) + assert.equal((await fx.counts()).vmInflight, 1) + const turn = await fx.other.prepare('SELECT * FROM session_turns').get() + assert.equal(turn.state, 'active') + assert.ok(turn.uncertain_at) }) + +unixTest( + 'independent Codex runtimes serialize one session, allow another, and await release before advancing FIFO', + { timeout: 10000 }, + async (t) => { + const entered = deferred() + const finishJob = deferred() + const releaseEntered = deferred() + const finishRelease = deferred() + let jobs = 0 + const fx = await fixture( + t, + async (_job, _req, res) => { + if (++jobs === 1) { + entered.resolve() + await finishJob.promise + } + success(res) + }, + { concurrency: 2 }, + ) + const reserve = fx.runtimes[0].admissions.tryReserve.bind(fx.runtimes[0].admissions) + let first = true + fx.runtimes[0].admissions.tryReserve = async (...args) => { + const gate = await reserve(...args) + if (gate.ok && first) { + first = false + const release = gate.release + gate.release = async () => { + releaseEntered.resolve() + await finishRelease.promise + return release() + } + } + return gate + } + const pending = [fx.run({ runtimeIndex: 0, sessionId: 'same-conversation', sessionTimeout: 5000 })] + try { + await entered.promise + pending.push(fx.run({ runtimeIndex: 1, sessionId: 'same-conversation', sessionTimeout: 5000 })) + await until( + async () => + (await fx.other.prepare("SELECT count(*) AS count FROM session_turns WHERE state = 'waiting'").get()) + .count === 1, + ) + assert.equal(fx.messages.length, 1) + const other = await fx.run({ runtimeIndex: 1, sessionId: 'different-conversation' }) + assert.equal(other.status, 200) + assert.equal(fx.messages.length, 2, 'a different session executes while the first kernel job is pending') + finishJob.resolve() + await releaseEntered.promise + assert.equal(fx.messages.length, 2, 'the same-session waiter still cannot execute before lease release') + finishRelease.resolve() + const completed = await Promise.all(pending) + assert.ok(completed.every((response) => response.status === 200)) + assert.equal(fx.messages.length, 3) + assert.equal((await fx.counts()).accountInflight, 0) + assert.equal((await fx.other.prepare('SELECT count(*) AS count FROM session_turns').get()).count, 0) + } finally { + finishJob.resolve() + finishRelease.resolve() + await Promise.allSettled(pending) + } + }, +) + +unixTest( + 'a successful Codex cancel response cannot prove execution ended or free its turn', + { timeout: 5000 }, + async (t) => { + const entered = deferred() + const cancelEntered = deferred() + const answerCancel = deferred() + const answered = deferred() + const fx = await fixture( + t, + (_job, _req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.flushHeaders() + entered.resolve() + }, + { + onCancel: async (_job, _req, res) => { + cancelEntered.resolve() + await answerCancel.promise + res.end('{"ok":true,"cancelled":true}', () => answered.resolve()) + }, + }, + ) + const controller = new AbortController() + const pending = fx.run({ runtimeIndex: 0, sessionId: 'cancel-conversation', signal: controller.signal }) + try { + await entered.promise + controller.abort() + await cancelEntered.promise + await pending + assert.equal((await fx.counts()).vmInflight, 1) + answerCancel.resolve() + await answered.promise + assert.equal((await fx.counts()).vmInflight, 1) + assert.equal(fx.cancellations[0].request_id, fx.messages[0].request_id) + const blocked = await fx.run({ runtimeIndex: 1, sessionId: 'cancel-conversation' }) + assert.equal(blocked.status, 429) + assert.equal(fx.messages.length, 1) + } finally { + controller.abort() + answerCancel.resolve() + await pending + } + }, +) + +unixTest('released Codex admissions still enforce the shared account RPM window', async (t) => { + const fx = await fixture(t) + assert.equal((await fx.run({ runtimeIndex: 0, quota: { max_rpm: 1 } })).status, 200) + const response = await fx.run({ runtimeIndex: 1, quota: { max_rpm: 1 } }) + assert.equal(response.status, 429) + assert.equal(fx.messages.length, 1) +}) + +for (const accounts of [ + [{ id: 'without-stable-id' }], + [{ chatgpt_account_id: 'first' }, { chatgpt_account_id: 'second' }], +]) { + unixTest( + `shared Codex rejects ${accounts.length === 1 ? 'missing stable identity' : 'multiple credential accounts'} before execution`, + async (t) => { + const fx = await fixture(t) + fs.writeFileSync(path.join(fx.root, 'vms', vmId, 'codex-credentials.json'), JSON.stringify({ accounts })) + const response = await fx.run({ runtimeIndex: 0 }) + assert.equal(response.status, 503) + assert.equal(response.body.error.code, 'codex_account_identity_unavailable') + assert.equal(fx.messages.length, 0) + }, + ) +} From 2d843c9da09152dd6ee727972c8f73e7b32239ea Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:12:10 +0800 Subject: [PATCH 146/473] test: cover VM operation completion and account patch boundary --- test/unit/vm-operation-producers.test.mjs | 168 ++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/test/unit/vm-operation-producers.test.mjs b/test/unit/vm-operation-producers.test.mjs index 968246a08..65d119af7 100644 --- a/test/unit/vm-operation-producers.test.mjs +++ b/test/unit/vm-operation-producers.test.mjs @@ -5,6 +5,7 @@ import os from 'node:os' import path from 'node:path' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { AccountsRepo } from '../../src/lib/db/repos/accounts-repo.mjs' import { createImportCommit } from '../../src/lib/oauth/import-commit.mjs' import { queryClaudeResetCredits, redeemClaudeResetCredit } from '../../src/lib/oauth/claude-reset-credits.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' @@ -255,3 +256,170 @@ test('credential clear cannot report SIGTERM delivery as confirmed kernel exit', assert.equal(await f.active(), 1) assert.equal(p.response.body.data.cleared.kernel.stopped, false) }) + +test('a known not-started credential failure releases instead of reserving an unknown operation', async (t) => { + const f = await fixture(t) + const commit = createImportCommit({ + ...f.ctx, + importWorkerCredential: async () => ({ ok: false, vmOperationNotStarted: true }), + }).commitImportedOauth + assert.equal((await commit(f.input)).ok, false) + assert.equal(await f.active(), 0) +}) + +test('Kubernetes credential import awaits official initialization while holding the VM operation', async (t) => { + const f = await fixture(t) + process.env.KIN_CRS_MOCK = '0' + let entered + let finish + const started = new Promise((resolve) => { + entered = resolve + }) + const completing = new Promise((resolve) => { + finish = resolve + }) + const commit = createImportCommit({ + ...f.ctx, + routingConfig: { official_cc: { enabled: true } }, + reloadSlotReady: async () => ({ ok: true, runtime: {} }), + importWorkerCredential: async () => ({ ok: true }), + runOfficialCcBootstrap: async () => { + entered() + await completing + return { ok: true } + }, + }).commitImportedOauth + const pending = commit({ ...f.input, skipOfficialCc: false }) + await started + try { + assert.equal(await f.active(), 1) + assert.equal((await admission(f.other)).code, 'vm_operation_active') + } finally { + finish() + } + const result = await pending + assert.equal(result.official_cc_bootstrap.completed, true) + assert.equal(await f.active(), 0) +}) + +test('Kubernetes remote unsupported or disabled official initialization never falls back to local execution', async (t) => { + const f = await fixture(t) + process.env.KIN_CRS_MOCK = '0' + let calls = 0 + const ctx = { + ...f.ctx, + routingConfig: { official_cc: { enabled: false } }, + reloadSlotReady: async () => ({ ok: true, runtime: {} }), + importWorkerCredential: async () => ({ ok: true }), + runOfficialCcBootstrap: async () => { + calls++ + throw new Error('unexpected official initialization') + }, + } + const commit = createImportCommit(ctx).commitImportedOauth + const disabled = await commit({ ...f.input, skipOfficialCc: false }) + assert.equal(disabled.official_cc_bootstrap.reason, 'disabled') + const remoteVm = { ...JSON.parse(fs.readFileSync(f.vmPath)), node_id: 'synthetic-remote-node' } + fs.writeFileSync(f.vmPath, JSON.stringify(remoteVm)) + ctx.routingConfig = { official_cc: { enabled: true } } + const remote = await commit({ ...f.input, skipOfficialCc: false }) + assert.equal(remote.official_cc_bootstrap.reason, 'remote_unsupported') + assert.equal(calls, 0) + assert.equal(await f.active(), 0) +}) + +test('confirmed kernel exit and successful restart release credential clear normally', async (t) => { + const f = await fixture(t, { codex: true }) + const events = [] + const p = panel(f, { + stopCodexKernel: () => { + events.push('signal') + return {} + }, + finishCodexRecycle: async () => { + events.push('exit') + return { exited: true } + }, + ensureCodexKernel: async () => { + events.push('start') + return { ok: true } + }, + }) + await p.handler({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/vm-1/credential/clear')) + assert.deepEqual(events, ['signal', 'exit', 'start']) + assert.equal(await f.active(), 0) + assert.equal(p.response.body.data.cleared.kernel.stopped, true) +}) + +test('a confirmed destroy followed by an unknown start result keeps the reset operation', async (t) => { + const f = await fixture(t) + const vm = { + ...f.vm, + proxy: { id: 'synthetic-proxy', scheme: 'socks5', host: '127.0.0.1', port: 9, url: 'socks5://127.0.0.1:9' }, + } + fs.writeFileSync(f.vmPath, JSON.stringify(vm)) + const p = panel(f, { + readBody: async () => ({ mode: 'keep' }), + destroySlot: async () => ({ ok: true, action: 'removed' }), + startSlotReady: async () => ({ ok: false, error: 'synthetic start timeout' }), + }) + await p.handler({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/vm-1/reset')) + assert.equal(p.response.status, 500) + assert.equal(await f.active(), 1) +}) + +test('a reset rejected before sending a destroy is known not-started and releases its operation', async (t) => { + const f = await fixture(t) + const p = panel(f, { + destroySlot: async () => ({ ok: false, error: 'synthetic preflight rejected', vmOperationNotStarted: true }), + }) + await p.handler({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/vm-1/reset')) + assert.equal(p.response.status, 500) + assert.equal(await f.active(), 0) +}) + +test('credential clear awaits the account identity patch before releasing its VM operation', async (t) => { + const f = await fixture(t) + const repo = new AccountsRepo(f.other) + await repo.insert({ account_id: 'vm-1', email: 'synthetic@example.invalid', name: 'synthetic' }) + let entered + let finish + let patched = false + let completed = false + const started = new Promise((resolve) => { + entered = resolve + }) + const completing = new Promise((resolve) => { + finish = resolve + }) + const p = panel(f, { + accountQuota: { + repo: { + get: (...args) => repo.get(...args), + async patchFields(...args) { + patched = true + entered() + await completing + return repo.patchFields(...args) + }, + }, + }, + stopCodexKernel: () => null, + }) + const pending = p + .handler({ method: 'POST' }, {}, new URL('http://localhost/api/panel/vms/vm-1/credential/clear')) + .finally(() => { + completed = true + }) + await Promise.race([started, pending]) + try { + assert.equal(patched, true) + assert.equal(completed, false) + assert.equal(await f.active(), 1) + } finally { + finish() + await pending + } + assert.equal((await repo.get('vm-1')).email, null) + assert.equal(await f.active(), 0) +}) From 8716eac9a5ae67de92397a75e25321df65254bc5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:12:32 +0800 Subject: [PATCH 147/473] test: verify routing lock duration and stale inherited VM limits --- test/unit/routing-admission-revision.test.mjs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/test/unit/routing-admission-revision.test.mjs b/test/unit/routing-admission-revision.test.mjs index b004ccd25..17ef8827a 100644 --- a/test/unit/routing-admission-revision.test.mjs +++ b/test/unit/routing-admission-revision.test.mjs @@ -52,6 +52,8 @@ test('shared device admission checks routing before every resource lock', async } const granted = await admissions.tryReserve({ ...input, + accountId: 'routing-seat-account', + vmId: 'routing-seat-vm', routingRevision: 7, seatKey: 'routing-device', seatCap: 2, @@ -98,3 +100,61 @@ test('scheduler rejects its old candidate after a routing change and refreshes c assert.equal(refreshed, 1) assert.equal((await pool.executionAdmissions.getCounts(candidate.accountId, candidate.vmId)).vmInflight, 0) }) + +test('configuration writes wait until admission commits its validated revision', async () => { + const db = getDb() + await db.prepare("INSERT INTO settings(key,value) VALUES('routing-admission-lock','7')").run() + let entered + let resume + const checked = new Promise((resolve) => { + entered = resolve + }) + const hold = new Promise((resolve) => { + resume = resolve + }) + const repo = new ExecutionAdmissionsRepo(db, { + assertRoutingRevision: async (revision) => { + const row = await db.prepare("SELECT value FROM settings WHERE key='routing-admission-lock' FOR SHARE").get() + assert.equal(Number(row.value), revision) + entered() + await hold + }, + }) + const pending = repo.tryReserve({ + ...input, + accountId: 'routing-lock-account', + vmId: 'routing-lock-vm', + routingRevision: 7, + }) + await checked + const client = await db.pool.connect() + const changed = client.query("UPDATE settings SET value='8' WHERE key='routing-admission-lock'") + try { + let blocked = false + for (let i = 0; i < 100; i++) { + const { rows } = await db.query('SELECT cardinality(pg_blocking_pids($1::integer)) AS blockers', [ + client.processID, + ]) + if (rows[0].blockers > 0) { + blocked = true + break + } + await new Promise((resolve) => setTimeout(resolve, 10)) + } + assert.equal(blocked, true, 'the configuration update must wait for the admission transaction') + } finally { + resume() + const lease = await pending + await changed + client.release() + await lease.release() + } +}) + +test('current global limits replace stale inherited VM policy while explicit overrides remain', () => { + const pool = new PoolScheduler({ routingState: { currentRevision: 8 } }) + const candidate = { maxConcurrency: 2, vm: { policy: { maxConcurrency: 9, concurrencyOverride: false } } } + assert.equal(pool.executionLimits(candidate).vmLimit, 2) + candidate.vm.policy.concurrencyOverride = true + assert.equal(pool.executionLimits(candidate).vmLimit, 9) +}) From 06d841cd5fb5d81366cf0a7684b408fcd7368ac3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:12:54 +0800 Subject: [PATCH 148/473] fix(k8s): validate routing revision inside shared admission transaction --- .../db/repos/execution-admissions-repo.mjs | 10 +++++++++- src/lib/pool/pool-scheduler.mjs | 19 +++++++++++++++++-- src/lib/pool/shared-execution-admissions.mjs | 1 + 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index ac1701e4c..56842b5c2 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -25,9 +25,16 @@ function jobError(code) { /** Short PostgreSQL transactions for global account/VM execution limits. */ export class ExecutionAdmissionsRepo { - constructor(db, { ownerId = 'controller_' + crypto.randomUUID() } = {}) { + constructor(db, { ownerId = 'controller_' + crypto.randomUUID(), assertRoutingRevision = null } = {}) { this.db = db this.ownerId = identifier(ownerId, 'ownerId') + this.routingRevisionCheck = assertRoutingRevision + } + + async assertRoutingRevision(revision) { + if (!this.routingRevisionCheck) return + if (!this.db.isTransaction) throw jobError('execution_transaction_required') + await this.routingRevisionCheck(revision) } /** Shared order: device (caller), account, then every involved VM sorted. */ @@ -91,6 +98,7 @@ export class ExecutionAdmissionsRepo { const rpmLimit = limit(input.rpmLimit, 'rpmLimit') const family = input.family == null ? null : identifier(input.family, 'family') return withTransaction(this.db, async () => { + await this.assertRoutingRevision(input.routingRevision) await this.lockResources({ accountId, vmIds: [vmId] }) const operation = await this.db .prepare('SELECT id FROM vm_operations WHERE vm_id = $1 AND released_at IS NULL') diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index ad5183339..663078499 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -332,6 +332,7 @@ export class PoolScheduler extends EventEmitter { runtimeRepo = null, workerHealth = null, executionAdmissions = null, + routingState = null, config = {}, } = {}) { super() @@ -343,8 +344,12 @@ export class PoolScheduler extends EventEmitter { this.runtimeRepo = runtimeRepo this.workerHealth = workerHealth this.shared = isKubernetesMode() + this.routingState = routingState this.executionAdmissions = this.shared - ? executionAdmissions || new SharedExecutionAdmissions(accountQuota?.db || getDb()) + ? executionAdmissions || + new SharedExecutionAdmissions(accountQuota?.db || getDb(), { + assertRoutingRevision: routingState ? (revision) => routingState.assertRevision(revision) : null, + }) : null this.tasks = new Set() this.failure = null @@ -1029,6 +1034,7 @@ export class PoolScheduler extends EventEmitter { keyScope = null, quotaRequest = null, } = {}) { + const routingRevision = this.routingState?.currentRevision const now = Date.now() this.runtimeRepo?.clearExpired?.(now) const summaries = listVms(this.projectRoot) @@ -1085,6 +1091,7 @@ export class PoolScheduler extends EventEmitter { const seatCap = seatCapOf(vm, this.config.default_session_slots) this.planner.noteCap(vm.id, seatCap) candidates.push({ + routingRevision, ok: true, vmId: vm.id, accountId, @@ -1672,7 +1679,10 @@ export class PoolScheduler extends EventEmitter { if (candidate.draining) accountLimit = Math.min(accountLimit, 1) return { accountLimit, - vmLimit: maxConcurrencyOf(candidate.vm, candidate.maxConcurrency), + vmLimit: + this.routingState && !candidate.vm?.policy?.concurrencyOverride + ? Math.max(0, Number(candidate.maxConcurrency)) + : maxConcurrencyOf(candidate.vm, candidate.maxConcurrency), rpmLimit: Number(this.accountQuota?.rpmLimitFor?.(candidate.account, policy) ?? candidate.account?.max_rpm ?? 0), fableLimit: Number(this.config.fable_max_per_account) || 0, family: isFableModel(candidate.model) ? FABLE_FAMILY_KEY : null, @@ -1697,10 +1707,15 @@ export class PoolScheduler extends EventEmitter { accountId: candidate.accountId, vmId: candidate.vmId, ...limits, + routingRevision: candidate.routingRevision, ...options, }) } catch (error) { if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) + if (error?.code === 'routing_config_stale' && this.routingState) { + await this.routingState.refresh() + return null + } throw error } if (!lease.ok) { diff --git a/src/lib/pool/shared-execution-admissions.mjs b/src/lib/pool/shared-execution-admissions.mjs index 31fafcd7f..596ee633c 100644 --- a/src/lib/pool/shared-execution-admissions.mjs +++ b/src/lib/pool/shared-execution-admissions.mjs @@ -60,6 +60,7 @@ export class SharedExecutionAdmissions { throw invalid('seatReservePct') if (input.seatHeadroom != null && !Number.isFinite(input.seatHeadroom)) throw invalid('seatHeadroom') const gate = await withTransaction(this.db, async () => { + await this.executions.assertRoutingRevision(input.routingRevision) await this.lockDevice(seatKey) const before = await this.readSeat(seatKey) // A move involves two VM books. Lock both in the same order as admissions. From 057ef3e98e37fdb2d27a138ee9be2d7cc4253ec8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:12:55 +0800 Subject: [PATCH 149/473] fix(k8s): hold durable VM claims through mutation completion --- src/lib/admin/panel-routes.mjs | 60 ++++-- src/lib/oauth/claude-reset-credits.mjs | 168 +++++++++------ src/lib/oauth/import-commit.mjs | 282 ++++++++++++++----------- 3 files changed, 315 insertions(+), 195 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 3cc365948..2ae495829 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -65,7 +65,7 @@ import { } from '../oauth/sub2api-account.mjs' import { officialCcHome, readOfficialCcStatus, scheduleOfficialCcBootstrap } from '../oauth/official-cc-bootstrap.mjs' import { invalidateLiveCredentialCache } from './panel-live-credentials.mjs' -import { ensureCodexKernel, stopCodexKernel } from '../transport/codex-kernel-supervisor.mjs' +import { ensureCodexKernel, stopCodexKernel, finishCodexRecycle } from '../transport/codex-kernel-supervisor.mjs' import { listOfficialModels, seedModelCatalog } from '../protocol/models.mjs' import { getModelPolicy, saveModelPolicy, resetModelPolicy, listPolicyModels } from '../protocol/model-policy.mjs' import { @@ -3286,7 +3286,7 @@ export function createPanelHandler(ctx) { // POST /api/panel/vms/:id/cooldown/clear — drop 401/quota leftover so the slot re-enters if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/cooldown\/clear$/.test(p)) { const id = p.split('/')[4] - const result = panel.clearVmCooldown({ + const result = await panel.clearVmCooldown({ cfg, accountQuota, stickyRouter, @@ -3395,11 +3395,12 @@ export function createPanelHandler(ctx) { if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) const body = await readBody(req, 32 * 1024).catch(() => ({})) const mode = body.mode === 'keep' ? 'keep' : 'traits' - return await withVmLock(vmPath, async () => { + return await withVmLock(vmPath, async (vmOperation) => { if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) const prev = JSON.parse(fs.readFileSync(vmPath, 'utf8')) - const gone = await destroySlot(prev) + const gone = await (ctx.destroySlot || destroySlot)(prev) if (!gone.ok) { + if (vmOperation && gone.vmOperationNotStarted !== true) await vmOperation.markUncertain() return json(res, 500, { ok: false, error: { message: gone.error || 'destroy failed', code: gone.code } }) } let { vm } = recreateVmFiles(cfg.paths.project, prev, { mode }) @@ -3422,6 +3423,7 @@ export function createPanelHandler(ctx) { schedulable: false, schedule_disabled_reason: 'slot SOCKS5 proxy is required', }) || vm + await flushVmDbSync() return json(res, 409, { ok: false, error: { code: 'proxy_required', message: 'No healthy SOCKS5 is available for this VM' }, @@ -3429,14 +3431,19 @@ export function createPanelHandler(ctx) { destroyed: gone.action, }) } - const boot = await startSlotReady(vm, cfg.paths.project, { recreate: true, routing: ctx.routingConfig }) + const boot = await (ctx.startSlotReady || startSlotReady)(vm, cfg.paths.project, { + recreate: true, + routing: ctx.routingConfig, + }) if (!boot.ok) { + if (vmOperation && boot.vmOperationNotStarted !== true) await vmOperation.markUncertain() vm = patchVmRecord(cfg.paths.project, id, { status: 'stopped', schedulable: false, schedule_disabled_reason: boot.error || 'runtime start failed', }) || vm + await flushVmDbSync() if (gone.action !== 'absent') { return json(res, 500, { ok: false, @@ -3463,6 +3470,7 @@ export function createPanelHandler(ctx) { schedulable: mode === 'keep' ? vm.schedulable !== false && !vm.schedule_disabled_reason : false, schedule_disabled_reason: mode === 'keep' ? vm.schedule_disabled_reason || null : 'no_credential', }) || vm + await flushVmDbSync() return json( res, 200, @@ -3488,7 +3496,7 @@ export function createPanelHandler(ctx) { return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'VM not found' } }) } await readBody(req, 32 * 1024).catch(() => ({})) - return await withVmLock(vmPath, async () => { + return await withVmLock(vmPath, async (vmOperation) => { if (!fs.existsSync(vmPath)) { return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'VM not found' } }) } @@ -3507,7 +3515,9 @@ export function createPanelHandler(ctx) { fs.rmSync(file, { force: true }) removed.push(path.relative(cfg.paths.project, file)) } - } catch {} + } catch (error) { + if (vmOperation) throw error + } } const vm = patchVmRecord(cfg.paths.project, id, (record) => { @@ -3533,23 +3543,45 @@ export function createPanelHandler(ctx) { // credential-less kernel back so the slot matches an empty slot. const kernel = { stopped: false, restarted: false, error: null } try { - kernel.stopped = !!stopCodexKernel(id) + const child = (ctx.stopCodexKernel || stopCodexKernel)(id) + if (child) { + const stopped = await (ctx.finishCodexRecycle || finishCodexRecycle)(child) + kernel.stopped = stopped.exited === true + if (!kernel.stopped) { + kernel.error = 'codex_kernel_stop_unconfirmed' + await vmOperation?.markUncertain() + } + } else if (isCodexVm(vm) && vmOperation) { + kernel.error = 'codex_kernel_owner_unknown' + await vmOperation.markUncertain() + } if (isCodexVm(vm)) { - const started = await ensureCodexKernel(slotExec(cfg.paths.project, getVm(cfg.paths.project, id) || vm), { - timeoutMs: 8000, - }) - kernel.restarted = !!started?.ok - if (!started?.ok) kernel.error = started?.reason || 'codex_kernel_start_failed' + if (!kernel.error) { + const started = await (ctx.ensureCodexKernel || ensureCodexKernel)( + slotExec(cfg.paths.project, getVm(cfg.paths.project, id) || vm), + { + timeoutMs: 8000, + }, + ) + kernel.restarted = !!started?.ok + if (!started?.ok) { + kernel.error = started?.reason || 'codex_kernel_start_failed' + if (vmOperation && started?.vmOperationNotStarted !== true) await vmOperation.markUncertain() + } + } } } catch (e) { kernel.error = String(e?.message || e).slice(0, 160) + await vmOperation?.markUncertain() } try { const repo = ctx.accountQuota?.repo const row = repo && typeof repo.get === 'function' ? await repo.get(id) : null - if (row && typeof repo.patch === 'function') repo.patch(id, { email: null, name: id }) + if (row && typeof repo.patchFields === 'function') await repo.patchFields(id, { email: null, name: id }) } catch {} const saved = getVm(cfg.paths.project, id) || vm + await ctx.accountQuota?.flush?.() + await flushVmDbSync() return json( res, 200, diff --git a/src/lib/oauth/claude-reset-credits.mjs b/src/lib/oauth/claude-reset-credits.mjs index 742366827..d6b610010 100644 --- a/src/lib/oauth/claude-reset-credits.mjs +++ b/src/lib/oauth/claude-reset-credits.mjs @@ -13,6 +13,7 @@ import { credentialModeOfVm, isApiKeyMode } from './credential-mode.mjs' import { isCodexVm } from '../vm/vm-kind.mjs' import { getVm } from '../vm/vm-registry.mjs' import { atomicWriteJson, withVmLock } from '../vm/vm-file.mjs' +import { flushVmDbSync } from '../vm/vm-db-sync.mjs' export const CLAUDE_RESET_USAGE_PATH = '/api/oauth/usage?cedar_ember=1&skip_spend=1' export const CLAUDE_RESET_REDEEM_PATH = '/api/organizations/%s/reset_rate_limits' @@ -321,16 +322,27 @@ function workerProblem(res) { return null } -async function callWorker(transport, exec, op, opts) { +async function callWorker(transport, exec, op, opts, vmOperation) { + let result try { - return await transport(exec, op, { timeoutMs: CALL_TIMEOUT_MS, ...opts }) - } catch { - return { ok: false, status: 0, transportError: true, body: null } + result = await transport(exec, op, { timeoutMs: CALL_TIMEOUT_MS, ...opts }) + } catch (error) { + result = { + ok: false, + status: 0, + transportError: true, + body: null, + vmOperationNotStarted: error?.vmOperationNotStarted === true, + } } + if (vmOperation && (result?.transportError || !result?.status) && result?.vmOperationNotStarted !== true) { + await vmOperation.markUncertain() + } + return result } -async function fetchBlock(transport, exec) { - const res = await callWorker(transport, exec, 'reset-status') +async function fetchBlock(transport, exec, vmOperation) { + const res = await callWorker(transport, exec, 'reset-status', undefined, vmOperation) const mapped = workerProblem(res) if (mapped) return mapped if (res?.transportError || !res?.status) return fail('CLAUDE_RESET_QUERY_FAILED', '限额重置状态查询失败', 502) @@ -348,8 +360,8 @@ function canonicalUuid(value) { return text } -async function fetchOrg(transport, exec) { - const res = await callWorker(transport, exec, 'profile') +async function fetchOrg(transport, exec, vmOperation) { + const res = await callWorker(transport, exec, 'profile', undefined, vmOperation) const mapped = workerProblem(res) if (mapped) return mapped if (res?.transportError || res?.status !== 200) @@ -376,11 +388,18 @@ function storedOutcome(record) { return publicOutcome({ ...record, replayed: true }) } -async function claim(transport, exec, org, grantId, operation) { +async function claim(transport, exec, org, grantId, operation, vmOperation) { const unknown = { outcome: 'unknown', reason: 'claim_unconfirmed' } - const res = await callWorker(transport, exec, 'reset-redeem', { - body: { organization_uuid: org, grant_id: grantId, request_id: operation }, - }) + const res = await callWorker( + transport, + exec, + 'reset-redeem', + { + body: { organization_uuid: org, grant_id: grantId, request_id: operation }, + }, + vmOperation, + ) + if (res?.vmOperationNotStarted === true) return { ...unknown, vmOperationNotStarted: true } if (workerProblem(res) || res?.transportError || !res?.status) return unknown if (res.status === 401 || res.status === 403) return { outcome: 'ineligible', reason: 'authorization_rejected' } if (res.status !== 200 || !res.body || typeof res.body !== 'object' || Array.isArray(res.body)) return unknown @@ -432,17 +451,17 @@ function fenceBlocks(fence, now) { async function persistCredits(projectRoot, vmId, credits) { if (!credits) return const file = path.join(projectRoot, 'vms', `${vmId}.json`) - await withVmLock(file, async () => { - let vm - try { - vm = JSON.parse(fs.readFileSync(file, 'utf8')) - } catch { - return - } - vm.claude_reset_credits = persistedCredits(credits) - vm.updated_at = new Date().toISOString() - atomicWriteJson(file, vm, { mode: 0o600 }) - }) + // Both callers hold the VM operation from before the remote call through this write. + let vm + try { + vm = JSON.parse(fs.readFileSync(file, 'utf8')) + } catch { + return + } + vm.claude_reset_credits = persistedCredits(credits) + vm.updated_at = new Date().toISOString() + atomicWriteJson(file, vm, { mode: 0o600 }) + await flushVmDbSync() } function persistedCredits(view) { @@ -474,11 +493,19 @@ export async function queryClaudeResetCredits({ } = {}) { const account = loadAccount(projectRoot, vmId) if (!account.ok) return account - const fetched = await fetchBlock(transport, account.exec) - if (!fetched.ok) return fetched - const credits = projectClaudeResetCredits(fetched.block, now()) - await persistCredits(projectRoot, vmId, credits) - return { ok: true, claude_reset_credits: credits } + return withVmLock( + path.join(projectRoot, 'vms', `${vmId}.json`), + async (vmOperation) => { + const current = loadAccount(projectRoot, vmId) + if (!current.ok) return current + const fetched = await fetchBlock(transport, current.exec, vmOperation) + if (!fetched.ok) return fetched + const credits = projectClaudeResetCredits(fetched.block, now()) + await persistCredits(projectRoot, vmId, credits) + return { ok: true, claude_reset_credits: credits } + }, + { vmId, kind: 'credit-query' }, + ) } export async function redeemClaudeResetCredit({ @@ -493,38 +520,51 @@ export async function redeemClaudeResetCredit({ const account = loadAccount(projectRoot, vmId) if (!account.ok) return account const operation = claudeResetOperationId(vmId, normalized.key) - return withAccountLease(vmId, now(), async () => { - const existing = await mutateState(projectRoot, async (state) => ({ - value: storedOutcome(state.operations[operation]), - })) - if (existing?.ok === false) return existing - if (existing) return { ok: true, ...existing } - const outcome = await redeemOnce({ - projectRoot, - vmId, - operation, - exec: account.exec, - transport, - now, - }) - if (!outcome.ok) return outcome - const saved = await mutateState(projectRoot, async (state) => { - state.operations[operation] = { - outcome: outcome.outcome, - ...(outcome.reason ? { reason: outcome.reason } : {}), - ...(outcome.cleared ? { cleared: outcome.cleared } : {}), - ...(outcome.cooldown_until ? { cooldown_until: outcome.cooldown_until } : {}), - ...(outcome.claude_reset_credits ? { credits: outcome.claude_reset_credits } : {}), - at: iso(now()), - } - return { write: true, value: outcome } - }) - return saved?.ok === false ? saved : outcome - }) + return withAccountLease(vmId, now(), () => + withVmLock( + path.join(projectRoot, 'vms', `${vmId}.json`), + async (vmOperation) => { + const current = loadAccount(projectRoot, vmId) + if (!current.ok) return current + const existing = await mutateState(projectRoot, async (state) => ({ + value: storedOutcome(state.operations[operation]), + })) + if (existing?.ok === false) return existing + if (existing) { + if (existing.outcome === 'unknown') await vmOperation?.markUncertain() + return { ok: true, ...existing } + } + const outcome = await redeemOnce({ + projectRoot, + vmId, + operation, + exec: current.exec, + transport, + now, + vmOperation, + }) + if (!outcome.ok) return outcome + const saved = await mutateState(projectRoot, async (state) => { + state.operations[operation] = { + outcome: outcome.outcome, + ...(outcome.reason ? { reason: outcome.reason } : {}), + ...(outcome.cleared ? { cleared: outcome.cleared } : {}), + ...(outcome.cooldown_until ? { cooldown_until: outcome.cooldown_until } : {}), + ...(outcome.claude_reset_credits ? { credits: outcome.claude_reset_credits } : {}), + at: iso(now()), + } + return { write: true, value: outcome } + }) + if (saved?.ok === false) await vmOperation?.markUncertain() + return saved?.ok === false ? saved : outcome + }, + { vmId, kind: 'credit-redeem' }, + ), + ) } -async function redeemOnce({ projectRoot, vmId, operation, exec, transport, now }) { - const orgResult = await fetchOrg(transport, exec) +async function redeemOnce({ projectRoot, vmId, operation, exec, transport, now, vmOperation }) { + const orgResult = await fetchOrg(transport, exec, vmOperation) if (!orgResult.ok) return orgResult const orgKey = claudeResetOrgId(orgResult.org) const owner = crypto.randomUUID() @@ -539,6 +579,7 @@ async function redeemOnce({ projectRoot, vmId, operation, exec, transport, now } return fail('CLAUDE_RESET_UNRESOLVED', '上一次重置结果未确认，暂时不能再兑换', 409) } if (fence?.operation === operation) { + if (!fence.outcome || fence.outcome === 'unknown') await vmOperation?.markUncertain() return { ok: true, ...publicOutcome({ @@ -550,7 +591,7 @@ async function redeemOnce({ projectRoot, vmId, operation, exec, transport, now } } const blocked = fenceBlocks(fence, now()) if (blocked) return blocked - const status = await fetchBlock(transport, exec) + const status = await fetchBlock(transport, exec, vmOperation) if (!status.ok) return status const grant = pickGrant(status.block, now()) if (!grant) return fail('CLAUDE_RESET_NOT_AVAILABLE', '当前没有可兑换的限额重置', 409) @@ -561,17 +602,20 @@ async function redeemOnce({ projectRoot, vmId, operation, exec, transport, now } at: iso(now()), }) if (marked?.ok === false) return marked - const claimed = await claim(transport, exec, orgResult.org, grant.id, operation) + const claimed = await claim(transport, exec, orgResult.org, grant.id, operation, vmOperation) + if (claimed.outcome === 'unknown' && claimed.vmOperationNotStarted !== true) await vmOperation?.markUncertain() const settled = await writeFence(projectRoot, orgKey, { operation, outcome: claimed.outcome, ...(claimed.reason ? { reason: claimed.reason } : {}), at: iso(now()), }) - if (settled?.ok === false) + if (settled?.ok === false) { + await vmOperation?.markUncertain() return { ok: true, outcome: 'unknown', reason: 'result_persistence_failed', replayed: false } + } if (claimed.outcome !== 'unknown') { - const fresh = await fetchBlock(transport, exec) + const fresh = await fetchBlock(transport, exec, vmOperation) if (fresh.ok) { claimed.credits = projectClaudeResetCredits(fresh.block, now()) await persistCredits(projectRoot, vmId, claimed.credits) diff --git a/src/lib/oauth/import-commit.mjs b/src/lib/oauth/import-commit.mjs index c5f13f275..e54d4ec3c 100644 --- a/src/lib/oauth/import-commit.mjs +++ b/src/lib/oauth/import-commit.mjs @@ -1,7 +1,7 @@ /** * Commit an imported OAuth / API-key / setup-token grant onto a slot. * Reloads the active slot service (and eager Rust kernel when routing says so) - * before the slot-service credential import; official CC bootstrap is scheduled after persist. + * before the slot-service credential import; Kubernetes awaits official CC bootstrap after persist. */ import fs from 'node:fs' import path from 'node:path' @@ -26,10 +26,14 @@ import { persistOauthToVm, writeWorkerCredentialFile } from './oauth-credentials import { officialCcUidGid, scheduleOfficialCcBootstrap, + runOfficialCcBootstrap, + normalizeOfficialCcConfig, + loadOfficialCcConfig, materializeOfficialClaudeCredentials, } from './official-cc-bootstrap.mjs' import { normalizeTiers } from '../pool/quota-tiers.mjs' import { slotHost } from '../vm/slot-host.mjs' +import { flushVmDbSync } from '../vm/vm-db-sync.mjs' export function createImportCommit(ctx) { function routing() { @@ -146,128 +150,168 @@ export function createImportCommit(ctx) { } async function commitImportedOauth({ vmId, vmPath, existing, oauth, source, name, skipOfficialCc = false }) { - const importedCredential = importedCredentialFromOauth(oauth, existing) - const workerExec = { - vmId, - vm: existing, - homeDir: path.join(ctx.cfg.paths.project, 'vms', vmId, 'cli-home'), - } - const bound = ctx.proxyPool?.getProxyForVm?.(vmId) - if (bound) { - // Re-bind even when the id already matches: a prior stale write may have - // left proxy_cli_enabled=false on a bound slot (#332). - bindVmProxy(ctx.cfg.paths.project, vmId, bound) - adoptVmProxyBinding(existing, getVm(ctx.cfg.paths.project, vmId)) - } - if (process.env.KIN_CRS_MOCK !== '1') { - const socket = existing.runtime?.worker_socket - const needReload = !socket || !fs.existsSync(socket) || isSlotProxyDesynced(existing, ctx.cfg.paths.project) - if (needReload) { - const boot = await reloadSlotReady(existing, ctx.cfg.paths.project, { routing: routing() }) - if (!boot.ok) { - return { ok: false, status: 502, error: { code: 'worker_start_failed', message: boot.error } } + return withVmLock( + vmPath, + async (vmOperation) => { + const current = getVm(ctx.cfg.paths.project, vmId) + if (!current) return { ok: false, status: 404, error: { code: 'vm_not_found', message: 'VM not found' } } + Object.assign(existing, current) + const importedCredential = importedCredentialFromOauth(oauth, existing) + const workerExec = { + vmId, + vm: existing, + homeDir: path.join(ctx.cfg.paths.project, 'vms', vmId, 'cli-home'), } - const written = patchVmRecord(ctx.cfg.paths.project, vmId, { - runtime: boot.runtime, - status: 'running', - schedulable: true, - schedule_disabled_reason: null, - }) - if (written) Object.assign(existing, written) - else { - existing.runtime = boot.runtime - existing.status = 'running' - existing.schedulable = true - existing.schedule_disabled_reason = null + const bound = ctx.proxyPool?.getProxyForVm?.(vmId) + if (bound) { + // Re-bind even when the id already matches: a prior stale write may have + // left proxy_cli_enabled=false on a bound slot (#332). + bindVmProxy(ctx.cfg.paths.project, vmId, bound) + adoptVmProxyBinding(existing, getVm(ctx.cfg.paths.project, vmId)) } - workerExec.vm = existing - } - } - const importCred = - typeof ctx.importWorkerCredential === 'function' ? ctx.importWorkerCredential : importWorkerCredential - let workerImport = null - for (let attempt = 0; attempt < 30; attempt++) { - workerImport = await importCred(workerExec, importedCredential) - if (workerImport.ok) break - if (process.env.KIN_CRS_MOCK === '1') break - await new Promise((resolve) => setTimeout(resolve, 100)) - } - if (!workerImport.ok) { - return { - ok: false, - status: workerImport.status || 502, - error: { - type: 'worker_error', - code: 'worker_credential_import_failed', - message: workerImport.error?.message || 'Go slot worker rejected credential import', - }, - } - } - await withVmLock(vmPath, () => { - persistOauthToVm( - vmPath, - { - ...importedCredential, - source: source || oauth.source || 'sessionKey-cookie-auth', - mode: importedCredential.mode || credentialModeFromOauth(oauth), - type: importedCredential.type || importedCredential.mode, - }, - { acceptLiveGrant: true }, - ) - try { - const accountId = importedCredential.account_uuid || existing.claude?.account_uuid || vmId - ctx.accountQuota.clearGrantRevokeLeftover(accountId) - ctx.poolScheduler?.clearAuthCooldownFor?.(vmId, accountId) - ctx.poolScheduler?.clearGrantRevokeCooldownFor?.(vmId, accountId) - } catch {} - try { - rebindOauthAccount({ ...existing, ...JSON.parse(fs.readFileSync(vmPath, 'utf8')), id: vmId }) - } catch {} - const refreshed = JSON.parse(fs.readFileSync(vmPath, 'utf8')) - if (name && refreshed.name !== name) { - const written = patchVmRecord(ctx.cfg.paths.project, vmId, { name }) - Object.assign(existing, written || refreshed, { name }) - } else { - Object.assign(existing, refreshed) - } - }) - try { - writeWorkerCredentialFile(workerExec.homeDir, importedCredential) - } catch {} - try { - const ids = officialCcUidGid(vmId) - materializeOfficialClaudeCredentials(workerExec.homeDir, ids) - } catch {} - const mode = importedCredential.mode || credentialModeFromOauth(oauth) - const ccUnsupported = !slotHost(existing).supports('official_cc') - const skipCc = skipOfficialCc || ccUnsupported || isCodexVm(existing) || !canOfficialCc(mode) - const routingConfig = routing() - const officialCc = skipCc - ? { - scheduled: false, - reason: ccUnsupported - ? 'remote_unsupported' - : isCodexVm(existing) - ? 'gpt_slot' - : skipOfficialCc - ? 'credential_edit' - : 'credential_mode_unsupported', + if (process.env.KIN_CRS_MOCK !== '1') { + const socket = existing.runtime?.worker_socket + const needReload = !socket || !fs.existsSync(socket) || isSlotProxyDesynced(existing, ctx.cfg.paths.project) + if (needReload) { + const boot = await (ctx.reloadSlotReady || reloadSlotReady)(existing, ctx.cfg.paths.project, { + routing: routing(), + }) + if (!boot.ok) { + if (vmOperation && boot.vmOperationNotStarted !== true) await vmOperation.markUncertain() + await flushVmDbSync() + return { ok: false, status: 502, error: { code: 'worker_start_failed', message: boot.error } } + } + const written = patchVmRecord(ctx.cfg.paths.project, vmId, { + runtime: boot.runtime, + status: 'running', + schedulable: true, + schedule_disabled_reason: null, + }) + if (written) Object.assign(existing, written) + else { + existing.runtime = boot.runtime + existing.status = 'running' + existing.schedulable = true + existing.schedule_disabled_reason = null + } + workerExec.vm = existing + } } - : scheduleOfficialCcBootstrap({ - vmId, - projectRoot: ctx.cfg.paths.project, - force: true, - collectIdentity: collectSlotIdentity, - onStats: officialCcStatsHandler(vmId, existing.claude?.account_uuid), - routingFile: ctx.routingConfigPath, - config: routingConfig.official_cc, - credentialMode: mode, - incomingOauth: { - account_uuid: importedCredential.account_uuid, - email: importedCredential.email, + const importCred = + typeof ctx.importWorkerCredential === 'function' ? ctx.importWorkerCredential : importWorkerCredential + let workerImport = null + for (let attempt = 0; attempt < 30; attempt++) { + workerImport = await importCred(workerExec, importedCredential) + if (workerImport.ok) break + if (vmOperation || process.env.KIN_CRS_MOCK === '1') break + await new Promise((resolve) => setTimeout(resolve, 100)) + } + if (!workerImport.ok) { + if (vmOperation && workerImport.vmOperationNotStarted !== true) await vmOperation.markUncertain() + await flushVmDbSync() + return { + ok: false, + status: workerImport.status || 502, + error: { + type: 'worker_error', + code: 'worker_credential_import_failed', + message: workerImport.error?.message || 'Go slot worker rejected credential import', + }, + } + } + persistOauthToVm( + vmPath, + { + ...importedCredential, + source: source || oauth.source || 'sessionKey-cookie-auth', + mode: importedCredential.mode || credentialModeFromOauth(oauth), + type: importedCredential.type || importedCredential.mode, }, - }) - return { ok: true, existing, official_cc_bootstrap: officialCc } + { acceptLiveGrant: true }, + ) + try { + const accountId = importedCredential.account_uuid || existing.claude?.account_uuid || vmId + ctx.accountQuota.clearGrantRevokeLeftover(accountId) + ctx.poolScheduler?.clearAuthCooldownFor?.(vmId, accountId) + ctx.poolScheduler?.clearGrantRevokeCooldownFor?.(vmId, accountId) + } catch {} + try { + rebindOauthAccount({ ...existing, ...JSON.parse(fs.readFileSync(vmPath, 'utf8')), id: vmId }) + } catch {} + const refreshed = JSON.parse(fs.readFileSync(vmPath, 'utf8')) + if (name && refreshed.name !== name) { + const written = patchVmRecord(ctx.cfg.paths.project, vmId, { name }) + Object.assign(existing, written || refreshed, { name }) + } else { + Object.assign(existing, refreshed) + } + try { + writeWorkerCredentialFile(workerExec.homeDir, importedCredential) + } catch (error) { + if (vmOperation) throw error + } + try { + const ids = officialCcUidGid(vmId) + materializeOfficialClaudeCredentials(workerExec.homeDir, ids) + } catch (error) { + if (vmOperation) throw error + } + const mode = importedCredential.mode || credentialModeFromOauth(oauth) + const ccUnsupported = !slotHost(existing).supports('official_cc') + const routingConfig = routing() || {} + const ccConfig = normalizeOfficialCcConfig( + routingConfig.official_cc || loadOfficialCcConfig(ctx.routingConfigPath), + ) + const ccMock = process.env.KIN_CRS_MOCK === '1' + const skipCc = + skipOfficialCc || + ccUnsupported || + isCodexVm(existing) || + !canOfficialCc(mode) || + ccMock || + ccConfig.enabled === false + let officialCc = skipCc + ? { + scheduled: false, + reason: ccUnsupported + ? 'remote_unsupported' + : isCodexVm(existing) + ? 'gpt_slot' + : skipOfficialCc + ? 'credential_edit' + : !canOfficialCc(mode) + ? 'credential_mode_unsupported' + : ccMock + ? 'mock' + : 'disabled', + } + : null + if (!skipCc) { + const bootstrapOptions = { + vmId, + projectRoot: ctx.cfg.paths.project, + force: true, + collectIdentity: collectSlotIdentity, + onStats: officialCcStatsHandler(vmId, existing.claude?.account_uuid), + routingFile: ctx.routingConfigPath, + config: ccConfig, + credentialMode: mode, + incomingOauth: { account_uuid: importedCredential.account_uuid, email: importedCredential.email }, + } + if (vmOperation) { + const completed = await (ctx.runOfficialCcBootstrap || runOfficialCcBootstrap)(bootstrapOptions) + if (!completed?.ok && completed?.vmOperationNotStarted !== true) await vmOperation.markUncertain() + officialCc = { scheduled: true, vmId, completed: true, ok: completed?.ok === true } + } else { + officialCc = scheduleOfficialCcBootstrap(bootstrapOptions) + } + } + await ctx.accountQuota?.flush?.() + await flushVmDbSync() + return { ok: true, existing, official_cc_bootstrap: officialCc } + }, + { vmId, kind: 'credential-import' }, + ) } return { From b26b2b6282230cb190518c149b1d8426fbd1cdf8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:14:05 +0800 Subject: [PATCH 150/473] test(codex): reproduce stale shared routing admission --- test/unit/shared-codex-execution.test.mjs | 78 ++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/test/unit/shared-codex-execution.test.mjs b/test/unit/shared-codex-execution.test.mjs index c18b3908e..3d71b6937 100644 --- a/test/unit/shared-codex-execution.test.mjs +++ b/test/unit/shared-codex-execution.test.mjs @@ -8,9 +8,10 @@ import { EventEmitter } from 'node:events' import { setTimeout as delay } from 'node:timers/promises' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' -import { handleCodexProtocol } from '../../src/lib/protocol/handle-codex.mjs' +import { handleCodexProtocol, pickCodexCandidates } from '../../src/lib/protocol/handle-codex.mjs' import { resetOpenAIAccountRuntime } from '../../src/lib/pool/openai-account-runtime.mjs' import { SharedCodexRuntime } from '../../src/lib/pool/shared-codex-runtime.mjs' +import { SharedRoutingConfig } from '../../src/lib/admin/shared-routing-config.mjs' const unixTest = process.platform === 'win32' ? test.skip : test const accountId = 'codex:synthetic-shared-account' @@ -172,6 +173,81 @@ async function fixture( } } +async function sharedRouting(fx, quota) { + const file = path.join(fx.root, 'synthetic-routing.json') + fs.writeFileSync(file, JSON.stringify({ codex: { quota } })) + const state = new SharedRoutingConfig({ + db: fx.db, + file, + normalize: (value) => value, + merge: (current, patch) => ({ ...current, ...patch }), + apply: async () => {}, + }) + await state.init() + return state +} + +unixTest('shared Codex candidates retain the policy revision and inherited VM limits they read', async (t) => { + const fx = await fixture(t, undefined, { concurrency: 9 }) + const file = path.join(fx.root, 'vms', `${vmId}.json`) + const vm = JSON.parse(fs.readFileSync(file, 'utf8')) + vm.policy = { maxConcurrency: 9, concurrencyOverride: false, maxRpm: 999, rpmOverride: false } + fs.writeFileSync(file, JSON.stringify(vm)) + const state = await sharedRouting(fx, { max_concurrency: 1, max_rpm: 2 }) + const runtime = new SharedCodexRuntime(fx.db) + const options = { routing: { codex: { quota: { max_concurrency: 9, max_rpm: 999 } } }, sharedRuntime: runtime } + const req = { headers: {}, apiKeyKind: 'user' } + const picked = await pickCodexCandidates(fx.root, req, options) + assert.equal(picked.candidates[0].concurrency, 1) + assert.equal(picked.candidates[0].maxRpm, 2) + assert.equal(picked.candidates[0].routingRevision, 1) + assert.equal(picked.quotaPolicy.max_rpm, 2) + await state.patch({ codex: { quota: { max_concurrency: 2, max_rpm: 3 } } }) + await assert.rejects(runtime.tryReserve(fx.root, vm, picked.candidates[0], picked.quotaPolicy), { + code: 'routing_config_stale', + }) + assert.equal((await fx.counts()).vmInflight, 0) + const fresh = await pickCodexCandidates(fx.root, req, options) + assert.equal(fresh.candidates[0].concurrency, 2) + assert.equal(fresh.candidates[0].maxRpm, 3) + assert.equal(fresh.candidates[0].routingRevision, 2) + const lease = await runtime.tryReserve(fx.root, vm, fresh.candidates[0], fresh.quotaPolicy) + assert.ok(lease) + await lease.release() +}) + +unixTest('shared Codex reselects after a routing race before sending any kernel job', async (t) => { + const fx = await fixture(t, undefined, { concurrency: 2 }) + const state = await sharedRouting(fx, { max_concurrency: 2 }) + const runtime = new SharedCodexRuntime(fx.db) + const held = await new SharedExecutionAdmissions(fx.other).tryReserve({ + accountId, + vmId: 'vm-shared-other', + accountLimit: 2, + vmLimit: 2, + fableLimit: 0, + rpmLimit: 0, + }) + const reserve = runtime.tryReserve.bind(runtime) + let changed = false + runtime.tryReserve = async (...args) => { + if (!changed) { + changed = true + await state.patch({ codex: { quota: { max_concurrency: 1 } } }) + } + return reserve(...args) + } + try { + const response = await fx.run({ sharedRuntime: runtime }) + assert.equal(changed, true) + assert.equal(response.status, 429) + assert.equal(fx.messages.length, 0, 'the old candidate cannot authorize an execution under the new revision') + assert.equal((await fx.counts()).accountInflight, 1) + } finally { + await held.release() + } +}) + unixTest('Kubernetes Codex persists its actual kernel job before submission and releases only after EOF', async (t) => { const fx = await fixture(t) const response = await fx.run() From f1acb491e29cfa3bae788c54637b1efa39ff0e9f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:15:37 +0800 Subject: [PATCH 151/473] fix(codex): validate captured routing policy during shared admission --- src/lib/pool/shared-codex-runtime.mjs | 14 ++++++- src/lib/protocol/handle-codex.mjs | 53 ++++++++++++++++++++------- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/lib/pool/shared-codex-runtime.mjs b/src/lib/pool/shared-codex-runtime.mjs index 6268b1f73..ce8b940e2 100644 --- a/src/lib/pool/shared-codex-runtime.mjs +++ b/src/lib/pool/shared-codex-runtime.mjs @@ -3,6 +3,7 @@ import { SharedExecutionAdmissions } from './shared-execution-admissions.mjs' import { SharedSessionTurns } from './shared-session-turns.mjs' import { readCodexAccounts } from '../vm/codex-slot.mjs' import { normalizeOpenAIQuotaPolicy } from './openai-quota-policy.mjs' +import { getSharedRoutingState } from '../admin/shared-routing-config.mjs' function identityError() { return Object.assign(new Error('Shared Codex execution requires exactly one account with chatgpt_account_id'), { @@ -14,10 +15,20 @@ function identityError() { export class SharedCodexRuntime { constructor(db = getDb(), options = {}) { this.db = db - this.admissions = new SharedExecutionAdmissions(db, options) + this.routingState = options.routingState ?? getSharedRoutingState(db) + this.admissions = new SharedExecutionAdmissions(db, { + ...options, + assertRoutingRevision: this.routingState + ? (revision) => this.routingState.assertRevision(revision) + : options.assertRoutingRevision, + }) this.turns = new SharedSessionTurns(db, options) } + async routingSnapshot(routing) { + return this.routingState ? this.routingState.refresh() : { config: routing } + } + accountId(projectRoot, vm) { const accounts = readCodexAccounts(projectRoot, vm.id) const id = accounts.length === 1 ? String(accounts[0]?.chatgpt_account_id || '').trim() : '' @@ -57,6 +68,7 @@ export class SharedCodexRuntime { rpmLimit: limits.max_rpm, fableLimit: 0, family: 'codex', + routingRevision: candidate.routingRevision, }) if (!gate.ok) return null let releasing = null diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index 808a7b4af..564120a04 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -214,7 +214,20 @@ export async function pickCodexCandidates( ) { const pin = pinnedVmId(req) const model = body?.model || null - const routingPolicy = normalizeCodexRouting(routing.codex || routing).quota + const snapshot = sharedRuntime ? await sharedRuntime.routingSnapshot(routing) : null + const liveRouting = snapshot?.config || routing + const routingPolicy = normalizeCodexRouting(liveRouting.codex || liveRouting).quota + const withPolicy = (ordered) => + sharedRuntime + ? { + ...ordered, + quotaPolicy: routingPolicy, + candidates: (ordered.candidates || []).map((candidate) => ({ + ...candidate, + routingRevision: snapshot.revision, + })), + } + : ordered const scope = await keyScopeFromRequest(req) const poolDenial = vmPoolDenial(scope) if (poolDenial) return { error: 'vm_pool_unavailable', message: poolDenial.message, ids: [], candidates: [] } @@ -233,7 +246,7 @@ export async function pickCodexCandidates( quotaPolicy: routingPolicy, runtimeSignals, }) - return { ...ordered, pin, sticky: false, sessionKey: null, stickyKeys: [] } + return { ...withPolicy(ordered), pin, sticky: false, sessionKey: null, stickyKeys: [] } } for (const item of listVms(projectRoot, { codex: { quota: routingPolicy } })) { if (!isCodexVm(item)) continue @@ -286,7 +299,7 @@ export async function pickCodexCandidates( if (continuesResponse && (boundLeft || ordered.error === 'candidates_exhausted')) { return { error: 'response_not_portable', ids: [], sessionKey, stickyKeys, boundVmId: bound.vmId } } - return { ...ordered, sessionKey, stickyKeys, ...(stickyRouter?.shared ? { stickyVersions } : {}) } + return { ...withPolicy(ordered), sessionKey, stickyKeys, ...(stickyRouter?.shared ? { stickyVersions } : {}) } } function ingestCodexHop(projectRoot, vmId, result, now = Date.now(), policy = null) { @@ -426,18 +439,26 @@ function clientGoneSignal(req, res) { */ async function admitCodexCandidate(projectRoot, req, opts, { deadline, signal }) { let woken = false - for (;;) { + selection: for (;;) { const picked = await pickCodexCandidates(projectRoot, req, opts) - const livePolicy = normalizeCodexRouting(opts.routing?.codex || opts.routing).quota + const livePolicy = picked.quotaPolicy || normalizeCodexRouting(opts.routing?.codex || opts.routing).quota if (picked.error && picked.error !== 'capacity_unavailable') return { picked } for (const candidate of picked.candidates || []) { - const lease = opts.sharedRuntime - ? await opts.sharedRuntime.tryReserve(projectRoot, getVm(projectRoot, candidate.id), candidate, livePolicy) - : tryAcquireOpenAISlot(candidate.id, { - concurrency: candidate.concurrency, - maxRpm: candidate.maxRpm, - quotaPolicy: livePolicy, - }) + let lease + try { + lease = opts.sharedRuntime + ? await opts.sharedRuntime.tryReserve(projectRoot, getVm(projectRoot, candidate.id), candidate, livePolicy) + : tryAcquireOpenAISlot(candidate.id, { + concurrency: candidate.concurrency, + maxRpm: candidate.maxRpm, + quotaPolicy: livePolicy, + }) + } catch (error) { + if (!opts.sharedRuntime || error?.code !== 'routing_config_stale') throw error + if (signal?.aborted) return { picked: { error: 'client_cancelled', ids: [] } } + if (Date.now() >= deadline) return { picked: { error: 'capacity_unavailable', ids: [] } } + continue selection + } if (lease) return { picked, vmId: candidate.id, lease } } // A woken waiter that still missed hands the wake to the next in line. @@ -937,7 +958,13 @@ export async function handleCodexProtocol({ }, }) } - ingestCodexHop(projectRoot, vm.id, result, Date.now(), normalizeCodexRouting(routing.codex).quota) + ingestCodexHop( + projectRoot, + vm.id, + result, + Date.now(), + picked.quotaPolicy || normalizeCodexRouting(routing.codex).quota, + ) if (result?.transport_retried) logBag.transport_retried = true if (result?.reasoning_history_retried) { logBag.attempt_count += 1 From 6732a8648a4a57c6483abc7f685fce4d8cb28946 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:18:02 +0800 Subject: [PATCH 152/473] feat: synchronize versioned routing configuration through PostgreSQL --- src/lib/admin/routing-runtime.mjs | 146 ++++++++++++++++++- src/lib/admin/shared-routing-config.mjs | 96 ++++++++++++ src/lib/cluster/placement.mjs | 4 +- src/lib/core/config.mjs | 2 + src/lib/core/routing-document.mjs | 38 +++++ src/lib/db/repos/routing-config-repo.mjs | 73 ++++++++++ src/lib/identity/crs-persona.mjs | 12 +- src/lib/oauth/official-cc-bootstrap.mjs | 3 +- src/lib/protocol/cache-ttl.mjs | 6 +- src/lib/transport/rust-kernel-supervisor.mjs | 5 +- src/server.mjs | 15 +- test/unit/shared-routing-config.test.mjs | 128 +++++++++++++++- 12 files changed, 508 insertions(+), 20 deletions(-) create mode 100644 src/lib/admin/shared-routing-config.mjs create mode 100644 src/lib/core/routing-document.mjs create mode 100644 src/lib/db/repos/routing-config-repo.mjs diff --git a/src/lib/admin/routing-runtime.mjs b/src/lib/admin/routing-runtime.mjs index f5422ad1a..ac0a0b9d5 100644 --- a/src/lib/admin/routing-runtime.mjs +++ b/src/lib/admin/routing-runtime.mjs @@ -54,8 +54,11 @@ import { normalizeVmConfig } from '../vm/machine-spec.mjs' import { syncWorkerTelemetry } from '../vm/vm-runtime.mjs' import { slotHost } from '../vm/slot-host.mjs' import { telemetryUploadInterval } from '../vm/worker-telemetry.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { SharedRoutingConfig } from './shared-routing-config.mjs' export function createRoutingRuntime(ctx) { + const shared = ctx.shared ?? isKubernetesMode() const getRouting = () => (typeof ctx.getRoutingConfig === 'function' ? ctx.getRoutingConfig() : ctx.routingConfig) const setRouting = (next) => { if (typeof ctx.setRoutingConfig === 'function') ctx.setRoutingConfig(next) @@ -82,6 +85,134 @@ export function createRoutingRuntime(ctx) { else ctx.failoverRunner = next } + function mergeSharedRouting(current, body = {}) { + if (body?.codex && Object.hasOwn(body.codex, 'quota')) validateOpenAIQuotaPatch(body.codex.quota) + validateInferenceWebsocketPatch(body) + const next = structuredClone(current) + for (const key of [ + 'sticky', + 'quota', + 'concurrency', + 'pool', + 'failover', + 'inference', + 'compatibility', + 'websocket', + 'codex', + 'vm', + 'telemetry', + 'official_cc', + 'usage_probe', + 'logging', + ]) { + if (body[key]) next[key] = { ...(next[key] || {}), ...body[key] } + } + if (body.codex?.quota) next.codex.quota = mergeOpenAIQuotaPolicy(current.codex?.quota, body.codex.quota) + if (body.vm?.smbios) next.vm.smbios = { ...(current.vm?.smbios || {}), ...body.vm.smbios } + if (body.health_probe) + next.health_probe = { + ...(current.health_probe || {}), + ...body.health_probe, + real: { ...(current.health_probe?.real || {}), ...(body.health_probe.real || {}) }, + match: { ...(current.health_probe?.match || {}), ...(body.health_probe.match || {}) }, + } + if (body.notify) next.notify = mergeNotifyConfig(current.notify, body.notify) + if (next.compatibility) { + delete next.compatibility.persona_rules + delete next.compatibility.persona_leak_append + next.compatibility.cch_enabled = next.compatibility.cch_enabled !== false + } + next.websocket = normalizeInferenceWebsocket(next.websocket) + next.codex = normalizeCodexRouting(next.codex) + next.pool = normalizePoolRouting(next.pool) + next.vm = normalizeVmConfig(next.vm) + next.inference = normalizeInferenceConfig(next.inference) + next.official_cc = normalizeOfficialCcConfig(next.official_cc) + next.health_probe = normalizeHealthProbeConfig(next.health_probe) + next.usage_probe = normalizeUsageProbeConfig(next.usage_probe) + next.notify = mergeNotifyConfig(next.notify) + next.logging = normalizeLoggingConfig(next.logging) + next.telemetry = { ...(next.telemetry || {}), upload_interval_seconds: telemetryUploadInterval(next) } + next.tiers = normalizeTiers( + body.tiers ? mergeTierMaps(current.tiers, body.tiers) : next.tiers, + next.quota, + next.concurrency, + ) + const defaults = next.tiers.default || {} + next.concurrency = { + ...(next.concurrency || {}), + default_max_per_account: Number(defaults.max_concurrency ?? 2), + default_key_concurrency: Number(defaults.max_concurrency ?? next.concurrency?.default_key_concurrency ?? 2), + default_max_rpm: Number(defaults.max_rpm ?? 0), + } + next.quota = { + ...(next.quota || {}), + safety_ratio: Number(defaults.limit_5h ?? defaults.safety_ratio ?? 0.85), + weekly_safety_ratio: Number(defaults.limit_7d ?? defaults.weekly_safety_ratio ?? 0.8), + warn_ratio: Number(defaults.warn_ratio ?? 0.75), + } + return next + } + + async function applySharedRouting(config, revision) { + setRouting(config) + setManualScheduleWins(config.pool?.manual_schedule_wins) + ctx.stickyRouter?.reloadConfig(config) + ctx.accountQuota?.reloadConfig(config) + getPool()?.reloadConfig?.(poolSchedulerConfig()) + ctx.failoverRunner?.reloadConfig?.(config.failover || {}) + getPool()?.rateLimitService?.setConfig?.(config.rate_limit || {}) + configureUnitCircuit(config) + const pending = [] + for (const [name, callback] of [ + ['health_monitor', () => getHealth()?.setConfig(config.health_probe)], + ['usage_monitor', () => getUsage()?.setConfig(config.usage_probe)], + ['notify_monitor', () => getNotify()?.setConfig(config.notify)], + [ + 'logging', + () => + ctx.requestLog?.setConfig({ + mode: config.logging.mode, + retainDays: config.logging.retain_days, + debugRetainDays: config.logging.debug_retain_days, + maxMb: config.logging.max_mb, + mutedErrorClasses: config.logging.muted_error_classes, + }), + ], + ]) { + try { + await callback() + } catch (error) { + pending.push({ component: name, error: error.code || error.message }) + } + } + // Desired state is committed. Remote/kernel and VM-file effects require + // their own durable VM operation and must not run in the PG transaction. + for (const vm of listVms(ctx.cfg.paths.project)) { + pending.push({ component: 'vm_projection', vm_id: vm.id, revision }) + } + return { status: pending.length ? 'pending' : 'applied', pending } + } + + const routingState = shared + ? new SharedRoutingConfig({ + db: ctx.db, + file: ctx.routingConfigPath, + normalize: (config) => mergeSharedRouting(config), + merge: mergeSharedRouting, + apply: applySharedRouting, + }) + : null + + async function initRoutingConfig() { + if (routingState) return routingState.init() + return loadRoutingConfig() + } + + async function refreshRouting() { + return routingState ? routingState.refresh() : null + } + function applyVmConcurrency(id, n, { override = true } = {}) { const vmPath = path.join(ctx.cfg.paths.project, 'vms', `${id}.json`) if (!fs.existsSync(vmPath)) return null @@ -429,7 +560,8 @@ export function createRoutingRuntime(ctx) { return applied } - async function persistRoutingPatch(body = {}) { + async function persistRoutingPatch(body = {}, options = {}) { + if (routingState) return routingState.patch(body, options) const hasCodexQuota = body?.codex && Object.prototype.hasOwnProperty.call(body.codex, 'quota') if (hasCodexQuota) validateOpenAIQuotaPatch(body.codex.quota) validateInferenceWebsocketPatch(body) @@ -573,6 +705,13 @@ export function createRoutingRuntime(ctx) { } function loadRoutingConfig() { + if (routingState) { + if (!routingState.currentRevision) + throw Object.assign(new Error('shared routing config needs async initialization'), { + code: 'routing_config_unavailable', + }) + return structuredClone(routingState.config) + } let raw try { raw = fs.readFileSync(ctx.routingConfigPath, 'utf8') @@ -600,6 +739,7 @@ export function createRoutingRuntime(ctx) { } function migrateLegacyOpenAIPolicies(doc) { + if (routingState) return { status: 'pending', revision: routingState.currentRevision } const quota = normalizeOpenAIQuotaPolicy(doc.codex?.quota) for (const summary of listVms(ctx.cfg.paths.project)) { if (!isCodexVm(summary)) continue @@ -697,6 +837,7 @@ export function createRoutingRuntime(ctx) { stickyRouter: ctx.stickyRouter, accountQuota: ctx.accountQuota, runtimeRepo, + routingState, workerHealth: rustKernelHealth, config: poolSchedulerConfig(), }) @@ -774,6 +915,9 @@ export function createRoutingRuntime(ctx) { } return { + routingState, + initRoutingConfig, + refreshRouting, persistRoutingPatch, applyOfficialCcRoutingBody, applyHealthProbeRoutingBody, diff --git a/src/lib/admin/shared-routing-config.mjs b/src/lib/admin/shared-routing-config.mjs new file mode 100644 index 000000000..c29afff74 --- /dev/null +++ b/src/lib/admin/shared-routing-config.mjs @@ -0,0 +1,96 @@ +import fs from 'node:fs' +import { getDb } from '../db/database.mjs' +import { RoutingConfigRepo } from '../db/repos/routing-config-repo.mjs' +import { manageRoutingDocument, clearRoutingDocument, publishRoutingDocument } from '../core/routing-document.mjs' + +const states = new WeakMap() +export function getSharedRoutingState(db = getDb()) { + return states.get(db) || null +} + +export class SharedRoutingConfig { + constructor({ db = getDb(), file, normalize, merge, apply }) { + this.db = db + this.repo = new RoutingConfigRepo(db) + this.file = file + this.normalize = normalize + this.merge = merge + this.apply = apply + this.currentRevision = 0 + this.config = null + this.application = null + this.applying = Promise.resolve() + manageRoutingDocument(file) + } + + async init() { + const value = await this.repo.seed(() => this.normalize(JSON.parse(fs.readFileSync(this.file, 'utf8')))) + await this.accept(value) + states.set(this.db, this) + return structuredClone(this.config) + } + + accept(snapshot) { + const operation = this.applying.then(async () => { + if (!snapshot || snapshot.revision <= this.currentRevision) return this.application + const application = await this.apply(structuredClone(snapshot.config), snapshot.revision) + this.config = structuredClone(snapshot.config) + this.currentRevision = snapshot.revision + this.application = application + publishRoutingDocument(this.file, snapshot) + return application + }) + this.applying = operation.catch(() => {}) + return operation + } + + async refresh() { + const value = await this.repo.get() + if (!value) + throw Object.assign(new Error('shared routing configuration missing'), { code: 'routing_config_unavailable' }) + await this.accept(value) + return { revision: this.currentRevision, config: structuredClone(this.config), application: this.application } + } + + async patch(body, options) { + const value = await this.repo.patch((current) => this.merge(current, body), options) + await this.accept(value) + return { revision: value.revision, application: this.application } + } + + assertRevision(revision) { + return this.repo.assertRevision(revision) + } + + start() { + if (this.timer) return + this.timer = setInterval(() => { + if (this.refreshing) return + this.refreshing = this.refresh() + .catch((error) => { + this.lastError = error.code || error.message + }) + .finally(() => { + this.refreshing = null + }) + }, 1000) + this.timer.unref?.() + } + + async stop() { + clearInterval(this.timer) + this.timer = null + await this.refreshing + await this.applying + } + + async rebind(db) { + await this.applying + states.delete(this.db) + this.db = db + this.repo = new RoutingConfigRepo(db) + this.currentRevision = 0 + clearRoutingDocument(this.file) + return this.init() + } +} diff --git a/src/lib/cluster/placement.mjs b/src/lib/cluster/placement.mjs index 37dbdf3ab..4466729e8 100644 --- a/src/lib/cluster/placement.mjs +++ b/src/lib/cluster/placement.mjs @@ -1,3 +1,4 @@ +import { readRoutingDocument } from '../core/routing-document.mjs' /** * VM placement on cluster nodes. A VM with `node_id` runs its slot container on * that node; everything else about it (vm json, scheduler, kernel transport) @@ -13,7 +14,6 @@ * the bridge is served by this same event loop. Remote lifecycle uses the Engine API. */ -import fs from 'node:fs' import { Transform } from 'node:stream' import { slotRuntimeOwner } from '../oauth/oauth-credentials.mjs' import { routingConfigFile } from '../core/config.mjs' @@ -144,7 +144,7 @@ function placementRouting() { const root = bound?.projectRoot if (!root) return null try { - return JSON.parse(fs.readFileSync(routingConfigFile(root), 'utf8')) + return readRoutingDocument(routingConfigFile(root)) } catch { return null } diff --git a/src/lib/core/config.mjs b/src/lib/core/config.mjs index 392fc2580..dff6516af 100644 --- a/src/lib/core/config.mjs +++ b/src/lib/core/config.mjs @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url' import { hashKey } from './security.mjs' import { atomicWriteJson } from '../vm/vm-file.mjs' import { loadDistillRules } from './distill-detect.mjs' +import { readRoutingDocument, routingDocumentRevision } from './routing-document.mjs' const __dirname = path.dirname(fileURLToPath(import.meta.url)) // this file lives at src/lib/core/ — the gateway root is src/ @@ -93,6 +94,7 @@ export function routingConfigFile(projectRoot = PROJECT) { export function readRoutingConfigFile(projectRoot = PROJECT) { const file = routingConfigFile(projectRoot) + if (routingDocumentRevision(file) !== null) return readRoutingDocument(file) let raw try { raw = fs.readFileSync(file, 'utf8') diff --git a/src/lib/core/routing-document.mjs b/src/lib/core/routing-document.mjs new file mode 100644 index 000000000..53bc8d2ed --- /dev/null +++ b/src/lib/core/routing-document.mjs @@ -0,0 +1,38 @@ +import fs from 'node:fs' +import path from 'node:path' + +const managed = new Set() +const snapshots = new Map() + +export function manageRoutingDocument(file) { + managed.add(path.resolve(file)) +} + +export function clearRoutingDocument(file) { + snapshots.delete(path.resolve(file)) +} + +export function routingDocumentRevision(file) { + const key = path.resolve(file) + return managed.has(key) ? snapshots.get(key)?.revision || 0 : null +} + +export function publishRoutingDocument(file, snapshot) { + const key = path.resolve(file) + if ((snapshots.get(key)?.revision || 0) > snapshot.revision) return + snapshots.set(key, structuredClone(snapshot)) +} + +/** Standalone keeps file semantics; shared files are read-only bootstrap inputs. */ +export function readRoutingDocument(file) { + const key = path.resolve(file) + if (managed.has(key)) { + const snapshot = snapshots.get(key) + if (!snapshot) + throw Object.assign(new Error('shared routing configuration is not initialized'), { + code: 'routing_config_unavailable', + }) + return structuredClone(snapshot.config) + } + return JSON.parse(fs.readFileSync(file, 'utf8')) +} diff --git a/src/lib/db/repos/routing-config-repo.mjs b/src/lib/db/repos/routing-config-repo.mjs new file mode 100644 index 000000000..3ea439b0e --- /dev/null +++ b/src/lib/db/repos/routing-config-repo.mjs @@ -0,0 +1,73 @@ +import { getDb, withTransaction } from '../database.mjs' + +const KEY = 'routing_config' + +function stale() { + return Object.assign(new Error('routing configuration revision changed'), { + code: 'routing_config_stale', + status: 409, + }) +} + +function decode(row) { + if (!row) return null + const value = JSON.parse(row.value) + if ( + !Number.isSafeInteger(value.revision) || + value.revision < 1 || + !value.config || + typeof value.config !== 'object' + ) { + throw new Error('invalid shared routing configuration') + } + return value +} + +export class RoutingConfigRepo { + constructor(db = getDb()) { + this.db = db + } + + async get() { + return decode(await this.db.prepare('SELECT value FROM settings WHERE key = $1').get(KEY)) + } + + async seed(load) { + const present = await this.get() + if (present) return present + const config = load() + return withTransaction(this.db, async () => { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [this.db.schema, KEY]) + const existing = await this.get() + if (existing) return existing + const value = { revision: 1, config } + await this.db + .prepare('INSERT INTO settings (key, value, updated_at) VALUES ($1,$2,$3)') + .run(KEY, JSON.stringify(value), new Date().toISOString()) + return value + }) + } + + async patch(change, { ifRevision } = {}) { + return withTransaction(this.db, async () => { + const current = decode(await this.db.prepare('SELECT value FROM settings WHERE key = $1 FOR UPDATE').get(KEY)) + if (!current || (ifRevision != null && ifRevision !== current.revision)) throw stale() + const revision = current.revision + 1 + if (!Number.isSafeInteger(revision)) throw new Error('routing revision overflow') + const next = { revision, config: change(structuredClone(current.config)) } + await this.db + .prepare('UPDATE settings SET value = $2, updated_at = $3 WHERE key = $1') + .run(KEY, JSON.stringify(next), new Date().toISOString()) + return next + }) + } + + /** Call inside admission's transaction, before resource locks. */ + async assertRevision(revision) { + return withTransaction(this.db, async () => { + const current = decode(await this.db.prepare('SELECT value FROM settings WHERE key = $1 FOR SHARE').get(KEY)) + if (!current || current.revision !== revision) throw stale() + return true + }) + } +} diff --git a/src/lib/identity/crs-persona.mjs b/src/lib/identity/crs-persona.mjs index 7f7dceb60..5e947d084 100644 --- a/src/lib/identity/crs-persona.mjs +++ b/src/lib/identity/crs-persona.mjs @@ -1,3 +1,4 @@ +import { readRoutingDocument } from '../core/routing-document.mjs' /** * Unofficial OAuth outbound persona. Official Claude Code inbound is * never rewritten. Mode comes from routing.compatibility.persona_inject: @@ -31,7 +32,6 @@ */ import { createHash } from 'node:crypto' import { resolveWorkstationProfile } from './workstation-profile.mjs' -import fs from 'node:fs' import { uuidFromSeed } from './identity-rewrite.mjs' import { OFFICIAL_CLI_VERSION } from './vm-identity.mjs' import { CCH_PLACEHOLDER } from './cch.mjs' @@ -1027,7 +1027,7 @@ export function personaExpansionToolsFromRouting(routing = {}) { export function personaExpansionFromRoutingFile(filePath) { if (!filePath) return { mode: DEFAULT_PERSONA_EXPANSION, extraNames: [] } try { - const routing = JSON.parse(fs.readFileSync(filePath, 'utf8')) + const routing = readRoutingDocument(filePath) return { mode: personaExpansionFromRouting(routing), extraNames: personaExpansionToolsFromRouting(routing), @@ -1047,7 +1047,7 @@ export const PERSONA_STANDING_MAX = 800 export function standingConstraintFromRoutingFile(filePath) { if (filePath) { try { - const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'))?.compatibility?.persona_standing + const raw = readRoutingDocument(filePath)?.compatibility?.persona_standing const text = String(raw ?? '').trim() if (text) return text.slice(0, PERSONA_STANDING_MAX) } catch { @@ -1250,7 +1250,7 @@ export function personaOptionsFromRoutingFile(filePath) { return { mode: DEFAULT_PERSONA_MODE, park: DEFAULT_PERSONA_PARK, agent: DEFAULT_PERSONA_AGENT } } try { - return personaOptionsFromRouting(JSON.parse(fs.readFileSync(filePath, 'utf8'))) + return personaOptionsFromRouting(readRoutingDocument(filePath)) } catch { return { mode: DEFAULT_PERSONA_MODE, park: DEFAULT_PERSONA_PARK, agent: DEFAULT_PERSONA_AGENT } } @@ -1311,7 +1311,7 @@ export function personaTemplateSettingsFromRouting(routing = {}) { export function personaTemplateSettingsFromRoutingFile(filePath) { if (!filePath) return personaTemplateSettingsFallback() try { - return personaTemplateSettingsFromRouting(JSON.parse(fs.readFileSync(filePath, 'utf8'))) + return personaTemplateSettingsFromRouting(readRoutingDocument(filePath)) } catch { return personaTemplateSettingsFallback() } @@ -1534,7 +1534,7 @@ export function detectProxiedOfficialCcFromRouting(routing = {}) { export function detectProxiedOfficialCcFromRoutingFile(filePath) { if (!filePath) return true try { - return detectProxiedOfficialCcFromRouting(JSON.parse(fs.readFileSync(filePath, 'utf8'))) + return detectProxiedOfficialCcFromRouting(readRoutingDocument(filePath)) } catch { return true } diff --git a/src/lib/oauth/official-cc-bootstrap.mjs b/src/lib/oauth/official-cc-bootstrap.mjs index 0ba87af37..74aca43c3 100644 --- a/src/lib/oauth/official-cc-bootstrap.mjs +++ b/src/lib/oauth/official-cc-bootstrap.mjs @@ -1,3 +1,4 @@ +import { readRoutingDocument } from '../core/routing-document.mjs' /** * Required post-OAuth workflow: wipe first-use Claude Code files, start the * slot's cli-node (the same Claude Code build the kernel runs, no separate @@ -106,7 +107,7 @@ export function normalizeOfficialCcConfig(raw = {}) { function loadRoutingDocument(routingFile) { if (!routingFile) return {} try { - return JSON.parse(fs.readFileSync(routingFile, 'utf8')) + return readRoutingDocument(routingFile) } catch { return {} } diff --git a/src/lib/protocol/cache-ttl.mjs b/src/lib/protocol/cache-ttl.mjs index 7d2181879..b9a564479 100644 --- a/src/lib/protocol/cache-ttl.mjs +++ b/src/lib/protocol/cache-ttl.mjs @@ -1,9 +1,9 @@ +import { readRoutingDocument } from '../core/routing-document.mjs' /** * Anthropic cache_control ttl for unofficial OAuth. * Default 1h (2× input). Customers may request 5m (1.25× input); * billing must use the TTL we actually sent. */ -import fs from 'node:fs' import { isAnthropicServerTool } from './web-search.mjs' export const DEFAULT_CACHE_TTL = '1h' @@ -35,7 +35,7 @@ export function cacheTtlFromRouting(routing = {}) { export function cacheTtlFromRoutingFile(filePath) { if (!filePath) return DEFAULT_CACHE_TTL try { - return cacheTtlFromRouting(JSON.parse(fs.readFileSync(filePath, 'utf8'))) + return cacheTtlFromRouting(readRoutingDocument(filePath)) } catch { return DEFAULT_CACHE_TTL } @@ -399,7 +399,7 @@ export function cacheBreakpointsFromRouting(routing = {}) { export function cacheBreakpointsFromRoutingFile(filePath) { if (!filePath) return normalizeCacheBreakpoints(null) try { - return cacheBreakpointsFromRouting(JSON.parse(fs.readFileSync(filePath, 'utf8'))) + return cacheBreakpointsFromRouting(readRoutingDocument(filePath)) } catch { return normalizeCacheBreakpoints(null) } diff --git a/src/lib/transport/rust-kernel-supervisor.mjs b/src/lib/transport/rust-kernel-supervisor.mjs index 7ee7bd8db..873f3f918 100644 --- a/src/lib/transport/rust-kernel-supervisor.mjs +++ b/src/lib/transport/rust-kernel-supervisor.mjs @@ -1,3 +1,4 @@ +import { readRoutingDocument, routingDocumentRevision } from '../core/routing-document.mjs' /** * Rust gateway-worker lifecycle. kin-kernel is container PID 1. * Host writes credentials; slot kernel only reads AT for inference. @@ -828,6 +829,8 @@ export function syncClaudeKernelConfigs(projectRoot, routing) { */ export function syncClaudeKernelConfigsFromFile(projectRoot, routingFile) { if (!projectRoot || !routingFile) return { updated: 0, skipped: true } + const revision = routingDocumentRevision(routingFile) + if (revision !== null) return { updated: 0, skipped: true, pending: true, revision } let mtimeMs try { mtimeMs = fs.statSync(routingFile).mtimeMs @@ -837,7 +840,7 @@ export function syncClaudeKernelConfigsFromFile(projectRoot, routingFile) { if (routingPersonaMtime.get(routingFile) === mtimeMs) return { updated: 0, skipped: true } let routing try { - routing = JSON.parse(fs.readFileSync(routingFile, 'utf8')) + routing = readRoutingDocument(routingFile) } catch { return { updated: 0, skipped: true } } diff --git a/src/server.mjs b/src/server.mjs index ac10f6c55..04ac8ea26 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -315,7 +315,7 @@ const routingRt = createRoutingRuntime({ const { persistRoutingPatch, - loadRoutingConfig, + initRoutingConfig, syncTierDefaultsIntoRouting, initPoolRuntime, poolSchedulerConfig, @@ -329,8 +329,8 @@ const { migrateLegacyOpenAIPolicies, } = routingRt -routingConfig = loadRoutingConfig() -migrateLegacyOpenAIPolicies(routingConfig) +routingConfig = await initRoutingConfig() +if (!kubernetesMode) migrateLegacyOpenAIPolicies(routingConfig) setManualScheduleWins(routingConfig.pool?.manual_schedule_wins) if (routingConfig.official_cc) { routingConfig.official_cc = normalizeOfficialCcConfig(routingConfig.official_cc) @@ -448,6 +448,7 @@ try { } await initPoolRuntime() +routingRt.routingState?.start() try { applyRoutingTierConcurrency(routingConfig.tiers) } catch (e) { @@ -633,8 +634,8 @@ backupService.onRestored(async (db) => { groupsRepo = new GroupsRepo(db) apiScheduler.reload(await apiEndpointStore.listRaw()) - routingConfig = loadRoutingConfig() - migrateLegacyOpenAIPolicies(routingConfig) + routingConfig = routingRt.routingState ? await routingRt.routingState.rebind(db) : await initRoutingConfig() + if (!kubernetesMode) migrateLegacyOpenAIPolicies(routingConfig) setManualScheduleWins(routingConfig.pool?.manual_schedule_wins) if (routingConfig.official_cc) { routingConfig.official_cc = normalizeOfficialCcConfig(routingConfig.official_cc) @@ -668,6 +669,7 @@ backupService.onRestored(async (db) => { }) backupService.onMaintenance(async (enabled) => { if (enabled) { + await routingRt.routingState?.stop() vmTestSchedule.stop() proxyPool.stopScheduler() requestLog.stopCleanupScheduler() @@ -682,6 +684,7 @@ backupService.onMaintenance(async (enabled) => { await pauseVmDbSync() } else { if (_shuttingDown) return + routingRt.routingState?.start() resumeVmDbSync() resumePriceCatalog() for (const monitor of maintenanceMonitors()) await monitor?.start?.({ immediate: false }) @@ -813,6 +816,7 @@ async function requireAuth(req, res) { ) } if (kubernetesMode) { + await routingRt.refreshRouting() const snapshot = await new SettingsRepo().refresh() refreshModelPolicy(snapshot.model_policy) refreshGptModelPolicy(snapshot.gpt_model_policy) @@ -1441,6 +1445,7 @@ async function shutdown(signal, { failed = false } = {}) { forceClose() process.exit(1) }, shutdownGraceMs + 5000) + await routingRt.routingState?.stop() try { vmTestSchedule.stop() } catch {} diff --git a/test/unit/shared-routing-config.test.mjs b/test/unit/shared-routing-config.test.mjs index 0ff41f20a..88397ac21 100644 --- a/test/unit/shared-routing-config.test.mjs +++ b/test/unit/shared-routing-config.test.mjs @@ -3,8 +3,13 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' +import { setImmediate } from 'node:timers/promises' import { createRoutingRuntime } from '../../src/lib/admin/routing-runtime.mjs' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { readRoutingDocument } from '../../src/lib/core/routing-document.mjs' +import { readRoutingConfigFile } from '../../src/lib/core/config.mjs' +import { cacheTtlFromRoutingFile } from '../../src/lib/protocol/cache-ttl.mjs' +import { personaModeFromRoutingFile } from '../../src/lib/identity/crs-persona.mjs' async function fixture(t) { const db = getDb() @@ -91,3 +96,124 @@ test('an explicitly stale routing revision is rejected instead of overwriting ne code: 'routing_config_stale', }) }) + +test('a new replica uses PG once seeded even if its local bootstrap document is invalid', async (t) => { + const { + peer, + apps: [first, second], + } = await fixture(t) + await first.runtime.persistRoutingPatch({ sticky: { ttl_seconds: 300 } }) + fs.writeFileSync(second.file, '{invalid bootstrap JSON') + let config + const runtime = createRoutingRuntime({ + db: peer, + shared: true, + cfg: { paths: { project: path.dirname(second.file) } }, + routingConfigPath: second.file, + getRoutingConfig: () => config, + setRoutingConfig: (value) => { + config = value + }, + }) + await runtime.initRoutingConfig() + assert.equal(config.sticky.ttl_seconds, 300) +}) + +test('synchronous routing consumers read committed snapshots after refresh without touching input files', async (t) => { + const { + apps: [first, second], + } = await fixture(t) + await first.runtime.persistRoutingPatch({ compatibility: { cache_ttl: '5m', persona_inject: 'zero' } }) + await second.runtime.refreshRouting() + fs.unlinkSync(second.file) + assert.equal(readRoutingDocument(second.file).compatibility.cache_ttl, '5m') + assert.equal(cacheTtlFromRoutingFile(second.file), '5m') + assert.equal(personaModeFromRoutingFile(second.file), 'zero') + const previous = process.env.KIN_ROUTING_FILE + process.env.KIN_ROUTING_FILE = second.file + try { + assert.equal(readRoutingConfigFile().compatibility.cache_ttl, '5m') + } finally { + if (previous === undefined) delete process.env.KIN_ROUTING_FILE + else process.env.KIN_ROUTING_FILE = previous + } +}) + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +test('a delayed routing refresh cannot overwrite a newer committed revision', async (t) => { + const { + apps: [first, second], + } = await fixture(t) + const state = second.runtime.routingState + const read = state.repo.get.bind(state.repo) + const entered = deferred() + const resume = deferred() + state.repo.get = async () => { + const value = await read() + entered.resolve() + await resume.promise + return value + } + const delayed = state.refresh() + await entered.promise + state.repo.get = read + await first.runtime.persistRoutingPatch({ sticky: { ttl_seconds: 300 } }) + await state.refresh() + resume.resolve() + await delayed + assert.equal(second.get().sticky.ttl_seconds, 300) + assert.equal(readRoutingDocument(second.file).sticky.ttl_seconds, 300) +}) + +test('routing revision validation holds the PG share lock until execution admission commits', async (t) => { + const { + db, + apps: [first, second], + } = await fixture(t) + const entered = deferred() + const resume = deferred() + const revision = first.runtime.routingState.currentRevision + const admission = withTransaction(db, async () => { + await first.runtime.routingState.assertRevision(revision) + entered.resolve() + await resume.promise + }) + await entered.promise + let committed = false + const patch = second.runtime.persistRoutingPatch({ sticky: { ttl_seconds: 300 } }).then(() => { + committed = true + }) + try { + await setImmediate() + assert.equal(committed, false) + } finally { + resume.resolve() + await admission + await patch + } + await assert.rejects(first.runtime.routingState.assertRevision(revision), { code: 'routing_config_stale' }) + await assert.rejects(first.runtime.routingState.assertRevision(undefined), { code: 'routing_config_stale' }) +}) + +test('shared config reports pending VM projection without writing its VM fixture', async (t) => { + const { + apps: [first], + } = await fixture(t) + const project = path.dirname(first.file) + const vmFile = path.join(project, 'vms', 'vm-projection.json') + const content = JSON.stringify({ id: 'vm-projection', status: 'running', policy: { maxConcurrency: 2 } }) + fs.writeFileSync(vmFile, content) + const result = await first.runtime.persistRoutingPatch({ tiers: { default: { max_concurrency: 1 } } }) + assert.equal(result.application.status, 'pending') + assert.ok( + result.application.pending.some((item) => item.component === 'vm_projection' && item.vm_id === 'vm-projection'), + ) + assert.equal(fs.readFileSync(vmFile, 'utf8'), content) +}) From 2f95daeedbed473ef28c79633306f41f2ddd04ef Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:19:00 +0800 Subject: [PATCH 153/473] test: specify durable shared background task ownership --- test/unit/shared-background-tasks.test.mjs | 227 +++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 test/unit/shared-background-tasks.test.mjs diff --git a/test/unit/shared-background-tasks.test.mjs b/test/unit/shared-background-tasks.test.mjs new file mode 100644 index 000000000..f78f6291c --- /dev/null +++ b/test/unit/shared-background-tasks.test.mjs @@ -0,0 +1,227 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { createInterface } from 'node:readline' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' + +async function service(db = getDb(), ownerId = 'owner-a') { + const { SharedBackgroundTasks } = await import('../../src/lib/core/shared-background-tasks.mjs') + return new SharedBackgroundTasks(db, { ownerId }) +} + +async function otherDatabase(t) { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + return other +} + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +const plan = (executionId, taskKey = 'synthetic-task') => ({ taskKey, executionId }) + +test('two PostgreSQL connections have one claim per task and completed executions remain deduplicated', async (t) => { + const first = await service() + const second = await service(await otherDatabase(t), 'owner-b') + const claims = await Promise.all([first.tryClaim(plan('one')), second.tryClaim(plan('one'))]) + assert.equal(claims.filter((claim) => claim.ok).length, 1) + assert.equal(claims.find((claim) => !claim.ok).code, 'background_task_active') + const winner = claims.find((claim) => claim.ok) + assert.deepEqual(await winner.complete(), { completed: true }) + assert.deepEqual(await winner.complete(), { completed: true }) + assert.equal((await second.tryClaim(plan('one'))).code, 'background_task_completed') + const next = await second.tryClaim(plan('two')) + assert.equal(next.ok, true) + await next.complete() +}) + +test('interval execution IDs use PostgreSQL time rather than either controller clock', async (t) => { + const first = await service() + const second = await service(await otherDatabase(t), 'owner-b') + t.mock.method(Date, 'now', () => 0) + const claimed = await first.tryClaim({ taskKey: 'interval', intervalMs: 1_000_000_000_000 }) + const dbNow = await second.now() + assert.equal(claimed.executionId, `interval:1000000000000:${Math.floor(dbNow / 1_000_000_000_000)}`) + await claimed.complete() + Date.now = () => 9_000_000_000_000 + assert.equal( + (await second.tryClaim({ taskKey: 'interval', intervalMs: 1_000_000_000_000 })).code, + 'background_task_completed', + ) +}) + +test('exceptions, ordinary error returns and explicit unknown outcomes block subsequent plans without TTL', async (t) => { + const first = await service() + const second = await service(await otherDatabase(t), 'owner-b') + await assert.rejects( + first.run(plan('one', 'exception'), () => { + throw new Error('synthetic remote outcome unknown') + }), + /synthetic remote outcome unknown/, + ) + const ordinary = await first.run(plan('one', 'ordinary'), () => ({ value: { ok: false } })) + assert.equal(ordinary.unknown, true) + const unknown = await first.tryClaim(plan('one', 'unknown')) + await unknown.markUnknown() + assert.deepEqual(await unknown.complete(), { completed: false }) + await getDb().exec("UPDATE background_task_runs SET claimed_at = clock_timestamp() - interval '10 years'") + for (const key of ['exception', 'ordinary', 'unknown']) { + assert.equal((await second.tryClaim(plan('later', key))).code, 'background_task_active') + } +}) + +test('explicit terminal failures and known not-started errors permit later plans', async (t) => { + const first = await service() + const second = await service(await otherDatabase(t), 'owner-b') + const result = await first.run(plan('one'), () => ({ terminal: true, value: { ok: false, code: 'rejected' } })) + assert.deepEqual(result.value, { ok: false, code: 'rejected' }) + assert.equal(result.unknown, false) + const notStarted = await first.run(plan('two'), () => ({ notStarted: true, value: null })) + assert.equal(notStarted.unknown, false) + await assert.rejects( + first.run(plan('three'), () => { + throw Object.assign(new Error('synthetic validation failure'), { taskNotStarted: true }) + }), + /synthetic validation failure/, + ) + const next = await second.tryClaim(plan('four')) + assert.equal(next.ok, true) + await next.complete() +}) + +test('shared results persist only when explicitly supplied and are returned to deduplicated callers', async (t) => { + const first = await service() + const second = await service(await otherDatabase(t), 'owner-b') + await first.run(plan('private', 'private-task'), () => ({ terminal: true, value: { secret: 'synthetic' } })) + assert.equal( + (await getDb().prepare('SELECT shared_result FROM background_task_runs WHERE task_key = $1').get('private-task')) + .shared_result, + null, + ) + const publicResult = { anthropic: { ok: true, message_id: 'synthetic-message' }, openai: null } + await first.run(plan('one', 'public-task'), () => ({ terminal: true, value: 42, sharedResult: publicResult })) + const skipped = await second.run(plan('one', 'public-task'), () => assert.fail('duplicate callback')) + assert.equal(skipped.ran, false) + assert.deepEqual(skipped.sharedResult, publicResult) +}) + +test('claim completion validates ownership and retries PostgreSQL after a failed completion', async (t) => { + const first = await service() + const second = await service(await otherDatabase(t), 'owner-b') + const claimed = await first.tryClaim(plan('one')) + assert.deepEqual(await second.complete({ ...claimed, ownerId: 'owner-b' }), { completed: false }) + await getDb().exec(`CREATE FUNCTION refuse_background_finish() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.finished_at IS NOT NULL THEN RAISE EXCEPTION 'synthetic completion failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER refuse_background_finish BEFORE UPDATE ON background_task_runs FOR EACH ROW EXECUTE FUNCTION refuse_background_finish()`) + await assert.rejects(claimed.complete(), { code: 'P0001' }) + assert.equal((await second.tryClaim(plan('later'))).code, 'background_task_active') + await getDb().exec('DROP TRIGGER refuse_background_finish ON background_task_runs') + assert.deepEqual(await claimed.complete(), { completed: true }) +}) + +test('closing the owner database cannot release its committed claim or run a fallback', async (t) => { + const other = await otherDatabase(t) + const first = await service(other) + const second = await service(getDb(), 'owner-b') + assert.equal((await first.tryClaim(plan('one'))).ok, true) + await other.close() + let called = false + await assert.rejects(first.run(plan('other', 'closed'), () => (called = true))) + assert.equal(called, false) + assert.equal((await second.tryClaim(plan('later'))).code, 'background_task_active') +}) + +test('claims cannot be created within an outer transaction that could roll them back after a side effect', async () => { + const tasks = await service() + await assert.rejects( + withTransaction(getDb(), () => tasks.tryClaim(plan('one'))), + { + code: 'background_task_transaction_active', + }, + ) +}) + +test('flush awaits the callback and its durable terminal state', async () => { + const tasks = await service() + const entered = deferred() + const finish = deferred() + const pending = tasks.run(plan('one'), async () => { + entered.resolve() + await finish.promise + return { terminal: true } + }) + await entered.promise + let flushed = false + const flushing = tasks.flush().then(() => { + flushed = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(flushed, false) + finish.resolve() + await Promise.all([pending, flushing]) + assert.equal( + (await getDb().prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get()).n, + 0, + ) +}) + +function spawnOwner(script) { + return spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: ['ignore', 'pipe', 'pipe'] }) +} + +function childSetup() { + const databaseUrl = new URL('../../src/lib/db/database.mjs', import.meta.url).href + const serviceUrl = new URL('../../src/lib/core/shared-background-tasks.mjs', import.meta.url).href + return `import { createDatabase } from ${JSON.stringify(databaseUrl)}; + import { SharedBackgroundTasks } from ${JSON.stringify(serviceUrl)}; + const db = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: process.env.KIN_DATABASE_SCHEMA }); + const tasks = new SharedBackgroundTasks(db);` +} + +async function firstLine(child) { + const lines = createInterface({ input: child.stdout }) + return Promise.race([ + once(lines, 'line').then(([line]) => JSON.parse(line)), + once(child, 'exit').then(() => { + throw new Error('synthetic owner exited before reporting') + }), + ]) +} + +test('two separate processes execute the same explicit plan once', async (t) => { + await service() + await getDb().exec('CREATE TABLE synthetic_background_effects (id integer)') + const script = `${childSetup()} + const result = await tasks.run({ taskKey: 'process-race', executionId: 'same-plan' }, async () => { + await db.prepare('INSERT INTO synthetic_background_effects VALUES (1)').run(); + return { terminal: true }; + }); + process.stdout.write(JSON.stringify({ ran: result.ran }) + '\\n'); + await db.close();` + const children = [spawnOwner(script), spawnOwner(script)] + t.after(() => children.forEach((child) => child.kill('SIGKILL'))) + const outcomes = await Promise.all(children.map(firstLine)) + assert.equal(outcomes.filter((outcome) => outcome.ran).length, 1) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM synthetic_background_effects').get()).n, 1) +}) + +test('SIGKILL does not make a background task available to a new owner', async (t) => { + const observer = await service() + const child = spawnOwner(`${childSetup()} + const claim = await tasks.tryClaim({ taskKey: 'killed', executionId: 'one' }); + process.stdout.write(JSON.stringify({ ok: claim.ok }) + '\\n'); + setInterval(() => {}, 1000);`) + t.after(() => child.kill('SIGKILL')) + assert.equal((await firstLine(child)).ok, true) + const exited = once(child, 'exit') + child.kill('SIGKILL') + await exited + assert.equal((await observer.tryClaim(plan('later', 'killed'))).code, 'background_task_active') +}) From 30022545f50d0c9d204103a405f6f097de7d36f3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:20:32 +0800 Subject: [PATCH 154/473] feat(k8s): persist background task execution claims --- src/lib/core/shared-background-tasks.mjs | 133 ++++++++++++++++++ .../013_background_task_runs.sql | 13 ++ .../db/repos/background-task-runs-repo.mjs | 79 +++++++++++ 3 files changed, 225 insertions(+) create mode 100644 src/lib/core/shared-background-tasks.mjs create mode 100644 src/lib/db/postgres-migrations/013_background_task_runs.sql create mode 100644 src/lib/db/repos/background-task-runs-repo.mjs diff --git a/src/lib/core/shared-background-tasks.mjs b/src/lib/core/shared-background-tasks.mjs new file mode 100644 index 000000000..dcd735593 --- /dev/null +++ b/src/lib/core/shared-background-tasks.mjs @@ -0,0 +1,133 @@ +import { randomUUID } from 'node:crypto' +import { getDb } from '../db/database.mjs' +import { BackgroundTaskRunsRepo } from '../db/repos/background-task-runs-repo.mjs' +import { isKubernetesMode } from './control-plane-mode.mjs' + +const services = new WeakMap() + +/** Unknown work remains occupied until an explicit recovery protocol proves it ended. */ +export class SharedBackgroundTasks { + constructor(db, { ownerId = randomUUID() } = {}) { + this.repo = new BackgroundTaskRunsRepo(db) + this.ownerId = ownerId + this.pending = new Set() + } + + now() { + return this.repo.now() + } + + async tryClaim(plan) { + const identity = await this.repo.tryClaim({ ...plan, ownerId: this.ownerId }) + if (!identity.ok) return identity + let unknown = false + let marking = null + let completing = null + return { + ...identity, + markUnknown: () => { + unknown = true + if (!marking) { + const pending = this.markUnknown(identity) + marking = pending + pending.catch(() => { + if (marking === pending) marking = null + }) + } + return marking + }, + complete: async (options) => { + if (unknown) { + if (marking) await marking + return { completed: false } + } + if (!completing) { + const pending = this.complete(identity, options) + completing = pending + pending.then( + (result) => { + if (!result.completed && completing === pending) completing = null + }, + () => { + if (completing === pending) completing = null + }, + ) + } + return completing + }, + } + } + + markUnknown(identity) { + return this.repo.markUnknown(identity) + } + + complete(identity, options) { + return this.repo.complete(identity, options) + } + + run(plan, callback) { + const pending = this._run(plan, callback) + this.pending.add(pending) + pending.then( + () => this.pending.delete(pending), + () => this.pending.delete(pending), + ) + return pending + } + + async _run(plan, callback) { + const claim = await this.tryClaim(plan) + if (!claim.ok) + return { + ran: false, + code: claim.code, + ...(Object.hasOwn(claim, 'sharedResult') ? { sharedResult: claim.sharedResult } : {}), + } + let outcome + try { + outcome = await callback() + } catch (error) { + if (error?.taskNotStarted === true) await this._complete(claim) + else await claim.markUnknown() + throw error + } + const terminal = outcome?.terminal === true || outcome?.notStarted === true + if (terminal) { + await this._complete( + claim, + Object.hasOwn(outcome, 'sharedResult') ? { sharedResult: outcome.sharedResult } : undefined, + ) + } else { + await claim.markUnknown() + } + return { ran: true, value: outcome?.value, unknown: !terminal } + } + + async _complete(claim, options) { + if (!(await claim.complete(options)).completed) { + throw Object.assign(new Error('background task completion unconfirmed'), { + code: 'background_task_completion_unconfirmed', + }) + } + } + + async flush() { + while (this.pending.size) await Promise.allSettled([...this.pending]) + } +} + +export function getSharedBackgroundTasks(db = getDb()) { + let service = services.get(db) + if (!service) { + service = new SharedBackgroundTasks(db) + services.set(db, service) + } + return service +} + +/** Tests can inject a service; standalone keeps its original local scheduling. */ +export function backgroundTasksFor(injected) { + if (injected !== undefined) return injected + return isKubernetesMode() ? getSharedBackgroundTasks() : null +} diff --git a/src/lib/db/postgres-migrations/013_background_task_runs.sql b/src/lib/db/postgres-migrations/013_background_task_runs.sql new file mode 100644 index 000000000..e853643b5 --- /dev/null +++ b/src/lib/db/postgres-migrations/013_background_task_runs.sql @@ -0,0 +1,13 @@ +CREATE TABLE background_task_runs ( + task_key text COLLATE "C" NOT NULL, + execution_id text COLLATE "C" NOT NULL, + owner_id text COLLATE "C" NOT NULL, + claimed_at timestamptz NOT NULL DEFAULT clock_timestamp(), + unknown_at timestamptz, + finished_at timestamptz, + shared_result jsonb, + PRIMARY KEY (task_key, execution_id) +); + +-- A new schedule cannot prove that an older remote side effect has ended. +CREATE UNIQUE INDEX idx_background_task_active ON background_task_runs (task_key) WHERE finished_at IS NULL; diff --git a/src/lib/db/repos/background-task-runs-repo.mjs b/src/lib/db/repos/background-task-runs-repo.mjs new file mode 100644 index 000000000..8911530af --- /dev/null +++ b/src/lib/db/repos/background-task-runs-repo.mjs @@ -0,0 +1,79 @@ +import { withTransaction } from '../database.mjs' + +function invalidPlan() { + return Object.assign(new Error('invalid background task plan'), { code: 'invalid_background_task', status: 400 }) +} + +function identifier(value, max) { + if (typeof value !== 'string' || !value.trim() || value.length > max) throw invalidPlan() + return value +} + +export class BackgroundTaskRunsRepo { + constructor(db) { + this.db = db + } + + async now() { + return (await this.db.prepare('SELECT extract(epoch FROM clock_timestamp()) * 1000 AS now_ms').get()).now_ms + } + + async tryClaim({ taskKey, executionId, intervalMs, ownerId }) { + identifier(taskKey, 512) + identifier(ownerId, 200) + if (executionId == null) { + if (!Number.isSafeInteger(intervalMs) || intervalMs <= 0) throw invalidPlan() + } else { + identifier(executionId, 1024) + if (intervalMs != null) throw invalidPlan() + } + if (this.db.isTransaction) { + throw Object.assign(new Error('background task requires an independently committed claim'), { + code: 'background_task_transaction_active', + }) + } + return withTransaction(this.db, async () => { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ + `vm2api:background:${this.db.schema}`, + taskKey, + ]) + const active = await this.db + .prepare('SELECT execution_id FROM background_task_runs WHERE task_key = $1 AND finished_at IS NULL') + .get(taskKey) + if (active) return { ok: false, code: 'background_task_active' } + const id = executionId ?? `interval:${intervalMs}:${Math.floor((await this.now()) / intervalMs)}` + const previous = await this.db + .prepare('SELECT shared_result FROM background_task_runs WHERE task_key = $1 AND execution_id = $2') + .get(taskKey, id) + if (previous) { + return { + ok: false, + code: 'background_task_completed', + ...(previous.shared_result == null ? {} : { sharedResult: previous.shared_result }), + } + } + await this.db + .prepare('INSERT INTO background_task_runs (task_key, execution_id, owner_id) VALUES ($1, $2, $3)') + .run(taskKey, id, ownerId) + return { ok: true, taskKey, executionId: id, ownerId } + }) + } + + async markUnknown({ taskKey, executionId, ownerId }) { + const result = await this.db + .prepare(`UPDATE background_task_runs SET unknown_at = COALESCE(unknown_at, clock_timestamp()) + WHERE task_key = $1 AND execution_id = $2 AND owner_id = $3 AND finished_at IS NULL`) + .run(taskKey, executionId, ownerId) + return { unknown: result.changes > 0 } + } + + async complete({ taskKey, executionId, ownerId }, { sharedResult } = {}) { + const result = await this.db + .prepare(`UPDATE background_task_runs + SET shared_result = CASE WHEN finished_at IS NULL THEN $4::jsonb ELSE shared_result END, + finished_at = COALESCE(finished_at, clock_timestamp()) + WHERE task_key = $1 AND execution_id = $2 AND owner_id = $3 AND unknown_at IS NULL`) + .run(taskKey, executionId, ownerId, sharedResult == null ? null : JSON.stringify(sharedResult)) + return { completed: result.changes > 0 } + } +} From ba59194b3957b997bafc9b6fed9bb8579821c8e4 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:21:21 +0800 Subject: [PATCH 155/473] test(search): reproduce missing shared execution admission and completion --- test/unit/shared-codex-search.test.mjs | 305 +++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 test/unit/shared-codex-search.test.mjs diff --git a/test/unit/shared-codex-search.test.mjs b/test/unit/shared-codex-search.test.mjs new file mode 100644 index 000000000..1feb03621 --- /dev/null +++ b/test/unit/shared-codex-search.test.mjs @@ -0,0 +1,305 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import { EventEmitter } from 'node:events' +import { setTimeout as delay } from 'node:timers/promises' +import fetch from 'node-fetch' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { handleCodexSearch } from '../../src/lib/protocol/handle-codex.mjs' +import { SharedCodexRuntime } from '../../src/lib/pool/shared-codex-runtime.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' +import { SharedRoutingConfig } from '../../src/lib/admin/shared-routing-config.mjs' +import { resetOpenAIAccountRuntime } from '../../src/lib/pool/openai-account-runtime.mjs' + +const accountId = 'codex:synthetic-search-account' +const vmId = 'vm-search-a' + +async function until(check) { + const deadline = Date.now() + 2000 + while (!(await check())) { + if (Date.now() >= deadline) assert.fail('condition did not become true') + await delay(5) + } +} + +function answer(res, status = 200, body = { output: 'synthetic search result' }) { + res.writeHead(status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(body)) +} + +async function fixture(t, handle = (_req, res) => answer(res), { vms = 1 } = {}) { + resetOpenAIAccountRuntime() + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-search-')) + for (const id of [vmId, 'vm-search-b'].slice(0, vms)) { + fs.mkdirSync(path.join(root, 'vms', id), { recursive: true }) + fs.writeFileSync( + path.join(root, 'vms', `${id}.json`), + JSON.stringify({ + id, + platform: 'openai', + family: 'codex', + schedulable: true, + status: 'running', + policy: { maxConcurrency: 1, concurrencyOverride: false, maxRpm: 99, rpmOverride: false }, + }), + ) + fs.writeFileSync( + path.join(root, 'vms', id, 'codex-credentials.json'), + JSON.stringify({ + accounts: [ + { + id: 'synthetic-account', + access_token: 'synthetic-access', + refresh_token: 'synthetic-refresh', + chatgpt_account_id: 'synthetic-search-account', + }, + ], + }), + ) + } + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const runtimes = [db, other].map( + (connection, index) => new SharedCodexRuntime(connection, { ownerId: `search-${index}` }), + ) + const calls = [] + const bindings = [] + const server = http.createServer((req, res) => { + void (async () => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + calls.push({ headers: req.headers, body: JSON.parse(Buffer.concat(chunks).toString()) }) + bindings.push( + await other + .prepare('SELECT * FROM execution_jobs WHERE request_id = $1') + .get(req.headers['x-client-request-id'] || ''), + ) + await handle(req, res, calls.length) + })().catch((error) => res.destroy(error)) + }) + t.after(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + await other.close() + fs.rmSync(root, { recursive: true, force: true }) + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + const url = `http://127.0.0.1:${server.address().port}/search` + return { + root, + db, + other, + calls, + bindings, + runtimes, + counts: () => runtimes[1].admissions.getCounts(accountId, vmId), + async run({ controller = 0, signal, timeoutMs, quota = {}, refresh, pin = null } = {}) { + const req = Object.assign(new EventEmitter(), { + headers: { 'user-agent': 'codex_exec/0.160.1', ...(pin ? { 'x-kin-vm': pin } : {}) }, + apiKeyKind: pin ? 'master' : 'user', + }) + const res = Object.assign(new EventEmitter(), { statusCode: 0, setHeader() {} }) + const logBag = {} + const aborted = () => req.emit('aborted') + signal?.addEventListener('abort', aborted, { once: true }) + try { + await handleCodexSearch({ + req, + res, + body: { id: 'search-request-id', model: 'gpt-6.1-sol', commands: { search_query: [{ q: 'synthetic' }] } }, + logBag, + stats: { requests: 0, errors: 0, by_route: {} }, + routing: { codex: { quota: { max_concurrency: 1, ...quota } }, pool: { fallback_wait_timeout_ms: 1000 } }, + projectRoot: root, + json(_res, status, body) { + res.statusCode = status + res.body = body + }, + ops: { + sharedRuntime: runtimes[controller], + // Bound the old truncated-body behavior as well as the implementation under test. + fetchImpl: (_url, init) => fetch(url, { timeout: 2500, ...init }), + refresh, + searchTimeoutMs: timeoutMs, + }, + }) + } finally { + signal?.removeEventListener('abort', aborted) + } + return { res, logBag } + }, + } +} + +test('Search holds global capacity until its complete HTTP body and persists the job before send', async (t) => { + let response + const fx = await fixture(t, (_req, res, count) => { + if (count > 1) return answer(res) + response = res + res.writeHead(200, { 'content-type': 'application/json' }) + res.write('{"output":') + }) + const first = fx.run() + try { + await until(() => fx.calls.length === 1) + const blocked = await fx.run({ controller: 1 }) + assert.equal(blocked.res.statusCode, 429) + assert.equal(fx.calls.length, 1) + assert.ok(fx.bindings[0], 'the independent database connection must observe the job before HTTP submission') + assert.equal((await fx.counts()).vmInflight, 1) + } finally { + response?.end('"complete"}') + await first + } + assert.equal((await fx.counts()).vmInflight, 0) + const row = await fx.other + .prepare( + 'SELECT j.ended_at, a.released_at FROM execution_jobs j JOIN execution_admissions a ON a.id=j.admission_id', + ) + .get() + assert.ok(row.ended_at && row.released_at >= row.ended_at) + assert.equal((await fx.other.prepare('SELECT count(*) AS count FROM session_turns').get()).count, 0) +}) + +test('a truncated Search body retains capacity and forbids retry on another VM', async (t) => { + const fx = await fixture( + t, + (_req, res) => { + res.writeHead(401, { 'content-type': 'application/json', 'content-length': '200' }) + res.write('{"error":') + setImmediate(() => res.destroy()) + }, + { vms: 2 }, + ) + const result = await fx.run() + assert.equal(fx.calls.length, 1) + assert.equal(result.res.statusCode, 502) + assert.equal(result.res.body.error.code, 'codex_search_execution_uncertain') + assert.equal((await fx.counts()).accountInflight, 1) + const job = await fx.other.prepare('SELECT * FROM execution_jobs').get() + assert.equal(job.ended_at, null) + const blocked = await fx.run({ controller: 1 }) + assert.equal(blocked.res.statusCode, 429) + assert.equal(fx.calls.length, 1) +}) + +for (const status of [401, 403]) { + test(`Search completes its ${status} job before refresh and retries with a new persisted ID`, async (t) => { + const fx = await fixture(t, (_req, res, count) => answer(res, count === 1 ? status : 200)) + let refreshes = 0 + const result = await fx.run({ + refresh: async () => { + refreshes += 1 + const previous = await fx.other.prepare('SELECT * FROM execution_jobs').get() + assert.ok(previous?.ended_at, 'the first HTTP job must be durably complete before refresh/retry') + return { ok: true, access_token: 'synthetic-new-access', refresh_token: 'synthetic-new-refresh' } + }, + }) + assert.equal(result.res.statusCode, 200) + assert.equal(refreshes, 1) + assert.equal(fx.calls.length, 2) + assert.ok(fx.bindings.every(Boolean)) + assert.notEqual(fx.calls[0].headers['x-client-request-id'], fx.calls[1].headers['x-client-request-id']) + const jobs = await fx.other.prepare('SELECT * FROM execution_jobs').all() + assert.equal(jobs.length, 2) + assert.ok(jobs.every((job) => job.ended_at)) + assert.equal((await fx.counts()).accountInflight, 0) + }) +} + +for (const phase of ['timeout', 'cancel']) { + test(`Search ${phase} after response headers retains its job and capacity`, async (t) => { + let timer + const fx = await fixture( + t, + (_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.write('{"output":') + // The finite drain also makes the old missing-timeout implementation fail without hanging the test. + timer = setTimeout(() => res.end('"late"}'), 200) + }, + { vms: 2 }, + ) + t.after(() => clearTimeout(timer)) + const controller = new AbortController() + const pending = fx.run({ signal: controller.signal, timeoutMs: phase === 'timeout' ? 30 : undefined }) + await until(() => fx.calls.length === 1) + if (phase === 'cancel') controller.abort() + const result = await pending + if (phase === 'timeout') { + assert.equal(result.res.statusCode, 502) + assert.equal(result.res.body.error.code, 'codex_search_execution_uncertain') + } else assert.equal(result.logBag.final_state, 'cancelled') + assert.equal(fx.calls.length, 1) + assert.equal((await fx.counts()).accountInflight, 1) + const job = await fx.other.prepare('SELECT * FROM execution_jobs').get() + assert.equal(job.ended_at, null) + }) +} + +for (const [phase, table, trigger, code, expectedCalls] of [ + ['binding', 'execution_jobs', 'INSERT', 'P6491', 0], + ['completion', 'execution_jobs', 'UPDATE OF ended_at', 'P6492', 1], + ['release', 'execution_admissions', 'UPDATE OF released_at', 'P6493', 1], +]) { + test(`a Search database ${phase} failure stops further HTTP jobs and retains capacity`, async (t) => { + const fx = await fixture(t, (_req, res) => answer(res, 500), { vms: 2 }) + await fx.db.exec(`CREATE FUNCTION reject_search_job() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic search ${phase}' USING ERRCODE = '${code}'; END; + $$; CREATE TRIGGER reject_search_job BEFORE ${trigger} ON ${table} FOR EACH ROW EXECUTE FUNCTION reject_search_job();`) + await assert.rejects(fx.run(), { code }) + assert.equal(fx.calls.length, expectedCalls) + assert.equal((await fx.counts()).accountInflight, 1) + assert.equal((await fx.other.prepare('SELECT * FROM execution_admissions').get()).released_at, null) + }) +} + +test('Search reselects captured PostgreSQL policy after a configuration revision race', async (t) => { + const fx = await fixture(t) + const file = path.join(fx.root, 'synthetic-routing.json') + fs.writeFileSync(file, JSON.stringify({ codex: { quota: { max_concurrency: 2, max_rpm: 0 } } })) + const state = new SharedRoutingConfig({ + db: fx.db, + file, + normalize: (value) => value, + merge: (current, patch) => ({ ...current, ...patch }), + apply: async () => {}, + }) + await state.init() + fx.runtimes[0] = new SharedCodexRuntime(fx.db) + const held = await new SharedExecutionAdmissions(fx.other).tryReserve({ + accountId, + vmId: 'vm-held-elsewhere', + accountLimit: 2, + vmLimit: 1, + rpmLimit: 0, + fableLimit: 0, + }) + const reserve = fx.runtimes[0].tryReserve.bind(fx.runtimes[0]) + let changed = false + fx.runtimes[0].tryReserve = async (...args) => { + if (!changed) { + changed = true + await state.patch({ codex: { quota: { max_concurrency: 1, max_rpm: 0 } } }) + } + return reserve(...args) + } + try { + const result = await fx.run({ quota: { max_concurrency: 9, max_rpm: 99 } }) + assert.equal(changed, true) + assert.equal(result.res.statusCode, 429) + assert.equal(fx.calls.length, 0) + } finally { + await held.release() + } +}) From 554b3cca09891d83b37ef0e6c9d6792333376d65 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:21:48 +0800 Subject: [PATCH 156/473] test: expose shared routing application state in panel API --- test/unit/shared-routing-panel.test.mjs | 67 +++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 test/unit/shared-routing-panel.test.mjs diff --git a/test/unit/shared-routing-panel.test.mjs b/test/unit/shared-routing-panel.test.mjs new file mode 100644 index 000000000..c2e86c275 --- /dev/null +++ b/test/unit/shared-routing-panel.test.mjs @@ -0,0 +1,67 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' + +function fixture(t, persist) { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-routing-panel-')) + t.after(() => { + fs.rmSync(project, { recursive: true, force: true }) + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) + const response = {} + let restored = 0 + const ctx = { + cfg: { paths: { project }, limits: { max_body_bytes: 4096 } }, + routingConfig: { logging: {}, inference: { engine: 'rust' } }, + stickyRouter: { reloadConfig() {} }, + accountQuota: { reloadConfig() {} }, + poolSchedulerConfig: () => ({}), + requestLog: { + setConfig() { + restored++ + }, + }, + persistRoutingPatch: persist, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + json(_res, status, payload) { + response.status = status + response.body = payload + return true + }, + readBody: async () => ({ pool: { queue_max: 7 } }), + } + return { + response, + ctx, + restored: () => restored, + run: () => createPanelHandler(ctx)({ method: 'PUT' }, {}, new URL('http://localhost/api/panel/routing')), + } +} + +test('shared routing HTTP response reports committed revision and pending VM application', async (t) => { + const application = { status: 'pending', pending: [{ component: 'vm_projection', vm_id: 'vm-test', revision: 9 }] } + const fx = fixture(t, async () => ({ revision: 9, application })) + await fx.run() + assert.equal(fx.response.status, 200) + assert.equal(fx.response.body.data.routing_revision, 9) + assert.deepEqual(fx.response.body.data.application, application) +}) + +test('shared routing PG failure does not restore a potentially newer local committed snapshot', async (t) => { + const fx = fixture(t, async () => { + throw new Error('synthetic PG failure') + }) + await fx.run() + assert.equal(fx.response.status, 503) + assert.equal(fx.restored(), 0) +}) From be18f2547536c822b565649fdb0505f68fb8c7e3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:22:37 +0800 Subject: [PATCH 157/473] test: defer shared routing engine application until after commit --- test/unit/shared-routing-panel.test.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/unit/shared-routing-panel.test.mjs b/test/unit/shared-routing-panel.test.mjs index c2e86c275..f9f032677 100644 --- a/test/unit/shared-routing-panel.test.mjs +++ b/test/unit/shared-routing-panel.test.mjs @@ -65,3 +65,13 @@ test('shared routing PG failure does not restore a potentially newer local commi assert.equal(fx.response.status, 503) assert.equal(fx.restored(), 0) }) + + +test('shared routing engine changes are committed as pending application', async (t) => { + const fx = fixture(t, async () => ({ revision: 10, application: { status: 'pending', pending: [] } })) + fx.ctx.routingConfig.inference.engine = 'go' + fx.ctx.readBody = async () => ({ inference: { engine: 'rust' } }) + await fx.run() + assert.equal(fx.response.status, 200) + assert.equal(fx.response.body.data.inference_runtime.status, 'pending') +}) From 97b4a087ae486814ffdc912f7b662fdeb2e387ff Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:24:51 +0800 Subject: [PATCH 158/473] fix(k8s): report pending routing application after committed updates --- src/lib/admin/panel-routes.mjs | 9 +++++++-- test/unit/shared-routing-panel.test.mjs | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 2ae495829..252c7b9f6 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -1,4 +1,5 @@ import { flushPendingWrites } from '../db/pending-writes.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' /** * /admin and /api/panel routes. Login / logout / me and { ok, data } * envelopes are unchanged; slot start/reload go through startSlotReady. @@ -5292,7 +5293,10 @@ export function createPanelHandler(ctx) { }, }) } - const engineRuntime = await prepareInheritedInferenceEngine(body) + const sharedRouting = isKubernetesMode() + const engineRuntime = sharedRouting + ? { ok: true, changed: false, items: [], ...(body?.inference ? { status: 'pending' } : {}) } + : await prepareInheritedInferenceEngine(body) if (!engineRuntime.ok) { return json(res, 503, { ok: false, @@ -5308,7 +5312,7 @@ export function createPanelHandler(ctx) { try { applied = appliedDuringSwitch ?? (await persistRoutingPatch(body)) } catch (error) { - await restoreRoutingRuntime(previousRoutingConfig) + if (!sharedRouting) await restoreRoutingRuntime(previousRoutingConfig) return json(res, 503, { ok: false, error: { code: 'routing_persist_failed', message: String(error?.message || error) }, @@ -5319,6 +5323,7 @@ export function createPanelHandler(ctx) { 200, panel.ok({ ...publicRoutingNotify(ctx.routingConfig), + ...(sharedRouting ? { routing_revision: applied.revision, application: applied.application } : {}), applied_concurrency: applied.concurrency, applied_rpm: applied.rpm, applied_session_slots: applied.session_slots, diff --git a/test/unit/shared-routing-panel.test.mjs b/test/unit/shared-routing-panel.test.mjs index f9f032677..0da85e4fc 100644 --- a/test/unit/shared-routing-panel.test.mjs +++ b/test/unit/shared-routing-panel.test.mjs @@ -66,7 +66,6 @@ test('shared routing PG failure does not restore a potentially newer local commi assert.equal(fx.restored(), 0) }) - test('shared routing engine changes are committed as pending application', async (t) => { const fx = fixture(t, async () => ({ revision: 10, application: { status: 'pending', pending: [] } })) fx.ctx.routingConfig.inference.engine = 'go' From c5f6ff3d1e3728c74ceb630929bc876298322a2a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:25:05 +0800 Subject: [PATCH 159/473] feat(search): retain shared execution capacity until HTTP completion --- src/lib/protocol/handle-codex.mjs | 115 ++++++++++++++++++++----- test/unit/shared-codex-search.test.mjs | 48 +++++++++++ 2 files changed, 143 insertions(+), 20 deletions(-) diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index 564120a04..ef0b5e695 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -3,6 +3,7 @@ */ import path from 'node:path' import { randomUUID } from 'node:crypto' +import fetch from 'node-fetch' import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { SharedCodexRuntime } from '../pool/shared-codex-runtime.mjs' import { getVm, listVms, persistCodexUsage, syncCodexQuotaSchedule } from '../vm/vm-registry.mjs' @@ -37,6 +38,7 @@ import { import { isLocalEgressProxy } from '../vm/egress.mjs' import { vmNodeId } from '../cluster/placement.mjs' import { hostProxyUrlForVm } from '../vm/slot-host.mjs' +import { createProxyAgent } from '../vm/proxy-agent.mjs' import { readCodexAccounts, upsertCodexAccount } from '../vm/codex-slot.mjs' import { scopeAccountFromVm } from './jev-intercept.mjs' import { @@ -44,7 +46,6 @@ import { CODEX_CATALOG_ORIGINATOR, CODEX_SEARCH_URL, CODEX_USER_AGENT, - makeProxyFetch, refreshCodexAccessToken, } from './codex-models.mjs' import { orderCodexSessionSlots, codexSlotAllowsModel, isCodexFailoverError } from '../pool/codex-slot-pool.mjs' @@ -1239,12 +1240,33 @@ function plainHeaders(headers) { } /** One search call on one slot. Resolves `{ status, text, headers }`, or `{ status: 0, error }`. */ -async function postCodexSearch({ projectRoot, vm, body, turnMetadata, fetchImpl, refresh = refreshCodexAccessToken }) { +async function postCodexSearch({ + projectRoot, + vm, + body, + turnMetadata, + fetchImpl, + refresh = refreshCodexAccessToken, + signal, + timeoutMs = CODEX_SEARCH_TIMEOUT_MS, + onExecutionStart, + onExecutionEnd, +}) { + const notStarted = { requestId: null, started: false, ended: true, reason: 'not_started' } const proxyUrl = hostProxyUrlForVm(vm) - if (!fetchImpl && !proxyUrl && !isLocalEgressProxy(vm.proxy)) return { status: 0, error: 'proxy_required' } - const doFetch = fetchImpl || makeProxyFetch(proxyUrl, CODEX_SEARCH_TIMEOUT_MS, { maxMs: CODEX_SEARCH_TIMEOUT_MS }) + if (!fetchImpl && !proxyUrl && !isLocalEgressProxy(vm.proxy)) + return { status: 0, error: 'proxy_required', execution: notStarted } + let agent + try { + if (!fetchImpl && proxyUrl) agent = createProxyAgent(proxyUrl.replace(/^socks5:\/\//i, 'socks5h://')) + } catch { + return { status: 0, error: 'upstream_transport', execution: notStarted } + } + const doFetch = fetchImpl || fetch const creds = searchCredentials(projectRoot, vm) const send = async (access) => { + if (signal?.aborted) return { status: 0, error: 'client_cancelled', execution: notStarted } + const requestId = randomUUID() // Same identity as the slot's kernel; the backend wants originator, User-Agent and version to agree. const headers = { authorization: `Bearer ${access}`, @@ -1253,19 +1275,48 @@ async function postCodexSearch({ projectRoot, vm, body, turnMetadata, fetchImpl, originator: CODEX_CATALOG_ORIGINATOR, 'user-agent': CODEX_USER_AGENT, version: CODEX_APP_VERSION, + 'x-client-request-id': requestId, } if (creds.accountId) headers['chatgpt-account-id'] = creds.accountId if (turnMetadata) headers['x-codex-turn-metadata'] = turnMetadata + const payload = JSON.stringify(body) + await onExecutionStart?.({ requestId, started: false, ended: false, reason: 'before_send' }) + const controller = new AbortController() + const abort = () => controller.abort() + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted) controller.abort() + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + controller.abort() + }, timeoutMs) + let out + let execution try { - const res = await doFetch(CODEX_SEARCH_URL, { method: 'POST', headers, body: JSON.stringify(body) }) - return { status: Number(res?.status) || 0, text: await res.text(), headers: plainHeaders(res?.headers) } + const res = await doFetch(CODEX_SEARCH_URL, { + method: 'POST', + headers, + body: payload, + agent, + signal: controller.signal, + ...(onExecutionStart ? { redirect: 'manual' } : {}), + }) + out = { status: Number(res?.status) || 0, text: await res.text(), headers: plainHeaders(res?.headers) } + execution = { requestId, started: true, ended: true, reason: 'http_body_complete' } } catch (e) { - return { status: 0, error: /abort/i.test(`${e?.name} ${e?.message}`) ? 'upstream_timeout' : 'upstream_transport' } + const aborted = timedOut || signal?.aborted || /abort/i.test(`${e?.name} ${e?.message}`) + out = { status: 0, error: aborted ? 'upstream_timeout' : 'upstream_transport' } + execution = { requestId, started: true, ended: false, reason: timedOut ? 'http_timeout' : 'http_transport' } + } finally { + clearTimeout(timer) + signal?.removeEventListener('abort', abort) } + await onExecutionEnd?.(execution) + return { ...out, execution } } - if (!creds.access && !creds.refresh) return { status: 0, error: 'no_oauth_token' } - let out = creds.access ? await send(creds.access) : { status: 401 } - if (out.status === 401 || out.status === 403) { + if (!creds.access && !creds.refresh) return { status: 0, error: 'no_oauth_token', execution: notStarted } + let out = creds.access ? await send(creds.access) : { status: 401, execution: notStarted } + if ((out.status === 401 || out.status === 403) && !signal?.aborted) { const access = await refreshSearchAccess(projectRoot, vm, creds.access, { proxyUrl, fetchImpl, refresh }) if (access) out = await send(access) } @@ -1319,7 +1370,8 @@ export async function handleCodexSearch({ for (const key of CODEX_SEARCH_DROP_FIELDS) delete outbound[key] const turnMetadata = headerString(req.headers, 'x-codex-turn-metadata') const post = ops.postCodexSearch || postCodexSearch - const pickOpts = { stickyRouter, sessions: null, body: outbound, excluded: new Set() } + const sharedRuntime = ops.sharedRuntime || (isKubernetesMode() ? new SharedCodexRuntime() : null) + const pickOpts = { stickyRouter, sessions: null, body: outbound, excluded: new Set(), routing, sharedRuntime } const deadline = Date.now() + codexWaitTimeoutMs(routing) const gone = clientGoneSignal(req, res) let last = null @@ -1347,24 +1399,47 @@ export async function handleCodexSearch({ logBag.vm_id = vm.id logBag.account_id = vm.id logBag.attempt_count = attempts - last = await post({ + const args = { projectRoot, vm, body: outbound, turnMetadata, fetchImpl: ops.fetchImpl, refresh: ops.refresh, - }) + signal: gone.signal, + timeoutMs: ops.searchTimeoutMs, + } + last = sharedRuntime ? await sharedRuntime.invoke(admitted.lease, post, args) : await post(args) + if (sharedRuntime && (last?.executionUncertain || last?.execution?.ended !== true)) { + logBag.final_state = gone.signal.aborted ? 'cancelled' : 'incomplete' + if (gone.signal.aborted) return + return reject( + 502, + 'api_error', + 'codex_search_execution_uncertain', + 'Codex search execution termination is unknown', + ) + } + if (gone.signal.aborted) { + logBag.final_state = 'cancelled' + return + } logBag.upstream_status = last.status || null - ingestCodexHop(projectRoot, vm.id, { - ok: last.status >= 200 && last.status < 300, - status: last.status, - headers: last.headers || {}, - body: parseJson(last.text), - }) + ingestCodexHop( + projectRoot, + vm.id, + { + ok: last.status >= 200 && last.status < 300, + status: last.status, + headers: last.headers || {}, + body: parseJson(last.text), + }, + Date.now(), + admitted.picked.quotaPolicy || codex.quota, + ) if (!isSearchFailover(last)) break } finally { - admitted.lease.release() + await admitted.lease.release() } } } finally { diff --git a/test/unit/shared-codex-search.test.mjs b/test/unit/shared-codex-search.test.mjs index 1feb03621..134e9c2c6 100644 --- a/test/unit/shared-codex-search.test.mjs +++ b/test/unit/shared-codex-search.test.mjs @@ -193,6 +193,54 @@ test('a truncated Search body retains capacity and forbids retry on another VM', assert.equal(fx.calls.length, 1) }) +test('a complete Search HTTP error releases its admission before failover to another VM', async (t) => { + const fx = await fixture( + t, + async (_req, res, count) => { + if (count > 1) { + const previous = await fx.other + .prepare(`SELECT j.ended_at, a.released_at FROM execution_jobs j + JOIN execution_admissions a ON a.id=j.admission_id WHERE j.request_id=$1`) + .get(fx.calls[0].headers['x-client-request-id']) + assert.ok(previous.ended_at && previous.released_at >= previous.ended_at) + } + answer(res, count === 1 ? 500 : 200) + }, + { vms: 2 }, + ) + const result = await fx.run() + assert.equal(result.res.statusCode, 200) + assert.equal(fx.calls.length, 2) + const jobs = await fx.other.prepare('SELECT * FROM execution_jobs').all() + assert.equal(jobs.length, 2) + assert.notEqual(jobs[0].admission_id, jobs[1].admission_id) + assert.ok(jobs.every((job) => job.ended_at)) + assert.equal((await fx.counts()).accountInflight, 0) +}) + +test('released Search admissions still enforce the shared account RPM across controllers', async (t) => { + const fx = await fixture(t) + assert.equal((await fx.run({ quota: { max_rpm: 1 } })).res.statusCode, 200) + assert.equal((await fx.run({ controller: 1, quota: { max_rpm: 1 } })).res.statusCode, 429) + assert.equal(fx.calls.length, 1) + assert.equal((await fx.counts()).accountInflight, 0) + assert.equal((await fx.counts()).rpmCount, 1) +}) + +test('shared Search does not follow an HTTP redirect as an unregistered second job', async (t) => { + const fx = await fixture(t, (_req, res, count) => { + if (count === 1) { + res.writeHead(307, { 'content-type': 'application/json', location: '/redirected-search' }) + res.end('{"redirected":true}') + } else answer(res) + }) + const result = await fx.run() + assert.equal(result.res.statusCode, 307) + assert.equal(fx.calls.length, 1) + assert.equal((await fx.other.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 1) + assert.equal((await fx.counts()).accountInflight, 0) +}) + for (const status of [401, 403]) { test(`Search completes its ${status} job before refresh and retries with a new persisted ID`, async (t) => { const fx = await fixture(t, (_req, res, count) => answer(res, count === 1 ? status : 200)) From 2592e9c922c247fcfa46d6c6aaa5daa72fc569c3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:25:21 +0800 Subject: [PATCH 160/473] test: reproduce duplicate background producers across controllers --- .../unit/shared-background-producers.test.mjs | 299 ++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 test/unit/shared-background-producers.test.mjs diff --git a/test/unit/shared-background-producers.test.mjs b/test/unit/shared-background-producers.test.mjs new file mode 100644 index 000000000..112f3d8d6 --- /dev/null +++ b/test/unit/shared-background-producers.test.mjs @@ -0,0 +1,299 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { RuntimeSettings, loadSettingsState } from '../../src/lib/db/settings-state.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { createHealthProbeMonitor, cachedClaudeMessage } from '../../src/lib/admin/health-probe.mjs' +import { createCredentialRefreshMonitor } from '../../src/lib/oauth/credential-refresh-monitor.mjs' +import { createUsageProbeMonitor } from '../../src/lib/oauth/usage-probe-monitor.mjs' +import { BackupService } from '../../src/lib/admin/backup-service.mjs' +import { VmTestScheduleRunner, VM_TEST_SCHEDULE_PREFIX } from '../../src/lib/admin/vm-test-schedule.mjs' + +const live = { + id: 'vm-synthetic', + status: 'running', + schedulable: true, + has_token: true, + has_refresh: true, + expires_at: 1, + proxy: { host: '127.0.0.1', port: 9, url: 'socks5://127.0.0.1:9' }, +} + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function fixture(t) { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + const first = new SharedBackgroundTasks(db, { ownerId: 'pod-a' }) + const second = new SharedBackgroundTasks(other, { ownerId: 'pod-b' }) + return { db, other, first, second } +} + +test('two health monitors execute one plan and share only approved snapshot fields', async (t) => { + const f = await fixture(t) + let calls = 0 + const opts = { + config: { enabled: true }, + listTargets: () => [live], + runChat: async () => { + calls++ + return { + ok: true, + status: 200, + text: 'synthetic-health', + headers: { authorization: 'synthetic-private' }, + body: { + id: 'msg_synthetic', + model: 'claude-haiku-4-5', + content: [{ type: 'text', text: 'synthetic-health' }], + usage: { input_tokens: 3, output_tokens: 2, provider_extension: { secret: 'synthetic-private' } }, + stop_reason: 'stop_sequence', + stop_sequence: 'synthetic-stop', + }, + } + }, + } + const first = createHealthProbeMonitor({ ...opts, sharedTasks: f.first }) + const second = createHealthProbeMonitor({ ...opts, sharedTasks: f.second }) + await first.runRealProbe() + await second.runRealProbe() + assert.equal(calls, 1) + const body = cachedClaudeMessage(second.rawSnapshot()) + assert.equal(body.id, 'msg_synthetic') + assert.equal(body.usage.output_tokens, 2) + assert.equal(body.stop_reason, 'stop_sequence') + assert.equal(body.stop_sequence, 'synthetic-stop') + const row = await f.other.prepare('SELECT shared_result FROM background_task_runs').get() + assert.equal(JSON.stringify(row.shared_result).includes('synthetic-private'), false) +}) + +test('credential refresh plans deduplicate across monitors and never persist the callback result', async (t) => { + const f = await fixture(t) + let calls = 0 + const opts = { + listTargets: () => [live], + refreshOne: async () => { + calls++ + return { ok: true, access_token: 'synthetic-private' } + }, + } + await createCredentialRefreshMonitor({ ...opts, sharedTasks: f.first }).runOnce() + await createCredentialRefreshMonitor({ ...opts, sharedTasks: f.second }).runOnce() + assert.equal(calls, 1) + assert.equal((await f.other.prepare('SELECT shared_result FROM background_task_runs').get()).shared_result, null) +}) + +test('usage timer and event probes share one persisted VM task plan across monitors', async (t) => { + const f = await fixture(t) + let calls = 0 + const opts = { + config: { enabled: true }, + listTargets: () => [live], + probeOne: async () => { + calls++ + return { ok: true, source: 'synthetic-usage' } + }, + } + await createUsageProbeMonitor({ ...opts, sharedTasks: f.first }).runOnce() + await createUsageProbeMonitor({ ...opts, sharedTasks: f.second }).probeNow(live.id) + assert.equal(calls, 1) +}) + +test('usage reconcile is awaited before the probe callback', async (t) => { + await fixture(t) + const entered = deferred() + const finish = deferred() + let reconciled = false + let calls = 0 + const monitor = createUsageProbeMonitor({ + listTargets: () => [live], + reconcile: async () => { + entered.resolve() + await finish.promise + reconciled = true + }, + probeOne: async () => { + assert.equal(reconciled, true) + calls++ + return { ok: true } + }, + }) + const pending = monitor.runOnce() + await entered.promise + try { + assert.equal(calls, 0) + } finally { + finish.resolve() + await pending + } + assert.equal(calls, 1) +}) + +test('usage timeout keeps the shared claim and flush waits for the original operation', async (t) => { + const f = await fixture(t) + const originalTimeout = globalThis.setTimeout + t.mock.method(globalThis, 'setTimeout', (callback, ms, ...args) => + originalTimeout(callback, ms === 10_000 ? 5 : ms, ...args), + ) + const finish = deferred() + let calls = 0 + const opts = { + config: { timeout_ms: 10_000 }, + listTargets: () => [live], + probeOne: async () => { + calls++ + await finish.promise + return { ok: true } + }, + } + const first = createUsageProbeMonitor({ ...opts, sharedTasks: f.first }) + const second = createUsageProbeMonitor({ ...opts, sharedTasks: f.second }) + await first.probeNow(live.id) + let flushed = false + const flushing = first.flush().then(() => { + flushed = true + }) + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(flushed, false) + await second.probeNow(live.id) + assert.equal(calls, 1) + } finally { + finish.resolve() + await flushing + } + assert.equal( + (await f.other.prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get()).n, + 1, + ) +}) + +test('stop prevents an already queued immediate monitor run from sending a new task', async (t) => { + const f = await fixture(t) + let calls = 0 + const monitor = createCredentialRefreshMonitor({ + sharedTasks: f.first, + listTargets: () => [live], + refreshOne: async () => { + calls++ + return { ok: true } + }, + }) + monitor.start({ immediate: true }) + monitor.stop() + await new Promise((resolve) => setImmediate(resolve)) + await monitor.flush() + assert.equal(calls, 0) +}) + +test('scheduled backups execute once across services without touching business files', async (t) => { + const f = await fixture(t) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'synthetic-bg-backup-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + let calls = 0 + const services = [f.first, f.second].map((sharedTasks) => { + const backup = new BackupService({ dataDir: root, sharedTasks }) + backup.getSchedule = async () => ({ enabled: true, interval_hours: 24, retention: 7 }) + backup.shouldRunScheduled = async () => true + backup.createBackup = async () => { + calls++ + return { status: 'ok', size_bytes: 0 } + } + return backup + }) + await Promise.all(services.map((backup) => backup.runScheduled())) + assert.equal(calls, 1) +}) + +async function schedules(t) { + const f = await fixture(t) + const settings = new RuntimeSettings(f.db) + await loadSettingsState(f.other) + const manager = new RuntimeSettings(f.other) + let calls = 0 + const create = (store, tasks, runTest = async () => ({ ok: true })) => + new VmTestScheduleRunner({ + projectRoot: '/synthetic-missing-project', + settings: store, + sharedTasks: tasks, + list: () => [live], + runTest: async () => { + calls++ + return runTest() + }, + }) + const first = create(settings, f.first) + await first.save(live.id, { + enabled: true, + frequency: 'daily', + time: '09:00', + timezone: 'UTC', + model: 'claude-haiku-4-5', + prompt: 'synthetic', + max_tokens: 16, + }) + await manager.refresh() + const second = create(manager, f.second) + await first.start() + await second.start() + t.after(() => { + first.stop() + second.stop() + }) + return { ...f, first, second, settings, manager, calls: () => calls, create } +} + +test('scheduled VM tests use one occurrence ID across controllers', async (t) => { + const f = await schedules(t) + await Promise.all([f.first._run(live.id), f.second._run(live.id)]) + assert.equal(f.calls(), 1) +}) + +test('a stale scheduled VM plan is refreshed and validated before invoking its callback', async (t) => { + const f = await schedules(t) + const key = `${VM_TEST_SCHEDULE_PREFIX}${live.id}` + await f.manager.set(key, { ...f.manager.get(key), revision: 'changed-by-other-controller', enabled: false }) + await f.first._run(live.id) + assert.equal(f.calls(), 0) +}) + +test('late VM test results cannot replace a concurrently changed schedule', async (t) => { + const f = await schedules(t) + const key = `${VM_TEST_SCHEDULE_PREFIX}${live.id}` + const old = f.settings.get(key) + f.first.runTest = async () => { + await f.manager.set(key, { ...old, revision: 'changed-by-other-controller', prompt: 'new prompt' }) + return { ok: true } + } + await f.first._run(live.id) + await f.manager.refresh() + assert.equal(f.manager.get(key).prompt, 'new prompt') + assert.equal(f.manager.get(key).revision, 'changed-by-other-controller') +}) + +test('VM schedule result SQL patches only the matching revision and preserves other fields', async (t) => { + const f = await schedules(t) + const { VmTestScheduleResultsRepo } = await import('../../src/lib/db/repos/vm-test-schedule-results-repo.mjs') + const repo = new VmTestScheduleResultsRepo(f.db) + const key = `${VM_TEST_SCHEDULE_PREFIX}${live.id}` + const before = f.settings.get(key) + const result = { last_run_at: new Date().toISOString(), last_ok: true, last_duration_ms: 3, last_error: null } + assert.equal(await repo.record(live.id, before.revision, result), true) + await f.manager.refresh() + assert.equal(f.manager.get(key).prompt, before.prompt) + await f.manager.set(key, { ...f.manager.get(key), revision: 'new-revision', prompt: 'new prompt' }) + assert.equal(await repo.record(live.id, before.revision, { ...result, last_ok: false }), false) + await f.manager.refresh() + assert.equal(f.manager.get(key).prompt, 'new prompt') + assert.equal(f.manager.get(key).last_ok, true) +}) From 3b3014d028bfe9a2beadfcfe6d666c7329300277 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:30:00 +0800 Subject: [PATCH 161/473] docs: record verified shared-state and routing checkpoint --- .../2026-10-21-kubernetes-control-plane.md | 13 +- ...-21-kubernetes-shared-snapshots-results.md | 53 ++++ .../shared-snapshots/verification.json | 255 ++++++++++++++++++ 3 files changed, 318 insertions(+), 3 deletions(-) create mode 100644 docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md create mode 100644 docs/plans/evidence/kubernetes/shared-snapshots/verification.json diff --git a/docs/plans/2026-10-21-kubernetes-control-plane.md b/docs/plans/2026-10-21-kubernetes-control-plane.md index 52a018c09..471287072 100644 --- a/docs/plans/2026-10-21-kubernetes-control-plane.md +++ b/docs/plans/2026-10-21-kubernetes-control-plane.md @@ -24,7 +24,7 @@ ## 为什么暂不开放多副本推理 -当前 key/user/账号/VM 并发与 RPM、Claude 座位、Codex 会话、同会话请求顺序、熔断及 OAuth single-flight 仍有进程内状态。AccountQuota 和 ProxyPool 的缓存快照存在跨实例覆盖问题。RuntimeSettings、路由、模型策略和 Sticky 映射也没有跨实例一致性协议。 +第一阶段发现的 key/user/账号/VM 并发与 RPM、设备座位、同会话顺序、Sticky 和缓存覆盖问题，已有后续共享状态实现。当前仍未完成执行端终止证明与自动接管、后台任务生产接线、一次性认证状态、全局维护、VM/凭据文件共享归属及所有管理入口的协调。共享熔断和 OAuth single-flight 也需继续核实。 更关键的是，远端执行取消目前是 best-effort。控制面失联后，不能仅因租约过期就重新分配仍可能执行中的账号/VM 容量。安全的多活需要执行端 fencing 或可确认终止的 orphan 处理协议。活凭据、VM 文件和 Pod 本地 socket 的归属也必须先明确。 @@ -68,6 +68,13 @@ `fb90450d` 已加入账号/VM/Fable/RPM 的共享准入、设备座位与 generation、发送前实际 kernel request ID 绑定，以及未知执行保留和停止重试。设备预算按账号跨 VM 计算。standalone 保留原同步调度路径。详细结果见 [共享执行验证](2026-10-21-kubernetes-execution-results.md)。全量 unit 为 3493 pass、7 fail、10 skip；全量 E2E 为 103 pass、17 fail、1 skip。失败差异和基线依据均在报告中列出。 -当前队列仍为进程内队列。同会话顺序和 Sticky 的跨副本条件更新未完成。HTTP/SSE EOF 终态还依赖正常 kernel 协议，尚无执行端 fencing；控制面死亡后的占位没有自动回收。独占锁和单副本限制继续保留。 +在 `fb90450d` 检查点，等待队列为进程内队列，同会话顺序和 Sticky 条件更新尚未实现；后续状态见下文共享快照增量。HTTP/SSE EOF 终态仍依赖正常 kernel 协议，尚无执行端 fencing；控制面死亡后的占位没有自动回收。独占锁和单副本限制继续保留。 -下一阶段需优先拆除整行快照覆盖：AccountQuota 的计数改为 PG 原子增量，响应头/probe 改为有数据库观测顺序的字段补丁，身份与并发/RPM 配置使用管理补丁。`panel-api` 的显示检查、Fable probe、手工清理以及 `panel-routes` 的身份修改也会调用整行 `save`，需要同时覆盖。保留已有 bigint 精度和 `countRequest:false` 行为。ProxyPool 的 probe 当前仍调用 `replaceAll` 删除重建代理表，需要拆成目标行配置、健康、绑定及删除操作，并校验 probe 对应的配置版本。 +该检查点之后已实施的快照改造起点为：AccountQuota 的计数改为 PG 原子增量，响应头/probe 改为有数据库观测顺序的字段补丁，身份与并发/RPM 配置使用管理补丁。`panel-api` 的显示检查、Fable probe、手工清理以及 `panel-routes` 的身份修改也会调用整行 `save`，需要同时覆盖。保留已有 bigint 精度和 `countRequest:false` 行为。ProxyPool 的 probe 当前仍调用 `replaceAll` 删除重建代理表，需要拆成目标行配置、健康、绑定及删除操作，并校验 probe 对应的配置版本。 + + +## 共享快照与请求接线增量 + +固定源码 `6732a8648a4a57c6483abc7f685fce4d8cb28946` 已在独立检出完成全量检查：unit 3658 pass、7 fail、10 skip；E2E 104 pass、17 fail、1 skip。全部失败名称及断言文本与上一轮记录一致。AccountQuota/ProxyPool/SchedulingRuntime 字段更新、Sticky PG 条件更新与墓碑、共享同会话 FIFO、Codex 执行准入、PG 路由 revision 及准入事务校验均已接入。后续 Search 和面板路由状态接口另有定向验证。 + +结果与范围见 [共享状态验证报告](2026-10-21-kubernetes-shared-snapshots-results.md)。控制面仍为单活动副本。后台任务接线、一次性认证状态、全局维护、所有 VM 管理入口、执行端 fencing 和多节点运行验收继续进行；没有因此关闭 #648。 diff --git a/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md b/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md new file mode 100644 index 000000000..4241c1375 --- /dev/null +++ b/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md @@ -0,0 +1,53 @@ +# Kubernetes 共享状态与请求接线验证（#648，进行中） + +固定源码提交：`6732a8648a4a57c6483abc7f685fce4d8cb28946`。本轮在独立检出执行全量检查，未部署当前容器。控制面仍使用独占保护和单活动副本；这些检查尚未满足 #648 的多节点扩缩容与异常退出验收。 + +## 已实现的增量 + +- 账号计数使用 PG 原子增量。身份和策略使用字段补丁。响应头、OAuth usage、probe、worker heartbeat 使用在远端 I/O 前取得的数据库观测序号，拒绝迟到观测；本地异步序号分配也保持调用顺序。账号缓存刷新保留当前进程的活动请求状态。 +- 代理配置、绑定和健康状态按行提交。健康及地理位置结果校验配置版本，不再删除重建整张代理表。调用方等待持久提交；刷新读取已提交快照，不等待网络 probe。 +- SchedulingRuntime 按字段及模型合并运行状态。配额清理和冷却副作用使用 PG 条件更新的实际结果。旧观测被拒绝时，不修改 VM 限制和当前粘性会话。 +- 同会话请求使用 PG FIFO。当前 turn 只有在执行结束和容量释放后才交给下一请求。未知执行和 owner 异常退出保留占位，无 TTL 自动回收。等待队列中的 owner 死亡也尚未自动清理。 +- Sticky 使用 PG generation 条件更新和解绑墓碑。首次绑定及墓碑后的绑定都受版本约束。Claude 和 Codex 在实际发送前等待原生会话绑定；条件冲突不发送旧请求体。迟到完成不能恢复其他副本已迁移的绑定。 +- Codex 复用持久账号/VM/RPM 准入、作业登记和同会话 FIFO。每个实际 kernel hop 有不同 request ID。截断、取消和数据库完成/释放失败保留容量，停止重试。Kubernetes 模式暂要求凭据文件只有一个带稳定 `chatgpt_account_id` 的账号；不能确认内核实际选择账号时拒绝发送。 +- 路由配置以 PG 中带 revision 的配置为准。首次从只读文件初始化，之后不以 Pod 文件覆盖 PG。互补 patch 在 PG 最新值上合并。同步消费者读取已提交快照；请求前和定时刷新更新本地策略。候选携带产生它的 revision，在准入事务中先校验配置，再取得账号/VM/设备锁。配置写入会等待该准入事务提交。 +- VM 变更使用持久操作记录，与推理准入互斥。已接入凭据导入、credits 查询/兑换、reset 和 clearcredentials 四条调用链。错误和未知远端结果保留操作记录。它不覆盖其他 VM 管理入口，也没有改变远端 Claude 凭据清理的既有行为。 + +## 全量检查 + +环境为 Node.js 22.23.2、任务专用 PostgreSQL 17.11、每测试随机 schema、合成 HTTP/Unix socket 上游。测试使用独立制品副本；没有操作真实 VM 或模型供应商。 + +| 检查 | `6732a864` 的结果 | +| --- | --- | +| 全量 unit | 318 文件，3675 项：3658 通过、7 失败、10 跳过 | +| 全量 E2E | 28 文件，122 项：104 通过、17 失败、1 跳过 | +| 本轮 78 个 MJS 文件语法与 Biome | 通过 | +| `git diff --check fb90450d..6732a864` | 通过 | + +7 项 unit 和 17 项 E2E 的失败名称及断言文本，逐项匹配此前 `fb90450d06fd3aaaee8a0a4b52aca13d2f675431` 的记录。未把这些失败改成通过或跳过。例如 unit 实际 TTL 为 `1h`、预期 `undefined`；E2E Responses 实际为 400 `protocol_not_allowed`、预期 200。 + +本次没有重新执行原始 `eee8f6e` 的整套测试。7 项 unit 在原始源码上的复核、17 项 E2E 的既有对照来源，见 [上一轮共享执行报告](2026-10-21-kubernetes-execution-results.md)。结构化统计及错误摘要哈希见 [本轮证据](evidence/kubernetes/shared-snapshots/verification.json)。完整测试日志保留于任务临时目录，证据包含其 SHA-256。 + +## 后续提交的定向检查 + +这些结果没有替代后续源码的全量回归： + +| 提交 | 检查范围与结果 | +| --- | --- | +| `12f19167ae5b2544add8e2a719993b5c972c644e` | PG 冷却副作用、共享调度与会话保护：170/170 通过 | +| `06d841cd5fb5d81366cf0a7684b408fcd7368ac3` | 配置 revision、准入事务和锁顺序：44/44 通过 | +| `c5f6ff3d1e3728c74ceb630929bc876298322a2a` | Codex 与 Search 生命周期：145/145 通过 | +| `97b4a087ae486814ffdc912f7b662fdeb2e387ff` | 路由面板接口与 VM 变更：34/34 通过 | + +前两项提交位于全量检查点之前。后两项在其后。Search 直接 HTTP 请求只有在响应体读取完成后才结束作业；401/403 重发前完成旧作业并使用新 ID。响应体截断、取消、超时及 PG 故障保留容量。Search 不增加原协议没有定义的同会话队列。 + +面板路由更新显式返回已提交 revision 和 application 状态。VM/kernel 投影未执行时返回 pending。PG 提交失败不回退到可能过期的本地配置，也不在提交前切换执行端引擎。 + +## 未完成和未验证 + +- 后台任务共享执行的生产接线、跨副本一次性 OAuth/终端 ticket 状态仍在后续增量中。 +- 全局维护与备份恢复、VM/凭据文件的权威来源、其他 VM 管理入口、共享熔断和跨 VM 组织兑换等尚未完成。 +- VM/kernel 远端配置投影为 pending。PG 配置提交不证明执行端已更新。 +- 当前 HTTP/SSE 结束标记依赖正常执行端协议。执行端 epoch 校验、旧作业终止证明和 owner 死亡后的自动恢复尚未实现。未知占位没有自动回收。 +- 两个不同 Kubernetes 节点上的活动副本、HPA、PodDisruptionBudget、滚动更新、缩容和异常退出接管尚未验证。旧镜像和单节点 k3s 证据不覆盖本轮代码。 +- Draft PR #651 的已发布分支仍是早期部署基础。后续本地增量尚未 push；按用户规则等待 push 确认。 diff --git a/docs/plans/evidence/kubernetes/shared-snapshots/verification.json b/docs/plans/evidence/kubernetes/shared-snapshots/verification.json new file mode 100644 index 000000000..4c198382c --- /dev/null +++ b/docs/plans/evidence/kubernetes/shared-snapshots/verification.json @@ -0,0 +1,255 @@ +{ + "commit": "6732a8648a4a57c6483abc7f685fce4d8cb28946", + "node_version": "v22.23.2", + "postgresql_version": "17.11", + "execution": "detached worktree; per-test random PostgreSQL schema; synthetic HTTP/Unix-socket upstreams; copied payload binaries", + "comparison_commit": "fb90450d06fd3aaaee8a0a4b52aca13d2f675431", + "suites": { + "unit": { + "files": 318, + "totals": { + "tests": 3675, + "pass": 3658, + "fail": 7, + "cancelled": 0, + "skipped": 10 + }, + "failures": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "error_sha256": "1afab608e2476a88783807d547c9b239ad4ac831679731a3ed4d44812b0c31d2", + "same_error_as_comparison": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "same_error_as_comparison": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "same_error_as_comparison": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "error_sha256": "7862d1144e6748d63aa6e98dabf5a9fa7d8d1582f7fbfe851d978eb15f6d5c92", + "same_error_as_comparison": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "error_sha256": "81b2ee3a4bb8b1ab4f351bede4398032dc9637c683b82b146630fd6857e94c2e", + "same_error_as_comparison": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "error_sha256": "01732f4307f58f479e9b3d8329d92a93d9c748c58028276022257d3645e77e87", + "same_error_as_comparison": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "same_error_as_comparison": true + } + ], + "log_sha256": "c7d6f161487d48fa11292ae4b9c7fd8e127b3107528f394879ec85198dbeb454" + }, + "e2e": { + "files": 28, + "totals": { + "tests": 122, + "pass": 104, + "fail": 17, + "cancelled": 0, + "skipped": 1 + }, + "failures": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "error_sha256": "a25cb59468e4decc45b1d8b3a66b0a4fbd2195af9f498da39aa83e92169825b5", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "error_sha256": "84e06146cf59485c3b46e05070f6a9a184ecb010a57ef8ab1b62a3ebae90c36a", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "error_sha256": "dfd84d60931072a864eb94f48ce13d4223e963ae919898236f1859bedc8d917e", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "error_sha256": "6ea819807eaee98e6a4c38817c9f011fb426e6a5ff499fa441d8f7bb07371461", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "error_sha256": "c23279127cad70a092d0cc7bb6ac400e05aff03e4614d1eaaa40252878c221c2", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "error_sha256": "97ae4156650fa0de2f613aa7753666781545cb2fe7818350a6df904ab696cdbc", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "error_sha256": "a2c6319267f1b752b63d96a41e15b1a5133952e88f0c145bc96dda53fd31cbe5", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "error_sha256": "ba0ac5e90ea008839723c328e8b172228820837ab895e425d2190c1aa5caa708", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "error_sha256": "8353154a58e84007d91c2d217c516dc771e359ddb43d527c7db6734ab5cff6ca", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "error_sha256": "085c13ab1e54fb324bce5520b12006fbf7f84b4a69a1c5cf9e09727b5b33da99", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "error_sha256": "0be25feaedaa697c09835d15c8cfdae694de2710330cadfadcbdfa146625a099", + "same_error_as_comparison": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "same_error_as_comparison": true + } + ], + "log_sha256": "8da214bdde5f89e9750d06acb552d126301daa48886510dddb7abe7090956213" + } + }, + "additional_targeted_checks": [ + { + "commit": "12f19167ae5b2544add8e2a719993b5c972c644e", + "scope": "cooldown side effects and shared scheduling", + "files": 1, + "totals": { + "tests": 170, + "pass": 170, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "8e72bff0b2339e31ee3a2b345fe513d2f589efade745ad9e2b550ac2393e2ac9" + }, + { + "commit": "06d841cd5fb5d81366cf0a7684b408fcd7368ac3", + "scope": "routing revision and admission lock order", + "files": 1, + "totals": { + "tests": 44, + "pass": 44, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "25cfb96af1d8b5050fb4a583490af5fa21bf1da741f6b2b3b22a30328fadb652" + }, + { + "commit": "c5f6ff3d1e3728c74ceb630929bc876298322a2a", + "scope": "Codex and Search execution lifecycles", + "files": 1, + "totals": { + "tests": 145, + "pass": 145, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "eeea710eede7aa9be04ad7d110a0fb6b8680a81360e4849ac5bedcdebf6234a0" + }, + { + "commit": "97b4a087ae486814ffdc912f7b662fdeb2e387ff", + "scope": "shared routing panel and mutation integration", + "files": 1, + "totals": { + "tests": 34, + "pass": 34, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "166030a23622ee8b5ad65d71beec6c14dcef9c1065c8339d566d03771882f41f" + } + ], + "limitations": [ + "Full suites are not all passing; all 7 unit and 17 E2E error texts match comparison records.", + "Later targeted checks are not a full-suite verification of later commits.", + "Kubernetes control plane remains single active replica with exclusive guard.", + "No current container deployment, multi-node inference, executor fencing, or automatic orphan recovery was verified.", + "Background task coordination and one-time OAuth/ticket state are separate ongoing increments.", + "VM files/credentials and global backup restore maintenance are not yet coordinated across all entry points.", + "Remote VM/kernel routing projection is pending; configuration commit does not prove it was applied." + ], + "static_checks": { + "commit": "6732a8648a4a57c6483abc7f685fce4d8cb28946", + "mjs_files": 78, + "node_syntax_errors": 0, + "biome_exit_code": 0, + "diff_check_exit_code": 0, + "diff_base": "fb90450d06fd3aaaee8a0a4b52aca13d2f675431", + "log_sha256": "6f4fbd68da9299121172e18f8c28ee6f3bbcd5d47b8586e291fd8966d084ed48" + } +} From 87d8090f659eee6b8b22546ee8239b3b057d2255 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:31:34 +0800 Subject: [PATCH 162/473] test(auth): reproduce process-local PKCE and shell ticket state --- test/unit/shared-auth-state.test.mjs | 162 +++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 test/unit/shared-auth-state.test.mjs diff --git a/test/unit/shared-auth-state.test.mjs b/test/unit/shared-auth-state.test.mjs new file mode 100644 index 000000000..e8b69ed35 --- /dev/null +++ b/test/unit/shared-auth-state.test.mjs @@ -0,0 +1,162 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' +import { createSlotShell } from '../../src/lib/vm/slot-shell.mjs' + +let instance = 0 + +async function fixture(t) { + const names = ['VM2API_CONTROL_PLANE_MODE', 'VM2API_DB_SECRET', 'KIN_FAKE_SESSION_OAUTH'] + const previous = Object.fromEntries(names.map((name) => [name, process.env[name]])) + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + process.env.VM2API_DB_SECRET = 'synthetic-shared-pkce-key' + process.env.KIN_FAKE_SESSION_OAUTH = '1' + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-auth-')) + t.after(async () => { + await other.close() + fs.rmSync(root, { recursive: true, force: true }) + for (const name of names) { + if (previous[name] === undefined) delete process.env[name] + else process.env[name] = previous[name] + } + }) + const controllers = await Promise.all( + [0, 1].map(async () => { + const suffix = `shared-auth-${++instance}` + return { + claude: await import(`../../src/lib/oauth/oauth-auth-url.mjs?${suffix}`), + codex: await import(`../../src/lib/oauth/codex-oauth.mjs?${suffix}`), + } + }), + ) + return { db, other, root, controllers } +} + +function callbackState(generated, kind) { + const outer = new URL(generated.auth_url) + return (kind === 'codex' ? new URL(outer.searchParams.get('authorize_url')) : outer).searchParams.get('state') +} + +for (const kind of ['claude', 'codex']) { + test(`${kind} PKCE survives a controller change and permits only one successful exchange`, async (t) => { + const fx = await fixture(t) + const [a, b] = fx.controllers.map((controller) => controller[kind]) + const generate = kind === 'codex' ? a.generateCodexAuthUrl : a.generateAuthUrl + const exchange = kind === 'codex' ? b.exchangeCodexAuthCode : b.exchangeAuthCode + const generated = await generate({ vmId: 'vm-auth', proxyUrl: '', db: fx.db }) + const args = { + sessionId: generated.session_id, + code: `synthetic-code#${callbackState(generated, kind)}`, + vmId: 'vm-auth', + db: fx.other, + fetchImpl: async () => ({ + status: 200, + json: async () => ({ access_token: 'synthetic-access', refresh_token: 'synthetic-refresh' }), + }), + } + const result = await exchange(args) + assert.ok(result.access_token) + await assert.rejects(exchange(args), { code: 'session_expired' }) + }) +} + +test('a wrong Codex VM or state cannot consume another controller PKCE session', async (t) => { + const fx = await fixture(t) + const a = fx.controllers[0].codex + const b = fx.controllers[1].codex + const generated = await a.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '', db: fx.db }) + let calls = 0 + const args = { + sessionId: generated.session_id, + code: `synthetic#${callbackState(generated, 'codex')}`, + vmId: 'vm-wrong', + db: fx.other, + fetchImpl: async () => { + calls += 1 + return { + status: 200, + json: async () => ({ access_token: 'synthetic-access', refresh_token: 'synthetic-refresh' }), + } + }, + } + await assert.rejects(b.exchangeCodexAuthCode(args), { code: 'session_vm_mismatch' }) + await assert.rejects(b.exchangeCodexAuthCode({ ...args, vmId: 'vm-auth', code: 'synthetic#wrong-state' }), { + code: 'state_mismatch', + }) + assert.equal(calls, 0) + await b.exchangeCodexAuthCode({ ...args, vmId: 'vm-auth' }) + assert.equal(calls, 1) +}) + +test('Kubernetes PKCE refuses issue without the configured database encryption key', async (t) => { + const fx = await fixture(t) + delete process.env.VM2API_DB_SECRET + const previous = process.env.KIN_DB_SECRET + delete process.env.KIN_DB_SECRET + try { + await assert.rejects( + async () => fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '', db: fx.db }), + { + code: 'oauth_session_encryption_required', + }, + ) + } finally { + if (previous !== undefined) process.env.KIN_DB_SECRET = previous + } +}) + +test('node shell tickets cross controllers and a wrong resource does not consume them', async (t) => { + const fx = await fixture(t) + const managers = [fx.db, fx.other].map((db) => { + const manager = new ClusterManager({ repo: { db }, dataDir: fx.root }) + for (const id of ['node-a', 'node-b']) manager.links.set(id, { state: 'ready', client: {} }) + return manager + }) + const { ticket } = await managers[0].issueShellTicket('node-a') + assert.equal(await managers[0].consumeShellTicket(ticket, 'node-b'), false) + assert.equal(await managers[1].consumeShellTicket(ticket, 'node-a'), true) + assert.equal(await managers[0].consumeShellTicket(ticket, 'node-a'), false) +}) + +test('slot login tickets cross controllers while a wrong VM does not consume them', async (t) => { + const fx = await fixture(t) + fs.mkdirSync(path.join(fx.root, 'vms')) + for (const id of ['vm-a', 'vm-b']) fs.writeFileSync(path.join(fx.root, 'vms', `${id}.json`), JSON.stringify({ id })) + const shells = [fx.db, fx.other].map((db) => + createSlotShell({ + projectRoot: fx.root, + db, + authLoginReady: async () => {}, + upgradeLoginReady: async () => { + throw new Error('synthetic terminal is unavailable') + }, + }), + ) + t.after(async () => Promise.all(shells.map((shell) => shell.drain()))) + const { ticket } = await shells[0].issueTicket('vm-a', { login: true }) + async function upgrade(shell, id) { + const writes = [] + assert.equal( + shell.handleUpgrade( + { url: `/api/panel/vms/${id}/shell?ticket=${ticket}&login=1` }, + { + write: (data) => writes.push(String(data)), + destroy() {}, + }, + Buffer.alloc(0), + ), + true, + ) + for (let i = 0; !writes.length && i < 100; i++) await new Promise((resolve) => setTimeout(resolve, 5)) + return writes[0] + } + assert.match(await upgrade(shells[0], 'vm-b'), /^HTTP\/1\.1 401 /) + assert.match(await upgrade(shells[1], 'vm-a'), /^HTTP\/1\.1 409 /) + assert.match(await upgrade(shells[0], 'vm-a'), /^HTTP\/1\.1 401 /) +}) From 6806270cd28959e28f3295a21c656346b64de753 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:34:37 +0800 Subject: [PATCH 163/473] test: cover shared schedule discovery and migration 013 restores --- test/unit/backup-service.test.mjs | 7 +- .../unit/shared-background-producers.test.mjs | 190 ++++++++++++++++++ 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index a6ec474e6..4150827d1 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,7 +79,7 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; @@ -106,16 +106,17 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'execution_jobs', 'session_turns', 'vm_operations', + 'background_task_runs', ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 12) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 13) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; diff --git a/test/unit/shared-background-producers.test.mjs b/test/unit/shared-background-producers.test.mjs index 112f3d8d6..82caace5e 100644 --- a/test/unit/shared-background-producers.test.mjs +++ b/test/unit/shared-background-producers.test.mjs @@ -11,6 +11,8 @@ import { createCredentialRefreshMonitor } from '../../src/lib/oauth/credential-r import { createUsageProbeMonitor } from '../../src/lib/oauth/usage-probe-monitor.mjs' import { BackupService } from '../../src/lib/admin/backup-service.mjs' import { VmTestScheduleRunner, VM_TEST_SCHEDULE_PREFIX } from '../../src/lib/admin/vm-test-schedule.mjs' +import { SharedRoutingConfig } from '../../src/lib/admin/shared-routing-config.mjs' +import { RoutingConfigRepo } from '../../src/lib/db/repos/routing-config-repo.mjs' const live = { id: 'vm-synthetic', @@ -93,6 +95,61 @@ test('credential refresh plans deduplicate across monitors and never persist the assert.equal((await f.other.prepare('SELECT shared_result FROM background_task_runs').get()).shared_result, null) }) +test('a normal credential error remains unknown and cannot be retried by another monitor', async (t) => { + const f = await fixture(t) + let calls = 0 + const opts = { + listTargets: () => [live], + refreshOne: async () => { + calls++ + return { ok: false, refresh_class: 'retryable' } + }, + } + await createCredentialRefreshMonitor({ ...opts, sharedTasks: f.first }).runOnce() + await createCredentialRefreshMonitor({ ...opts, sharedTasks: f.second }).runOnce() + assert.equal(calls, 1) + const row = await f.other.prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.ok(row.unknown_at) + assert.equal(row.finished_at, null) +}) + +test('Kubernetes health uses the default shared service and refreshes routing before executing', async (t) => { + const f = await fixture(t) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'synthetic-health-routing-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const file = path.join(root, 'routing.json') + fs.writeFileSync(file, JSON.stringify({ health_probe: { enabled: true, real: { prompt: 'old prompt' } } })) + const previousMode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (previousMode === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previousMode + }) + let monitor + const routing = new SharedRoutingConfig({ + db: f.db, + file, + normalize: (value) => value, + merge: (current, patch) => ({ ...current, ...patch }), + apply: (config) => monitor?.setConfig(config.health_probe, { restart: false }), + }) + await routing.init() + monitor = createHealthProbeMonitor({ + config: { enabled: true, real: { prompt: 'old prompt' } }, + listTargets: () => [live], + runChat: async (_vm, config) => { + assert.equal(config.prompt, 'new prompt') + return { ok: true, text: 'synthetic', body: { id: 'synthetic-message' } } + }, + }) + await new RoutingConfigRepo(f.other).patch((current) => ({ + ...current, + health_probe: { enabled: true, real: { prompt: 'new prompt' } }, + })) + assert.equal((await monitor.runRealProbe()).ok, true) + assert.equal((await f.other.prepare('SELECT count(*) AS n FROM background_task_runs').get()).n, 1) +}) + test('usage timer and event probes share one persisted VM task plan across monitors', async (t) => { const f = await fixture(t) let calls = 0 @@ -297,3 +354,136 @@ test('VM schedule result SQL patches only the matching revision and preserves ot assert.equal(f.manager.get(key).prompt, 'new prompt') assert.equal(f.manager.get(key).last_ok, true) }) + +test('a controller discovers remote schedule creation, updates and deletion using its lifecycle poll', async (t) => { + const f = await fixture(t) + const polls = [] + const originalInterval = globalThis.setInterval + t.mock.method(globalThis, 'setInterval', (callback, ms, ...args) => { + if (ms !== 1000) return originalInterval(callback, ms, ...args) + const timer = { callback, unref() {} } + polls.push(timer) + return timer + }) + const originalClear = globalThis.clearInterval + t.mock.method(globalThis, 'clearInterval', (timer) => { + if (timer?.callback) timer.stopped = true + else originalClear(timer) + }) + const settings = new RuntimeSettings(f.db) + await loadSettingsState(f.other) + const manager = new RuntimeSettings(f.other) + let calls = 0 + const create = (store, sharedTasks) => + new VmTestScheduleRunner({ + projectRoot: '/synthetic-missing', + settings: store, + sharedTasks, + list: () => [live], + runTest: async () => { + calls++ + return { ok: true } + }, + }) + const writer = create(manager, f.second) + const observer = create(settings, f.first) + await observer.start() + t.after(() => observer.stop()) + assert.equal(polls.length, 1) + const config = { + enabled: true, + frequency: 'daily', + time: '09:00', + timezone: 'UTC', + model: 'claude-haiku-4-5', + prompt: 'synthetic', + max_tokens: 16, + } + await writer.save(live.id, config) + assert.equal(observer.jobs.has(live.id), false) + await polls[0].callback() + assert.equal(observer.jobs.get(live.id).getPattern(), '0 9 * * *') + let clock = await f.first.now() + f.first.now = async () => clock + await observer._run(live.id) + await writer.stop() + clock += 24 * 3600_000 + await observer._run(live.id) + assert.equal(calls, 2) + await writer.save(live.id, { ...config, time: '12:30' }) + await polls[0].callback() + assert.equal(observer.jobs.get(live.id).getPattern(), '30 12 * * *') + await writer.remove(live.id) + await polls[0].callback() + assert.equal(observer.jobs.has(live.id), false) + await observer.stop() + assert.equal(polls[0].stopped, true) +}) + +test('a late discovery cannot replace a job accepted by a newer discovery', async (t) => { + const f = await schedules(t) + const oldRefresh = f.settings.refresh.bind(f.settings) + const entered = deferred() + const finish = deferred() + let reads = 0 + f.settings.refresh = async () => { + await oldRefresh() + if (++reads === 1) { + entered.resolve() + await finish.promise + } + } + const older = f.first.reload() + await entered.promise + const key = `${VM_TEST_SCHEDULE_PREFIX}${live.id}` + await f.manager.set(key, { ...f.manager.get(key), revision: 'new-revision', time: '12:30', cron: '30 12 * * *' }) + await f.first.reload() + const accepted = f.first.jobs.get(live.id) + finish.resolve() + await older + assert.equal(f.first.jobs.get(live.id), accepted) +}) + +test('stop awaits its pending discovery and does not publish jobs after stopping', async (t) => { + const f = await schedules(t) + const refresh = f.settings.refresh.bind(f.settings) + const entered = deferred() + const finish = deferred() + f.settings.refresh = async () => { + await refresh() + entered.resolve() + await finish.promise + } + const discovering = f.first.reload() + await entered.promise + let stopped = false + const stopping = Promise.resolve(f.first.stop()).then(() => { + stopped = true + }) + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(stopped, false) + } finally { + finish.resolve() + await Promise.all([discovering, stopping]) + } + assert.equal(f.first.jobs.size, 0) +}) + +test('removing or revising a schedule never frees an unknown execution claim', async (t) => { + const f = await schedules(t) + f.first.runTest = async () => ({ ok: false, error: 'synthetic unknown' }) + await f.first._run(live.id) + const key = `${VM_TEST_SCHEDULE_PREFIX}${live.id}` + const plan = f.settings.get(key) + await f.manager.remove(key) + await f.first.reload() + await f.manager.set(key, { ...plan, revision: 'recreated-plan' }) + await f.second.reload() + await f.second._run(live.id) + assert.equal(f.calls(), 0) + assert.equal( + (await f.db.prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get()).n, + 1, + ) +}) From 36b3c0303f47f00da4f830f3b433fee1acb7caf0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:36:57 +0800 Subject: [PATCH 164/473] test: specify global maintenance and restore coordination --- test/unit/global-maintenance.test.mjs | 301 ++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 test/unit/global-maintenance.test.mjs diff --git a/test/unit/global-maintenance.test.mjs b/test/unit/global-maintenance.test.mjs new file mode 100644 index 000000000..5f20229c2 --- /dev/null +++ b/test/unit/global-maintenance.test.mjs @@ -0,0 +1,301 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import { createDatabase, getDb, quoteIdentifier } from '../../src/lib/db/database.mjs' + +async function fixture(t, { acknowledge = true } = {}) { + const { GlobalMaintenanceCoordinator, coordinationSchemaFor } = await import( + '../../src/lib/db/global-maintenance.mjs' + ) + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const schema = coordinationSchemaFor(db.schema) + const first = new GlobalMaintenanceCoordinator(db, { ownerId: 'pod-a' }) + const second = new GlobalMaintenanceCoordinator(peer, { ownerId: 'pod-b' }) + const operations = [] + t.after(async () => { + await Promise.allSettled(operations.map((operation) => operation.close())) + await Promise.allSettled([first.stop(), second.stop()]) + await db.pool.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(schema)} CASCADE`) + }) + await first.start() + await second.start() + if (acknowledge) { + await first.acknowledgeEpoch(1) + await second.acknowledgeEpoch(1) + } + return { db, peer, first, second, schema, operations, coordinationSchemaFor } +} + +async function waitForPhase(coordinator, phase) { + for (let attempt = 0; attempt < 200; attempt++) { + if ((await coordinator.getState()).phase === phase) return + await delay(5) + } + assert.fail(`maintenance did not reach ${phase}`) +} + +test('coordination DDL belongs only to a private schema derived from this application schema', async (t) => { + const { db, schema, coordinationSchemaFor } = await fixture(t) + assert.notEqual(schema, 'public') + assert.notEqual(schema, db.schema) + assert.ok(Buffer.byteLength(schema) <= 63) + assert.notEqual(coordinationSchemaFor('another_tenant'), schema) + const namespace = ( + await db.pool.query( + `SELECT pg_get_userbyid(nspowner) AS owner, + current_user AS expected_owner, + EXISTS (SELECT 1 FROM aclexplode(nspacl) WHERE grantee = 0) AS public_access + FROM pg_namespace WHERE nspname = $1`, + [schema], + ) + ).rows[0] + assert.equal(namespace.owner, namespace.expected_owner) + assert.equal(namespace.public_access, false) + const tables = ( + await db.pool.query( + `SELECT table_name FROM information_schema.tables + WHERE table_schema = $1 ORDER BY table_name`, + [schema], + ) + ).rows.map((row) => row.table_name) + assert.deepEqual(tables, ['activities', 'control', 'members']) + assert.equal((await db.pool.query('SELECT to_regclass($1) AS name', [`${db.schema}.activities`])).rows[0].name, null) +}) + +test('each replica acknowledges its loaded epoch before durable admission of master or endpoint work', async (t) => { + const { db, first, second, schema } = await fixture(t, { acknowledge: false }) + await assert.rejects(first.admit({ kind: 'master_request', requestId: 'master-1' }), { + code: 'maintenance_epoch_unbound', + }) + await first.acknowledgeEpoch(1) + await second.acknowledgeEpoch(1) + const gate = await first.admit({ kind: 'api_endpoint', requestId: 'endpoint-1' }) + assert.equal(gate.epoch, 1) + const row = ( + await db.pool.query(`SELECT * FROM ${quoteIdentifier(schema)}.activities WHERE id = $1`, [gate.activityId]) + ).rows[0] + assert.equal(row.owner_id, 'pod-a') + assert.equal(row.request_id, 'endpoint-1') + assert.equal(row.completed_at, null) + await gate.complete({ ended: true, reason: 'response_eof' }) + assert.equal( + ( + await db.pool.query( + `SELECT completed_at IS NOT NULL AS ended FROM ${quoteIdentifier(schema)}.activities WHERE id = $1`, + [gate.activityId], + ) + ).rows[0].ended, + true, + ) +}) + +test('DRAINING rejects new work on a busy replica while its admitted scope can finish SQL', async (t) => { + const { db, first, second, operations } = await fixture(t) + const gate = await second.admit({ kind: 'request', requestId: 'sse-existing' }) + const pending = first.beginMaintenance({ kind: 'restore', timeoutMs: 5000 }) + await waitForPhase(second, 'draining') + await assert.rejects(second.admit({ kind: 'request' }), { code: 'database_maintenance' }) + await gate.run(async () => { + second.assertActivity() + await db.prepare("INSERT INTO settings (key, value) VALUES ('maintenance-existing-write', 'true')").run() + }) + await gate.complete({ ended: true }) + const operation = await pending + operations.push(operation) + assert.ok(await db.prepare("SELECT key FROM settings WHERE key = 'maintenance-existing-write'").get()) +}) + +test('uncertain remote work releases its local lock but durably prevents destructive maintenance', async (t) => { + const { db, first, second, schema } = await fixture(t) + const gate = await second.admit({ kind: 'api_endpoint', requestId: 'truncated-upstream' }) + await gate.complete({ ended: false, reason: 'transport_error' }) + const row = ( + await db.pool.query(`SELECT * FROM ${quoteIdentifier(schema)}.activities WHERE id = $1`, [gate.activityId]) + ).rows[0] + assert.ok(row.uncertain_at) + assert.equal(row.completed_at, null) + await assert.rejects(first.beginMaintenance({ kind: 'restore', timeoutMs: 2000 }), (error) => { + assert.equal(error.code, 'maintenance_claims_active') + assert.equal(error.claims.activities, 1) + return true + }) + assert.equal((await second.getState()).phase, 'draining') + await gate.complete({ ended: true }) + assert.equal( + ( + await db.pool.query(`SELECT completed_at FROM ${quoteIdentifier(schema)}.activities WHERE id = $1`, [ + gate.activityId, + ]) + ).rows[0].completed_at, + null, + ) +}) + +test('every existing execution, key, VM, background and session ledger prevents restore', async (t) => { + const { db, first } = await fixture(t) + const cases = [ + [ + 'execution_admissions', + "INSERT INTO execution_admissions (id,owner_id,request_id,account_id,vm_id) VALUES ('claim','old','r','a','v')", + 'DELETE FROM execution_admissions', + ], + [ + 'execution_jobs', + "INSERT INTO execution_admissions (id,owner_id,request_id,account_id,vm_id,released_at) VALUES ('claim','old','r','a','v',clock_timestamp()); INSERT INTO execution_jobs(request_id,admission_id) VALUES ('job','claim')", + 'DELETE FROM execution_admissions', + ], + [ + 'api_key_admissions', + "INSERT INTO api_key_admissions (id,key_id,owner_id) VALUES ('claim','key','old')", + 'DELETE FROM api_key_admissions', + ], + [ + 'vm_operations', + "INSERT INTO vm_operations (id,vm_id,owner_id,kind) VALUES ('claim','v','old','restart')", + 'DELETE FROM vm_operations', + ], + [ + 'background_task_runs', + "INSERT INTO background_task_runs(task_key,execution_id,owner_id) VALUES ('task','claim','old')", + 'DELETE FROM background_task_runs', + ], + [ + 'session_turns', + "INSERT INTO session_turns(id,session_key,owner_id,state) VALUES ('claim','session','old','waiting')", + 'DELETE FROM session_turns', + ], + ] + for (const [kind, insert, cleanup] of cases) { + await db.exec(insert) + await assert.rejects(first.beginMaintenance({ kind: 'restore', timeoutMs: 2000 }), (error) => { + assert.equal(error.code, 'maintenance_claims_active', kind) + assert.equal(error.claims[kind], 1, kind) + return true + }) + await first.cancelDrain((await first.getState()).operationId) + await db.exec(cleanup) + } +}) + +test('restoring latch and epoch survive replacement of the entire application schema', async (t) => { + const { db, first, second, operations } = await fixture(t) + const operation = await first.beginMaintenance({ kind: 'restore' }) + operations.push(operation) + const state = await operation.beginRestore() + assert.equal(state.epoch, 2) + await db.pool.query(`DROP SCHEMA ${quoteIdentifier(db.schema)} CASCADE; CREATE SCHEMA ${quoteIdentifier(db.schema)}`) + assert.equal((await second.getState()).phase, 'restoring') + assert.equal((await second.getState()).epoch, 2) + await operation.close() + await assert.rejects(second.admit({ kind: 'request' }), { code: 'database_maintenance' }) + await assert.rejects(second.cancelDrain(operation.operationId), { code: 'maintenance_operation_changed' }) +}) + +test('losing the restore owner session never clears the durable restoring latch', async (t) => { + const { db, first, second, operations } = await fixture(t) + const operation = await first.beginMaintenance({ kind: 'restore' }) + operations.push(operation) + await operation.beginRestore() + await db.pool.query('SELECT pg_terminate_backend($1)', [operation.backendPid]) + await delay(10) + await assert.rejects(operation.beginRebind(), { code: 'maintenance_connection_lost' }) + await assert.rejects(second.admit({ kind: 'request' }), { code: 'database_maintenance' }) + assert.equal((await second.getState()).phase, 'restoring') +}) + +test('restore resumes only after every participating Pod acknowledges the new epoch', async (t) => { + const { first, second, operations } = await fixture(t) + const operation = await first.beginMaintenance({ kind: 'restore' }) + operations.push(operation) + await operation.beginRestore() + await operation.beginRebind() + await first.acknowledgeEpoch(2) + await assert.rejects(operation.resume(), { code: 'maintenance_rebind_pending' }) + await second.acknowledgeEpoch(2) + await operation.resume() + const gate = await second.admit({ kind: 'request' }) + assert.equal(gate.epoch, 2) + await gate.complete({ ended: true }) + // Backup/schema-DDL exclusion lasts until explicit close, after traffic resumes. + await assert.rejects(second.beginMaintenance({ kind: 'backup' }), { code: 'maintenance_busy' }) + await operation.close() +}) + +test('concurrent zero-to-one and one-to-zero transitions do not leak or briefly drop the Pod holder', async (t) => { + const { first, second, operations } = await fixture(t) + let gates = await Promise.all(Array.from({ length: 20 }, () => second.admit({ kind: 'request' }))) + for (let round = 0; round < 3; round++) { + const closing = gates.map((gate) => gate.complete({ ended: true })) + gates = await Promise.all(Array.from({ length: 20 }, () => second.admit({ kind: 'request' }))) + await Promise.all(closing) + } + const pending = first.beginMaintenance({ kind: 'backup', timeoutMs: 5000 }) + await waitForPhase(second, 'draining') + await assert.rejects(second.admit({ kind: 'request' }), { code: 'database_maintenance' }) + await Promise.all(gates.map((gate) => gate.complete({ ended: true }))) + const operation = await pending + operations.push(operation) + await operation.resume() +}) + +test('an inherited asynchronous scope cannot authorize SQL after its activity completed', async (t) => { + const { first } = await fixture(t) + const gate = await first.admit({ kind: 'request' }) + let release + const wait = new Promise((resolve) => { + release = resolve + }) + const child = gate.run(async () => { + await wait + assert.throws(() => first.assertActivity(), { code: 'maintenance_activity_closed' }) + }) + await gate.complete({ ended: true }) + release() + await child + assert.throws(() => gate.run(() => {}), { code: 'maintenance_activity_closed' }) +}) + +test('failed durable admission does not start work or retain an empty shared holder', async (t) => { + const { db, first, second, schema, operations } = await fixture(t) + await db.pool.query(`CREATE FUNCTION ${quoteIdentifier(schema)}.reject_activity() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic admission failure' USING ERRCODE='P6483'; END $$; + CREATE TRIGGER reject_activity BEFORE INSERT ON ${quoteIdentifier(schema)}.activities + FOR EACH ROW EXECUTE FUNCTION ${quoteIdentifier(schema)}.reject_activity()`) + await assert.rejects(second.admit({ kind: 'request' }), { code: 'P6483' }) + const operation = await first.beginMaintenance({ kind: 'backup', timeoutMs: 2000 }) + operations.push(operation) + await operation.resume() +}) + +test('failed durable completion retains its live scope and blocks restore until completion commits', async (t) => { + const { db, first, second, schema, operations } = await fixture(t) + const gate = await second.admit({ kind: 'request' }) + await db.pool.query(`CREATE FUNCTION ${quoteIdentifier(schema)}.reject_completion() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic completion failure' USING ERRCODE='P6484'; END $$; + CREATE TRIGGER reject_completion BEFORE UPDATE ON ${quoteIdentifier(schema)}.activities + FOR EACH ROW EXECUTE FUNCTION ${quoteIdentifier(schema)}.reject_completion()`) + await assert.rejects(gate.complete({ ended: true }), { code: 'P6484' }) + await gate.run(() => second.assertActivity()) + const pending = first.beginMaintenance({ kind: 'restore', timeoutMs: 5000 }) + await waitForPhase(second, 'draining') + await db.pool.query(`DROP TRIGGER reject_completion ON ${quoteIdentifier(schema)}.activities`) + await gate.complete({ ended: true }) + operations.push(await pending) +}) + +test('stopping or losing a Pod preserves all of its unfinished durable activities', async (t) => { + const { db, first, second, schema } = await fixture(t) + const gate = await second.admit({ kind: 'background', requestId: 'remote-refresh' }) + await second.stop() + await assert.rejects(gate.complete({ ended: true }), { code: 'maintenance_connection_lost' }) + const row = ( + await db.pool.query(`SELECT completed_at FROM ${quoteIdentifier(schema)}.activities WHERE id = $1`, [ + gate.activityId, + ]) + ).rows[0] + assert.equal(row.completed_at, null) + await assert.rejects(first.beginMaintenance({ kind: 'restore', timeoutMs: 2000 }), { + code: 'maintenance_claims_active', + }) +}) From 37a638e2189a74653899686e95c6081079e453d4 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:38:09 +0800 Subject: [PATCH 165/473] test: protect saved schedules from late discovery snapshots --- .../unit/shared-background-producers.test.mjs | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/test/unit/shared-background-producers.test.mjs b/test/unit/shared-background-producers.test.mjs index 82caace5e..b026651c6 100644 --- a/test/unit/shared-background-producers.test.mjs +++ b/test/unit/shared-background-producers.test.mjs @@ -422,16 +422,17 @@ test('a controller discovers remote schedule creation, updates and deletion usin test('a late discovery cannot replace a job accepted by a newer discovery', async (t) => { const f = await schedules(t) - const oldRefresh = f.settings.refresh.bind(f.settings) + const oldRead = f.first.resultRepo.listPlans.bind(f.first.resultRepo) const entered = deferred() const finish = deferred() let reads = 0 - f.settings.refresh = async () => { - await oldRefresh() + f.first.resultRepo.listPlans = async () => { + const snapshot = await oldRead() if (++reads === 1) { entered.resolve() await finish.promise } + return snapshot } const older = f.first.reload() await entered.promise @@ -444,15 +445,36 @@ test('a late discovery cannot replace a job accepted by a newer discovery', asyn assert.equal(f.first.jobs.get(live.id), accepted) }) +test('a late discovery cannot replace a locally saved schedule revision', async (t) => { + const f = await schedules(t) + const read = f.first.resultRepo.listPlans.bind(f.first.resultRepo) + const entered = deferred() + const finish = deferred() + f.first.resultRepo.listPlans = async () => { + const snapshot = await read() + entered.resolve() + await finish.promise + return snapshot + } + const older = f.first.reload() + await entered.promise + await f.first.save(live.id, { ...f.first.get(live.id), time: '12:30' }) + const accepted = f.first.jobs.get(live.id) + finish.resolve() + await older + assert.equal(f.first.jobs.get(live.id), accepted) +}) + test('stop awaits its pending discovery and does not publish jobs after stopping', async (t) => { const f = await schedules(t) - const refresh = f.settings.refresh.bind(f.settings) + const read = f.first.resultRepo.listPlans.bind(f.first.resultRepo) const entered = deferred() const finish = deferred() - f.settings.refresh = async () => { - await refresh() + f.first.resultRepo.listPlans = async () => { + const snapshot = await read() entered.resolve() await finish.promise + return snapshot } const discovering = f.first.reload() await entered.promise From 7be4623702ccb15b954b365f1b3c7c004f51d527 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:40:59 +0800 Subject: [PATCH 166/473] test(auth): cover process loss and expiry while awaiting row locks --- test/fixtures/shared-auth-worker.mjs | 33 +++ test/unit/shared-auth-state.test.mjs | 331 +++++++++++++++++++++++++++ 2 files changed, 364 insertions(+) create mode 100644 test/fixtures/shared-auth-worker.mjs diff --git a/test/fixtures/shared-auth-worker.mjs b/test/fixtures/shared-auth-worker.mjs new file mode 100644 index 000000000..23e0e1956 --- /dev/null +++ b/test/fixtures/shared-auth-worker.mjs @@ -0,0 +1,33 @@ +import { openDatabase, closeDatabase } from '../../src/lib/db/database.mjs' +import { generateCodexAuthUrl, exchangeCodexAuthCode } from '../../src/lib/oauth/codex-oauth.mjs' +import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' + +process.once('message', async ({ operation, args }) => { + try { + const db = await openDatabase() + let result + if (operation === 'issueCodex') result = await generateCodexAuthUrl(args) + else if (operation === 'exchangeCodex' || operation === 'claimCodexForever') { + await exchangeCodexAuthCode({ ...args, fetchImpl: async () => { + if (operation === 'claimCodexForever') { + process.send({ event: 'provider_started' }) + await new Promise(() => {}) + } + return { status: 200, json: async () => ({ access_token: 'synthetic', refresh_token: 'synthetic' }) } + } }) + result = { exchanged: true } + } else { + const manager = new ClusterManager({ repo: { db }, dataDir: '/tmp' }) + manager.links.set('node-a', { state: 'ready', client: {} }) + result = operation === 'issueShell' ? await manager.issueShellTicket('node-a') : { + accepted: await manager.consumeShellTicket(args.ticket, args.nodeId), + } + } + process.send({ result }) + } catch (error) { + process.send({ error: error.code || 'child_failed' }) + } finally { + await closeDatabase() + process.disconnect() + } +}) diff --git a/test/unit/shared-auth-state.test.mjs b/test/unit/shared-auth-state.test.mjs index e8b69ed35..24ede63e0 100644 --- a/test/unit/shared-auth-state.test.mjs +++ b/test/unit/shared-auth-state.test.mjs @@ -3,9 +3,13 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' +import crypto from 'node:crypto' +import { fork } from 'node:child_process' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' import { createSlotShell } from '../../src/lib/vm/slot-shell.mjs' +import { createClusterRoutes } from '../../src/lib/cluster/cluster-routes.mjs' +import { OneTimeAuthRepo } from '../../src/lib/db/repos/one-time-auth-repo.mjs' let instance = 0 @@ -43,6 +47,333 @@ function callbackState(generated, kind) { return (kind === 'codex' ? new URL(outer.searchParams.get('authorize_url')) : outer).searchParams.get('state') } +function childController(operation, args) { + const child = fork(new URL('../fixtures/shared-auth-worker.mjs', import.meta.url), [], { execArgv: [], silent: true }) + const result = new Promise((resolve, reject) => { + child.on('message', (message) => { + if (message.event === 'provider_started') resolve(message) + else if (message.error) reject(Object.assign(new Error('synthetic controller failed'), { code: message.error })) + else resolve(message.result) + }) + child.once('error', reject) + child.once('exit', (code) => { + if (code) reject(new Error('synthetic controller exited')) + }) + }) + const exited = new Promise((resolve) => child.once('exit', resolve)) + child.send({ operation, args }) + return { child, result, exited } +} + +test('independent processes can issue and consume PKCE and shell grants exactly once', async (t) => { + await fixture(t) + const issue = childController('issueCodex', { vmId: 'vm-auth', proxyUrl: '' }) + const generated = await issue.result + await issue.exited + const args = { + sessionId: generated.session_id, + vmId: 'vm-auth', + code: `synthetic#${callbackState(generated, 'codex')}`, + } + const exchange = childController('exchangeCodex', args) + assert.equal((await exchange.result).exchanged, true) + await exchange.exited + const replay = childController('exchangeCodex', args) + await assert.rejects(replay.result, { code: 'session_expired' }) + await replay.exited + const issuedShell = childController('issueShell', {}) + const { ticket } = await issuedShell.result + await issuedShell.exited + const consume = childController('consumeShell', { ticket, nodeId: 'node-a' }) + assert.equal((await consume.result).accepted, true) + await consume.exited + const repeat = childController('consumeShell', { ticket, nodeId: 'node-a' }) + assert.equal((await repeat.result).accepted, false) + await repeat.exited +}) + +test('PKCE is encrypted at rest and shell tickets persist only their digest', async (t) => { + const fx = await fixture(t) + const generated = await fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }) + const row = await fx.other.prepare('SELECT * FROM oauth_pkce_sessions').get() + assert.match(row.encrypted_session, /^enc:v1:/) + assert.equal(row.id_hash, crypto.createHash('sha256').update(generated.session_id).digest('hex')) + assert.ok(!row.encrypted_session.includes('codeVerifier')) + const manager = new ClusterManager({ repo: { db: fx.db }, dataDir: fx.root }) + manager.links.set('node-a', { state: 'ready', client: {} }) + const { ticket } = await manager.issueShellTicket('node-a') + const shell = await fx.other.prepare('SELECT * FROM shell_tickets').get() + assert.equal(shell.ticket_hash, crypto.createHash('sha256').update(ticket).digest('hex')) + assert.ok(!JSON.stringify(shell).includes(ticket)) +}) + +test('a PKCE claim prevents concurrent provider requests from another connection', async (t) => { + const fx = await fixture(t) + const generated = await fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }) + let release + const hold = new Promise((resolve) => { + release = resolve + }) + let started + const began = new Promise((resolve) => { + started = resolve + }) + let calls = 0 + const args = { + sessionId: generated.session_id, + vmId: 'vm-auth', + code: 'code=synthetic', + fetchImpl: async () => { + calls += 1 + started() + await hold + return { status: 200, json: async () => ({ access_token: 'synthetic', refresh_token: 'synthetic' }) } + }, + } + const first = fx.controllers[0].codex.exchangeCodexAuthCode({ ...args, db: fx.db }) + try { + await Promise.race([began, first]) + await assert.rejects(fx.controllers[1].codex.exchangeCodexAuthCode({ ...args, db: fx.other }), { + code: 'session_in_use', + }) + assert.equal(calls, 1) + } finally { + release() + await first + } +}) + +test('a provider rejection preserves the Codex session for a later controller retry', async (t) => { + const fx = await fixture(t) + const generated = await fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }) + const args = { sessionId: generated.session_id, vmId: 'vm-auth', code: 'code=synthetic' } + await assert.rejects( + fx.controllers[0].codex.exchangeCodexAuthCode({ + ...args, + fetchImpl: async () => ({ status: 403, json: async () => ({}) }), + }), + { code: 'exchange_rejected' }, + ) + assert.equal((await fx.other.prepare('SELECT claim_id FROM oauth_pkce_sessions').get()).claim_id, null) + await fx.controllers[1].codex.exchangeCodexAuthCode({ + ...args, + db: fx.other, + fetchImpl: async () => ({ + status: 200, + json: async () => ({ access_token: 'synthetic', refresh_token: 'synthetic' }), + }), + }) +}) + +test('a provider rejection preserves the Claude session for a later retry', async (t) => { + const fx = await fixture(t) + const previous = process.env.KIN_OAUTH_AUTH_BIN + const file = path.join(fx.root, 'synthetic-oauth-auth') + fs.writeFileSync( + file, + `#!/usr/bin/env node +process.stdin.resume(); process.stdin.once('end',()=>process.stdout.write(JSON.stringify({ok:false,error:{code:'synthetic_provider_rejected',message:'synthetic rejection'}})+'\\n')); +`, + { mode: 0o700 }, + ) + process.env.KIN_OAUTH_AUTH_BIN = file + delete process.env.KIN_FAKE_SESSION_OAUTH + try { + const generated = await fx.controllers[0].claude.generateAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }) + const args = { sessionId: generated.session_id, vmId: 'vm-auth', code: 'code=synthetic' } + await assert.rejects(fx.controllers[0].claude.exchangeAuthCode(args), { code: 'synthetic_provider_rejected' }) + assert.equal((await fx.other.prepare('SELECT claim_id FROM oauth_pkce_sessions').get()).claim_id, null) + process.env.KIN_FAKE_SESSION_OAUTH = '1' + await fx.controllers[1].claude.exchangeAuthCode({ ...args, db: fx.other }) + } finally { + if (previous === undefined) delete process.env.KIN_OAUTH_AUTH_BIN + else process.env.KIN_OAUTH_AUTH_BIN = previous + } +}) + +test('a crashed PKCE owner remains claimed until the original session expires', async (t) => { + const fx = await fixture(t) + const generated = await fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }) + const args = { sessionId: generated.session_id, vmId: 'vm-auth', code: 'code=synthetic' } + const owner = childController('claimCodexForever', args) + try { + await owner.result + } finally { + owner.child.kill('SIGKILL') + await owner.exited + } + let calls = 0 + const retry = { + ...args, + db: fx.other, + fetchImpl: async () => { + calls += 1 + assert.fail('no provider request expected') + }, + } + await assert.rejects(fx.controllers[1].codex.exchangeCodexAuthCode(retry), { code: 'session_in_use' }) + await fx.db.prepare("UPDATE oauth_pkce_sessions SET expires_at=clock_timestamp()-interval '1 second'").run() + await assert.rejects(fx.controllers[1].codex.exchangeCodexAuthCode(retry), { code: 'session_expired' }) + assert.equal(calls, 0) +}) + +for (const [phase, event, expectedCalls] of [ + ['claim', 'UPDATE OF claim_id', 0], + ['consume', 'DELETE', 1], +]) { + test(`a PKCE PostgreSQL ${phase} failure never uses local fallback`, async (t) => { + const fx = await fixture(t) + const generated = await fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }) + await fx.db.exec(`CREATE FUNCTION reject_pkce() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic failure' USING ERRCODE='P6494'; END; $$; + CREATE TRIGGER reject_pkce BEFORE ${event} ON oauth_pkce_sessions FOR EACH ROW EXECUTE FUNCTION reject_pkce();`) + let calls = 0 + await assert.rejects( + fx.controllers[0].codex.exchangeCodexAuthCode({ + sessionId: generated.session_id, + vmId: 'vm-auth', + code: 'code=synthetic', + fetchImpl: async () => { + calls += 1 + return { status: 200, json: async () => ({ access_token: 'synthetic', refresh_token: 'synthetic' }) } + }, + }), + { code: 'P6494' }, + ) + assert.equal(calls, expectedCalls) + const row = await fx.other.prepare('SELECT claim_id FROM oauth_pkce_sessions').get() + assert.equal(!!row.claim_id, phase === 'consume') + if (phase === 'consume') { + await assert.rejects( + fx.controllers[1].codex.exchangeCodexAuthCode({ + sessionId: generated.session_id, + vmId: 'vm-auth', + code: 'code=synthetic', + db: fx.other, + fetchImpl: async () => { + calls += 1 + assert.fail('a consumed provider code must not be retried') + }, + }), + { code: 'session_in_use' }, + ) + assert.equal(calls, 1) + } + }) +} + +test('PostgreSQL issue failures create neither PKCE nor a local shell fallback', async (t) => { + const fx = await fixture(t) + await fx.db.exec(`CREATE FUNCTION reject_auth_issue() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic issue failure' USING ERRCODE='P6495'; END; $$; + CREATE TRIGGER reject_auth_issue BEFORE INSERT ON oauth_pkce_sessions FOR EACH ROW EXECUTE FUNCTION reject_auth_issue(); + CREATE TRIGGER reject_auth_issue BEFORE INSERT ON shell_tickets FOR EACH ROW EXECUTE FUNCTION reject_auth_issue();`) + await assert.rejects(fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }), { + code: 'P6495', + }) + const manager = new ClusterManager({ repo: { db: fx.db }, dataDir: fx.root }) + manager.links.set('node-a', { state: 'ready', client: {} }) + await assert.rejects(manager.issueShellTicket('node-a'), { code: 'P6495' }) + assert.equal(manager.tickets.size, 0) + assert.equal((await fx.other.prepare('SELECT count(*) AS count FROM oauth_pkce_sessions').get()).count, 0) + assert.equal((await fx.other.prepare('SELECT count(*) AS count FROM shell_tickets').get()).count, 0) +}) + +test('shell scope mismatch does not consume a grant and concurrent consumers get one winner', async (t) => { + const fx = await fixture(t) + const a = new OneTimeAuthRepo(fx.db) + const b = new OneTimeAuthRepo(fx.other) + const { ticket } = await a.issueShell('slot', 'vm-a', { scope: 'login' }) + await assert.rejects(b.consumeShell('slot', ticket, 'vm-a'), { code: 'shell_ticket_scope_mismatch' }) + assert.equal(await b.consumeShell('node', ticket, 'vm-a'), null) + const results = await Promise.all([a, b].map((repo) => repo.consumeShell('slot', ticket, 'vm-a', { scope: 'login' }))) + assert.equal(results.filter(Boolean).length, 1) +}) + +for (const kind of ['PKCE', 'shell']) { + test(`${kind} that expires while waiting for a database row lock cannot be consumed`, async (t) => { + const fx = await fixture(t) + const repo = new OneTimeAuthRepo(fx.other) + let consume + let table + if (kind === 'PKCE') { + const generated = await fx.controllers[0].codex.generateCodexAuthUrl({ vmId: 'vm-auth', proxyUrl: '' }) + table = 'oauth_pkce_sessions' + consume = () => repo.claimPkce('codex', generated.session_id, () => {}) + } else { + const { ticket } = await new OneTimeAuthRepo(fx.db).issueShell('node', 'node-a') + table = 'shell_tickets' + consume = () => repo.consumeShell('node', ticket, 'node-a') + } + await fx.db.exec(`UPDATE ${table} SET expires_at=clock_timestamp()+interval '500 milliseconds'`) + const client = await fx.db.pool.connect() + let pending + try { + await client.query('BEGIN') + await client.query(`SELECT * FROM ${table} FOR UPDATE`) + const xid = (await client.query('SELECT pg_current_xact_id()::text AS xid')).rows[0].xid + pending = consume() + let blocked = false + for (let i = 0; i < 100; i++) { + const row = await fx.db + .prepare(`SELECT count(*) AS count FROM pg_locks + WHERE locktype='transactionid' AND transactionid::text=$1 AND NOT granted`) + .get(xid) + if (row.count) { + blocked = true + break + } + await new Promise((resolve) => setTimeout(resolve, 5)) + } + assert.equal(blocked, true) + await new Promise((resolve) => setTimeout(resolve, 550)) + } finally { + await client.query('COMMIT') + client.release() + } + if (kind === 'PKCE') await assert.rejects(pending, { code: 'session_expired' }) + else assert.equal(await pending, null) + }) +} + +test('expired shell tickets are rejected and a database failure cannot authorize a node shell', async (t) => { + const fx = await fixture(t) + const a = new ClusterManager({ repo: { db: fx.db }, dataDir: fx.root }) + const b = new ClusterManager({ repo: { db: fx.other }, dataDir: fx.root }) + a.links.set('node-a', { state: 'ready', client: {} }) + const first = await a.issueShellTicket('node-a') + await fx.db.prepare("UPDATE shell_tickets SET expires_at=clock_timestamp()-interval '1 second'").run() + assert.equal(await b.consumeShellTicket(first.ticket, 'node-a'), false) + const next = await a.issueShellTicket('node-a') + await fx.other.close() + await assert.rejects(b.consumeShellTicket(next.ticket, 'node-a')) + assert.equal(await a.consumeShellTicket(next.ticket, 'node-a'), true) +}) + +test('cluster upgrade awaits ticket authorization and rejects a PostgreSQL failure', async () => { + for (const consumeShellTicket of [ + async () => false, + async () => { + throw new Error('synthetic PG failure') + }, + ]) { + const routes = createClusterRoutes({ + manager: { consumeShellTicket }, + json() {}, + readBody() {}, + ok: (value) => value, + }) + const writes = [] + const socket = { write: (value) => writes.push(String(value)), destroy() {} } + assert.equal( + routes.handleUpgrade({ url: '/api/panel/cluster/nodes/node-a/shell?ticket=synthetic' }, socket, Buffer.alloc(0)), + true, + ) + await routes.drain() + assert.match(writes[0], /^HTTP\/1\.1 (401|503) /) + } +}) + for (const kind of ['claude', 'codex']) { test(`${kind} PKCE survives a controller change and permits only one successful exchange`, async (t) => { const fx = await fixture(t) From 4b7da94203c3608a4fc1e662d648ed8353f74c63 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:43:05 +0800 Subject: [PATCH 167/473] feat(auth): persist one-time PKCE sessions and shell tickets across controllers --- deploy/k8s/secret.example.yaml | 12 ++ docs/deploy/kubernetes.md | 4 + src/lib/admin/panel-routes.mjs | 4 +- src/lib/cluster/cluster-manager.mjs | 11 +- src/lib/cluster/cluster-routes.mjs | 24 ++-- .../postgres-migrations/014_one_time_auth.sql | 20 +++ src/lib/db/repos/one-time-auth-repo.mjs | 126 ++++++++++++++++ src/lib/oauth/codex-oauth.mjs | 46 +++--- src/lib/oauth/oauth-auth-url.mjs | 60 +++++--- src/lib/vm/slot-shell.mjs | 136 +++++++++++------- 10 files changed, 346 insertions(+), 97 deletions(-) create mode 100644 deploy/k8s/secret.example.yaml create mode 100644 src/lib/db/postgres-migrations/014_one_time_auth.sql create mode 100644 src/lib/db/repos/one-time-auth-repo.mjs diff --git a/deploy/k8s/secret.example.yaml b/deploy/k8s/secret.example.yaml new file mode 100644 index 000000000..93b212ed4 --- /dev/null +++ b/deploy/k8s/secret.example.yaml @@ -0,0 +1,12 @@ +# Example only; not included by kustomization.yaml. Create the real Secret from a protected env file. +apiVersion: v1 +kind: Secret +metadata: + name: vm2api-secrets +type: Opaque +stringData: + VM2API_DATABASE_URL: REPLACE_WITH_DATABASE_URL + VM2API_ADMIN_PASSWORD: REPLACE_WITH_ADMIN_PASSWORD + VM2API_API_KEY: REPLACE_WITH_API_KEY + # Required for Kubernetes PKCE sessions. All replicas must retain the same existing business encryption key. + VM2API_DB_SECRET: REPLACE_WITH_EXISTING_BUSINESS_ENCRYPTION_KEY diff --git a/docs/deploy/kubernetes.md b/docs/deploy/kubernetes.md index ec5e4656e..69b248125 100644 --- a/docs/deploy/kubernetes.md +++ b/docs/deploy/kubernetes.md @@ -24,6 +24,10 @@ VM2API_API_KEY VM2API_DB_SECRET ``` +`VM2API_DB_SECRET` 是现有数据库字段加密密钥，兼容旧名 `KIN_DB_SECRET`。Kubernetes 模式下生成浏览器 OAuth 授权链接必须配置该密钥；缺失时拒绝发起 PKCE，不能将 verifier 明文存入 PostgreSQL。各副本使用同一密钥，不能在升级或导入时替换。字段示例见 `deploy/k8s/secret.example.yaml`；该文件不参与默认 Kustomize 部署，不能直接使用占位值。 + +PKCE 会话通过 PostgreSQL claim 防止并发换票，成功后消费，供应商失败后恢复原会话供重试。进程在换票期间退出时，claim 保留至原会话到期，之后必须重新发起授权。终端 ticket 在 PostgreSQL 只存随机票据摘要，有效期为 30 秒，成功匹配资源后只能消费一次；错误节点、VM 或登录模式不消耗其他资源的票据。 + 管理员用户名默认为 `admin`。已有 SQLite 安装应先按 PostgreSQL 文档完成离线导入，保留原业务加密密钥和对应的 VM、配置及备份文件，之后再启动 Pod。 ```sh diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 252c7b9f6..1da9b5a93 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -4597,7 +4597,7 @@ export function createPanelHandler(ctx) { } try { if (isCodexVm(existing)) { - const generated = generateCodexAuthUrl({ + const generated = await generateCodexAuthUrl({ vmId: id, proxyUrl: slotProxy.proxyUrl, installationId: existing.device_id || existing.codex?.installation_id, @@ -4629,7 +4629,7 @@ export function createPanelHandler(ctx) { }), ) } - const generated = generateAuthUrl({ + const generated = await generateAuthUrl({ vmId: id, proxyUrl: slotProxy.proxyUrl, flavor, diff --git a/src/lib/cluster/cluster-manager.mjs b/src/lib/cluster/cluster-manager.mjs index 4fe95bc48..e3c475a8c 100644 --- a/src/lib/cluster/cluster-manager.mjs +++ b/src/lib/cluster/cluster-manager.mjs @@ -5,7 +5,9 @@ */ import crypto from 'node:crypto' -import { withTransaction } from '../db/database.mjs' +import { getDb, withTransaction } from '../db/database.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { OneTimeAuthRepo } from '../db/repos/one-time-auth-repo.mjs' import fs from 'node:fs' import net from 'node:net' import path from 'node:path' @@ -123,6 +125,7 @@ export class ClusterManager { onNodesChanged = null, onLinkState = null, assertNodeEditable = () => {}, + authState = null, gateBin = process.env.KIN_CLUSTER_GATE_BIN || path.join(process.cwd(), 'bin', 'kin-slot-gate'), }) { this.gateBin = gateBin @@ -154,6 +157,7 @@ export class ClusterManager { this.health = new Map() this.installs = new Map() this.tickets = new Map() + this.authState = isKubernetesMode() ? authState || new OneTimeAuthRepo(repo.db || getDb()) : null this._healthTimer = null } @@ -657,6 +661,7 @@ export class ClusterManager { ? await this.repo.updateLabel(id, edit.spec.label, edit.node.updated_at) : await this.repo.update(id, edit.spec, edit.node.updated_at) if (!saved) throw nodeEditConflict() + if (candidate && this.authState) await this.authState.revokeShell('node', id) if (candidate) { // Reconnect and placement may change while PostgreSQL executes the CAS. // Keep this final check and adoption synchronous; a rejection rolls back the CAS. @@ -992,6 +997,7 @@ export class ClusterManager { issueShellTicket(id) { this.client(id) + if (this.authState) return this.authState.issueShell('node', id, { ttlMs: SHELL_TICKET_TTL_MS }) const now = Date.now() for (const [t, v] of this.tickets) if (v.exp <= now) this.tickets.delete(t) const ticket = crypto.randomBytes(24).toString('base64url') @@ -999,8 +1005,9 @@ export class ClusterManager { return { ticket, expires_in: SHELL_TICKET_TTL_MS / 1000 } } - /** Single use: the ticket is burned whether or not it matches. */ + /** Shared tickets reject a wrong resource without consuming the grant. */ consumeShellTicket(ticket, id) { + if (this.authState) return this.authState.consumeShell('node', ticket, id).then(Boolean) const entry = this.tickets.get(ticket) this.tickets.delete(ticket) return !!entry && entry.nodeId === id && entry.exp > Date.now() diff --git a/src/lib/cluster/cluster-routes.mjs b/src/lib/cluster/cluster-routes.mjs index dda0e070c..655f576e2 100644 --- a/src/lib/cluster/cluster-routes.mjs +++ b/src/lib/cluster/cluster-routes.mjs @@ -82,7 +82,7 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { return respond(res, 200, ok(await manager.setDirectLink(id, body))) } if (sub === '/direct-link' && m === 'DELETE') return respond(res, 200, ok(await manager.clearDirectLink(id))) - if (sub === '/shell-ticket' && m === 'POST') return respond(res, 200, ok(manager.issueShellTicket(id))) + if (sub === '/shell-ticket' && m === 'POST') return respond(res, 200, ok(await manager.issueShellTicket(id))) if (sub === '/docker/info' && m === 'GET') return respond(res, 200, ok(await dockerInfo(manager.docker(id)))) if (sub === '/preflight' && m === 'POST') { @@ -193,16 +193,20 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { return true } const id = sm[1] - if (!manager.consumeShellTicket(url.searchParams.get('ticket') || '', id)) { - rejectUpgrade(socket, 401, 'Unauthorized') - return true + const accept = (authorized) => { + if (!authorized) return rejectUpgrade(socket, 401, 'Unauthorized') + if (draining || socket.destroyed) return rejectUpgrade(socket, 503, 'Service Unavailable') + const cols = clampInt(url.searchParams.get('cols'), 10, 500, 80) + const rows = clampInt(url.searchParams.get('rows'), 5, 200, 24) + wss.handleUpgrade(req, socket, head, (ws) => { + track(new Promise((resolve) => ws.once('close', resolve))) + track(attachShell(ws, id, { cols, rows })) + }) } - const cols = clampInt(url.searchParams.get('cols'), 10, 500, 80) - const rows = clampInt(url.searchParams.get('rows'), 5, 200, 24) - wss.handleUpgrade(req, socket, head, (ws) => { - track(new Promise((resolve) => ws.once('close', resolve))) - track(attachShell(ws, id, { cols, rows })) - }) + const consumed = manager.consumeShellTicket(url.searchParams.get('ticket') || '', id) + if (typeof consumed?.then === 'function') { + track(consumed.then(accept, () => rejectUpgrade(socket, 503, 'Service Unavailable'))) + } else accept(consumed) return true } diff --git a/src/lib/db/postgres-migrations/014_one_time_auth.sql b/src/lib/db/postgres-migrations/014_one_time_auth.sql new file mode 100644 index 000000000..5cd873754 --- /dev/null +++ b/src/lib/db/postgres-migrations/014_one_time_auth.sql @@ -0,0 +1,20 @@ +CREATE TABLE oauth_pkce_sessions ( + kind text NOT NULL CHECK (kind IN ('anthropic', 'codex')), + id_hash text NOT NULL, + resource_id text NOT NULL, + encrypted_session text NOT NULL, + expires_at timestamptz NOT NULL, + claim_id text, + PRIMARY KEY (kind, id_hash) +); +CREATE INDEX oauth_pkce_sessions_expiry ON oauth_pkce_sessions (expires_at); + +CREATE TABLE shell_tickets ( + purpose text NOT NULL CHECK (purpose IN ('slot', 'node')), + ticket_hash text NOT NULL, + resource_id text NOT NULL, + scope text NOT NULL, + expires_at timestamptz NOT NULL, + PRIMARY KEY (purpose, ticket_hash) +); +CREATE INDEX shell_tickets_expiry ON shell_tickets (expires_at); diff --git a/src/lib/db/repos/one-time-auth-repo.mjs b/src/lib/db/repos/one-time-auth-repo.mjs new file mode 100644 index 000000000..13a60fe71 --- /dev/null +++ b/src/lib/db/repos/one-time-auth-repo.mjs @@ -0,0 +1,126 @@ +import crypto from 'node:crypto' +import { getDb, withTransaction } from '../database.mjs' +import { encryptString, decryptString, encryptionEnabled, isEncrypted } from '../secure.mjs' + +function fail(code) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} + +export function authTokenHash(value) { + return crypto + .createHash('sha256') + .update(String(value || '')) + .digest('hex') +} + +function requireEncryption() { + if (!encryptionEnabled()) throw fail('oauth_session_encryption_required') +} + +function sessionFrom(row) { + requireEncryption() + try { + if (!isEncrypted(row.encrypted_session)) throw new Error('invalid session') + return JSON.parse(decryptString(row.encrypted_session)) + } catch { + throw fail('oauth_session_invalid') + } +} + +/** One-time browser authentication state. No connection lock is held across provider I/O. */ +export class OneTimeAuthRepo { + constructor(db = getDb()) { + this.db = db + } + + async issuePkce(kind, id, session, ttlMs) { + requireEncryption() + const encrypted = encryptString(JSON.stringify(session)) + await this.db.prepare('DELETE FROM oauth_pkce_sessions WHERE expires_at<=clock_timestamp()').run() + const row = await this.db + .prepare(`INSERT INTO oauth_pkce_sessions + (kind, id_hash, resource_id, encrypted_session, expires_at) + VALUES ($1,$2,$3,$4,clock_timestamp()+$5::bigint*interval '1 millisecond') RETURNING expires_at`) + .get(kind, authTokenHash(id), session.vmId, encrypted, ttlMs) + return { expiresAt: row.expires_at.getTime() } + } + + async peekPkce(kind, id) { + const row = await this.db + .prepare(`SELECT * FROM oauth_pkce_sessions + WHERE kind=$1 AND id_hash=$2 AND expires_at>clock_timestamp()`) + .get(kind, authTokenHash(id)) + return row ? sessionFrom(row) : null + } + + async claimPkce(kind, id, validate) { + return withTransaction(this.db, async () => { + const row = await this.db + .prepare(`SELECT * FROM oauth_pkce_sessions + WHERE kind=$1 AND id_hash=$2 AND expires_at>clock_timestamp() FOR UPDATE`) + .get(kind, authTokenHash(id)) + if (!row) throw fail('session_expired') + const session = sessionFrom(row) + validate(session) + if (row.claim_id) throw fail('session_in_use') + const claimId = crypto.randomUUID() + const claimed = await this.db + .prepare(`UPDATE oauth_pkce_sessions SET claim_id=$3 + WHERE kind=$1 AND id_hash=$2 AND expires_at>clock_timestamp()`) + .run(kind, authTokenHash(id), claimId) + if (claimed.changes !== 1) throw fail('session_expired') + return { session, claimId } + }) + } + + async finishPkce(kind, id, claimId, consume) { + const result = consume + ? await this.db + .prepare('DELETE FROM oauth_pkce_sessions WHERE kind=$1 AND id_hash=$2 AND claim_id=$3') + .run(kind, authTokenHash(id), claimId) + : await this.db + .prepare('UPDATE oauth_pkce_sessions SET claim_id=NULL WHERE kind=$1 AND id_hash=$2 AND claim_id=$3') + .run(kind, authTokenHash(id), claimId) + if (result.changes !== 1) throw fail('oauth_session_claim_lost') + } + + async issueShell(purpose, resourceId, { scope = 'ops', ttlMs = 30_000, maxTickets = 0 } = {}) { + const ticket = crypto.randomBytes(24).toString('base64url') + return withTransaction(this.db, async () => { + await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1),hashtext($2))', [ + `vm2api:auth:${this.db.schema}`, + `shell:${purpose}`, + ]) + await this.db.prepare('DELETE FROM shell_tickets WHERE expires_at<=clock_timestamp()').run() + if (maxTickets) { + const row = await this.db.prepare('SELECT count(*) AS count FROM shell_tickets WHERE purpose=$1').get(purpose) + if (row.count >= maxTickets) throw fail('shell_limit') + } + await this.db + .prepare(`INSERT INTO shell_tickets (purpose,ticket_hash,resource_id,scope,expires_at) + VALUES ($1,$2,$3,$4,clock_timestamp()+$5::bigint*interval '1 millisecond')`) + .run(purpose, authTokenHash(ticket), resourceId, scope, ttlMs) + return { ticket, expires_in: ttlMs / 1000 } + }) + } + + async consumeShell(purpose, ticket, resourceId, { scope = 'ops' } = {}) { + return withTransaction(this.db, async () => { + const row = await this.db + .prepare(`SELECT * FROM shell_tickets WHERE purpose=$1 AND ticket_hash=$2 + AND resource_id=$3 AND expires_at>clock_timestamp() FOR UPDATE`) + .get(purpose, authTokenHash(ticket), resourceId) + if (!row) return null + if (row.scope !== scope) throw fail('shell_ticket_scope_mismatch') + const consumed = await this.db + .prepare('DELETE FROM shell_tickets WHERE purpose=$1 AND ticket_hash=$2 AND expires_at>clock_timestamp()') + .run(purpose, authTokenHash(ticket)) + if (consumed.changes !== 1) return null + return { resourceId: row.resource_id, scope: row.scope } + }) + } + + async revokeShell(purpose, resourceId) { + await this.db.prepare('DELETE FROM shell_tickets WHERE purpose=$1 AND resource_id=$2').run(purpose, resourceId) + } +} diff --git a/src/lib/oauth/codex-oauth.mjs b/src/lib/oauth/codex-oauth.mjs index 92aacd4ab..cb91f1563 100644 --- a/src/lib/oauth/codex-oauth.mjs +++ b/src/lib/oauth/codex-oauth.mjs @@ -3,6 +3,8 @@ * Matches codex-proxy-rs: loopback callback is pasted back, exchange via slot SOCKS5. */ import crypto from 'node:crypto' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { OneTimeAuthRepo } from '../db/repos/one-time-auth-repo.mjs' import { CODEX_APP_VERSION, CODEX_OAUTH_AUTHORIZE_URL, @@ -104,7 +106,7 @@ function buildDesktopAuthUrl({ state, codeChallenge, installationId }) { return outer.toString() } -export function generateCodexAuthUrl({ vmId, proxyUrl, installationId } = {}) { +export function generateCodexAuthUrl({ vmId, proxyUrl, installationId, db } = {}) { sweepExpired() if (!vmId) throw fail('vm_required', 'vm_id required (先创建虚拟机)') if (proxyUrl == null) throw fail('proxy_required', '虚拟机未绑定 SOCKS5，请先分配代理再生成授权链接') @@ -114,38 +116,48 @@ export function generateCodexAuthUrl({ vmId, proxyUrl, installationId } = {}) { const sessionId = crypto.randomBytes(16).toString('hex') const createdAt = Date.now() const install = String(installationId || crypto.randomUUID()) - sessions.set(sessionId, { + const session = { state, codeVerifier, proxyUrl: proxyUrl === '' ? '' : String(proxyUrl), vmId: String(vmId), installationId: install, createdAt, - }) - return { + } + const generated = { auth_url: buildDesktopAuthUrl({ state, codeChallenge, installationId: install }), session_id: sessionId, expires_at: createdAt + CODEX_OAUTH_SESSION_TTL_MS, vm_id: String(vmId), flavor: 'codex', } + if (isKubernetesMode()) { + return new OneTimeAuthRepo(db) + .issuePkce('codex', sessionId, session, CODEX_OAUTH_SESSION_TTL_MS) + .then(({ expiresAt }) => ({ ...generated, expires_at: expiresAt })) + } + sessions.set(sessionId, session) + return generated } -export async function exchangeCodexAuthCode({ sessionId, code, proxyUrl, vmId, fetchImpl } = {}) { +export async function exchangeCodexAuthCode({ sessionId, code, proxyUrl, vmId, fetchImpl, db } = {}) { sweepExpired() - const session = sessions.get(sessionId) - if (!session || Date.now() - session.createdAt > CODEX_OAUTH_SESSION_TTL_MS) { + const parsed = parseCodexOAuthCallback(code) + const store = isKubernetesMode() ? new OneTimeAuthRepo(db) : null + const validate = (session) => { + if (vmId && session.vmId && session.vmId !== String(vmId)) + throw fail('session_vm_mismatch', '授权会话与当前虚拟机不匹配') + if (!parsed.code) throw fail('code_required', '请粘贴 Codex 回调 URL 或授权码') + if (parsed.state && parsed.state !== session.state) throw fail('state_mismatch', '回调 state 与授权会话不匹配') + if ((proxyUrl ?? session.proxyUrl) == null) throw fail('proxy_required', '虚拟机未绑定 SOCKS5，无法换票') + } + const claim = store ? await store.claimPkce('codex', sessionId, validate) : null + const session = store ? claim.session : sessions.get(sessionId) + if (!session || (!store && Date.now() - session.createdAt > CODEX_OAUTH_SESSION_TTL_MS)) { if (session) sessions.delete(sessionId) throw fail('session_expired', '授权会话不存在或已过期，请重新生成授权链接') } - if (vmId && session.vmId && session.vmId !== String(vmId)) { - throw fail('session_vm_mismatch', '授权会话与当前虚拟机不匹配') - } - const parsed = parseCodexOAuthCallback(code) - if (!parsed.code) throw fail('code_required', '请粘贴 Codex 回调 URL 或授权码') - if (parsed.state && parsed.state !== session.state) { - throw fail('state_mismatch', '回调 state 与授权会话不匹配') - } + if (!store) validate(session) const px = proxyUrl ?? session.proxyUrl if (px == null) throw fail('proxy_required', '虚拟机未绑定 SOCKS5，无法换票') const tok = await exchangeCodexAuthorizationCode({ @@ -155,9 +167,11 @@ export async function exchangeCodexAuthCode({ sessionId, code, proxyUrl, vmId, f fetchImpl, }) if (!tok.ok) { + if (store) await store.finishPkce('codex', sessionId, claim.claimId, false) throw fail(tok.error || 'exchange_failed', 'Codex OAuth 换票失败，请重新生成授权链接') } - sessions.delete(sessionId) + if (store) await store.finishPkce('codex', sessionId, claim.claimId, true) + else sessions.delete(sessionId) return { access_token: tok.access_token, refresh_token: tok.refresh_token, diff --git a/src/lib/oauth/oauth-auth-url.mjs b/src/lib/oauth/oauth-auth-url.mjs index ba72958a7..068cd828d 100644 --- a/src/lib/oauth/oauth-auth-url.mjs +++ b/src/lib/oauth/oauth-auth-url.mjs @@ -9,6 +9,8 @@ import crypto from 'node:crypto' import { exchangeTokenViaCookieAuth } from './cookie-auth.mjs' import { enrichOauthIdentity, flattenOauthIdentity } from './oauth-identity.mjs' import { FULL_OAUTH_SCOPE } from './oauth-contract.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { OneTimeAuthRepo } from '../db/repos/one-time-auth-repo.mjs' export const CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e' export const SESSION_TTL_MS = 30 * 60 * 1000 @@ -107,7 +109,8 @@ export function resetAuthUrlSessions() { sessions.clear() } -export function peekAuthUrlSession(sessionId) { +export function peekAuthUrlSession(sessionId, { db } = {}) { + if (isKubernetesMode()) return new OneTimeAuthRepo(db).peekPkce('anthropic', sessionId) sweepExpired() return sessions.get(sessionId) || null } @@ -119,7 +122,7 @@ export function buildAuthorizationURL(state, codeChallenge, scope = SCOPE_OAUTH, return `${spec.authorizeUrl}?code=true&client_id=${CLIENT_ID}&response_type=code&redirect_uri=${encodedRedirectURI}&scope=${encodedScope}&code_challenge=${codeChallenge}&code_challenge_method=S256&state=${state}` } -export function generateAuthUrl({ vmId, proxyUrl, flavor } = {}) { +export function generateAuthUrl({ vmId, proxyUrl, flavor, db } = {}) { sweepExpired() if (normalizeOauthFlavor(flavor) === 'claude_code') { throw fail('claude_code_in_slot', '官方 Claude Code 登录必须由槽内 cli-node 执行，不能由宿主机生成授权链接') @@ -134,7 +137,7 @@ export function generateAuthUrl({ vmId, proxyUrl, flavor } = {}) { const codeChallenge = generateCodeChallenge(codeVerifier) const sessionId = generateSessionId() const createdAt = Date.now() - sessions.set(sessionId, { + const session = { state, codeVerifier, flavor: spec.flavor, @@ -145,14 +148,21 @@ export function generateAuthUrl({ vmId, proxyUrl, flavor } = {}) { proxyUrl: proxyUrl === '' ? '' : String(proxyUrl), vmId: String(vmId), createdAt, - }) - return { + } + const generated = { auth_url: buildAuthorizationURL(state, codeChallenge, spec.scope, spec.flavor), session_id: sessionId, expires_at: createdAt + SESSION_TTL_MS, vm_id: String(vmId), flavor: spec.flavor, } + if (isKubernetesMode()) { + return new OneTimeAuthRepo(db) + .issuePkce('anthropic', sessionId, session, SESSION_TTL_MS) + .then(({ expiresAt }) => ({ ...generated, expires_at: expiresAt })) + } + sessions.set(sessionId, session) + return generated } function parseAuthCode(fullCode) { @@ -175,25 +185,31 @@ async function exchangeCodeForToken(authCode, codeVerifier, state, proxyUrl, ses }) } -export async function exchangeAuthCode({ sessionId, code, proxyUrl, vmId, fetchImpl = null } = {}) { +export async function exchangeAuthCode({ sessionId, code, proxyUrl, vmId, fetchImpl = null, db } = {}) { sweepExpired() - const session = sessions.get(sessionId) - if (!session || Date.now() - session.createdAt > SESSION_TTL_MS) { + const parsed = parseAuthCode(code) + const store = isKubernetesMode() ? new OneTimeAuthRepo(db) : null + const validate = (session) => { + if (vmId && session.vmId && session.vmId !== String(vmId)) + throw fail('session_vm_mismatch', '授权会话与当前虚拟机不匹配') + if (!parsed.code) throw fail('code_required', '请粘贴授权码') + if (session.flavor === 'setup_token' && parsed.state && parsed.state !== session.state) + throw fail('state_mismatch', '授权 state 不匹配，请重新生成授权链接') + if ((proxyUrl ?? session.proxyUrl) == null) throw fail('proxy_required', '虚拟机未绑定 SOCKS5，无法换票') + } + const claim = store ? await store.claimPkce('anthropic', sessionId, validate) : null + const session = store ? claim.session : sessions.get(sessionId) + if (!session || (!store && Date.now() - session.createdAt > SESSION_TTL_MS)) { if (session) sessions.delete(sessionId) throw fail('session_expired', '授权会话不存在或已过期，请重新生成授权链接') } - if (vmId && session.vmId && session.vmId !== String(vmId)) { - throw fail('session_vm_mismatch', '授权会话与当前虚拟机不匹配') - } - const parsed = parseAuthCode(code) - if (!parsed.code) throw fail('code_required', '请粘贴授权码') - if (session.flavor === 'setup_token' && parsed.state && parsed.state !== session.state) - throw fail('state_mismatch', '授权 state 不匹配，请重新生成授权链接') + if (!store) validate(session) const px = proxyUrl ?? session.proxyUrl - if (px == null) throw fail('proxy_required', '虚拟机未绑定 SOCKS5，无法换票') + const consume = () => + store ? store.finishPkce('anthropic', sessionId, claim.claimId, true) : sessions.delete(sessionId) if (process.env.KIN_FAKE_SESSION_OAUTH === '1' || process.env.KIN_FAKE_SESSION_OAUTH === 'true') { - sessions.delete(sessionId) + await consume() const now = Math.floor(Date.now() / 1000) return { access_token: 'sk-ant-oat01-FAKE-AUTH-URL', @@ -213,8 +229,14 @@ export async function exchangeAuthCode({ sessionId, code, proxyUrl, vmId, fetchI // Full OAuth paste codes carry the provider's exchange state, not necessarily // the authorize nonce. PKCE still binds the code to this VM's live session. const exchangeState = session.flavor === 'setup_token' ? parsed.state || session.state : parsed.state - const token = await exchangeCodeForToken(parsed.code, session.codeVerifier, exchangeState, px, session) - sessions.delete(sessionId) + let token + try { + token = await exchangeCodeForToken(parsed.code, session.codeVerifier, exchangeState, px, session) + } catch (error) { + if (store) await store.finishPkce('anthropic', sessionId, claim.claimId, false) + throw error + } + await consume() const expiresIn = Number(token.expires_in || 0) const now = Math.floor(Date.now() / 1000) const identity = flattenOauthIdentity(token) diff --git a/src/lib/vm/slot-shell.mjs b/src/lib/vm/slot-shell.mjs index b6167a2dd..6498c9162 100644 --- a/src/lib/vm/slot-shell.mjs +++ b/src/lib/vm/slot-shell.mjs @@ -24,6 +24,8 @@ import fs from 'node:fs' import path from 'node:path' import { ensureRemoteOfficialCredentialLink, reconcileSlotCredentials } from '../cluster/remote-slot-files.mjs' import { officialCcUidGid } from './vm-runtime.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { OneTimeAuthRepo } from '../db/repos/one-time-auth-repo.mjs' const SHELL_RE = /^\/api\/panel\/vms\/([^/]+)\/shell$/ const TICKET_TTL_MS = 30_000 @@ -376,8 +378,10 @@ export function createSlotShell({ importOauth = null, authLoginReady = null, upgradeLoginReady = null, + db, } = {}) { const tickets = new Map() + const authState = isKubernetesMode() ? new OneTimeAuthRepo(db) : null const liveTokens = new Set() let sessions = 0 const wss = new WebSocketServer({ noServer: true, maxPayload: 1 << 20 }) @@ -408,6 +412,21 @@ export function createSlotShell({ } if (draining) return refuse(503, 'control_plane_draining', 'Control plane is draining') if (sessions >= MAX_SESSIONS) return refuse(429, 'shell_limit', `终端会话已达上限 ${MAX_SESSIONS}`) + if (authState) { + try { + return { + ok: true, + ...(await authState.issueShell('slot', vm.id, { + scope: login ? 'login' : 'ops', + ttlMs: TICKET_TTL_MS, + maxTickets: MAX_TICKETS, + })), + } + } catch (error) { + if (error.code === 'shell_limit') return refuse(429, 'shell_limit', '终端 ticket 过多，请稍后再试') + throw error + } + } const now = Date.now() for (const [t, v] of tickets) if (v.exp <= now) tickets.delete(t) if (tickets.size >= MAX_TICKETS) return refuse(429, 'shell_limit', '终端 ticket 过多，请稍后再试') @@ -416,8 +435,12 @@ export function createSlotShell({ return { ok: true, ticket, expires_in: TICKET_TTL_MS / 1000 } } - /** Single use: the ticket is burned whether or not it matches. */ - function consumeTicket(ticket, vmId) { + /** Shared tickets reject a wrong VM or mode without consuming the grant. */ + function consumeTicket(ticket, vmId, login) { + if (authState) + return authState + .consumeShell('slot', ticket, vmId, { scope: login ? 'login' : 'ops' }) + .then((entry) => (entry ? { vmId: entry.resourceId, login: entry.scope === 'login' } : null)) const entry = tickets.get(ticket) tickets.delete(ticket) if (!entry || entry.vmId !== vmId || entry.exp <= Date.now()) return null @@ -440,60 +463,77 @@ export function createSlotShell({ } const vmId = decodeURIComponent(sm[1]) const ticket = url.searchParams.get('ticket') || '' - const entry = consumeTicket(ticket, vmId) - if (!entry) { - rejectUpgrade(socket, 401, 'Unauthorized') - return true - } - const vm = getVm(projectRoot, vmId) - if (!vm || isCodexVm(vm)) { - rejectUpgrade(socket, 404, 'Not Found') - return true - } const requestedLogin = url.searchParams.get('login') === '1' - if (requestedLogin !== !!entry.login) { - rejectUpgrade(socket, 409, 'Conflict') + if (authState) { + track( + consumeTicket(ticket, vmId, requestedLogin) + .then(acceptTicket) + .catch((error) => + rejectUpgrade( + socket, + error.code === 'shell_ticket_scope_mismatch' ? 409 : 503, + error.code === 'shell_ticket_scope_mismatch' ? 'Conflict' : 'Service Unavailable', + ), + ), + ) return true } - const login = !!entry.login - const accept = () => { - if (draining) { - rejectUpgrade(socket, 503, 'Service Unavailable') - return + return acceptTicket(consumeTicket(ticket, vmId)) + + function acceptTicket(entry) { + if (!entry) { + rejectUpgrade(socket, 401, 'Unauthorized') + return true } - if (sessions >= MAX_SESSIONS) { - rejectUpgrade(socket, 429, 'Too Many Requests') - return + const vm = getVm(projectRoot, vmId) + if (!vm || isCodexVm(vm)) { + rejectUpgrade(socket, 404, 'Not Found') + return true } - sessions += 1 - const size = { - cols: clampInt(url.searchParams.get('cols'), 10, 500, 80), - rows: clampInt(url.searchParams.get('rows'), 5, 200, 24), - login, + if (requestedLogin !== !!entry.login) { + rejectUpgrade(socket, 409, 'Conflict') + return true } - wss.handleUpgrade(req, socket, head, (ws) => { - track(new Promise((resolve) => ws.once('close', resolve))) - track(attach(ws, vm, size)) - }) - } - if (!login) { - accept() + const login = !!entry.login + const accept = () => { + if (draining || socket.destroyed) { + rejectUpgrade(socket, 503, 'Service Unavailable') + return + } + if (sessions >= MAX_SESSIONS) { + rejectUpgrade(socket, 429, 'Too Many Requests') + return + } + sessions += 1 + const size = { + cols: clampInt(url.searchParams.get('cols'), 10, 500, 80), + rows: clampInt(url.searchParams.get('rows'), 5, 200, 24), + login, + } + wss.handleUpgrade(req, socket, head, (ws) => { + track(new Promise((resolve) => ws.once('close', resolve))) + track(attach(ws, vm, size)) + }) + } + if (!login) { + accept() + return true + } + track( + Promise.resolve() + .then(() => (upgradeLoginReady || authLoginReady || assertAuthLoginReady)(projectRoot, vm)) + .then(accept, (err) => + rejectUpgrade( + socket, + 409, + String(err?.message || 'Conflict') + .replace(/[\r\n]+/g, ' ') + .slice(0, 120) || 'Conflict', + ), + ), + ) return true } - track( - Promise.resolve() - .then(() => (upgradeLoginReady || authLoginReady || assertAuthLoginReady)(projectRoot, vm)) - .then(accept, (err) => - rejectUpgrade( - socket, - 409, - String(err?.message || 'Conflict') - .replace(/[\r\n]+/g, ' ') - .slice(0, 120) || 'Conflict', - ), - ), - ) - return true } async function attach(ws, vm, size) { From e5a38abd87c1191a2e6dd3e6e5eb2e8482a6a0ec Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:43:18 +0800 Subject: [PATCH 168/473] feat: persist global maintenance phases outside restored schema --- src/lib/db/global-maintenance.mjs | 578 ++++++++++++++++++++++++++ test/unit/global-maintenance.test.mjs | 9 +- 2 files changed, 586 insertions(+), 1 deletion(-) create mode 100644 src/lib/db/global-maintenance.mjs diff --git a/src/lib/db/global-maintenance.mjs b/src/lib/db/global-maintenance.mjs new file mode 100644 index 000000000..9cc8de8d6 --- /dev/null +++ b/src/lib/db/global-maintenance.mjs @@ -0,0 +1,578 @@ +/** Global activity drain and durable restore latch, outside the restored application schema. */ +import crypto from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' +import pg from 'pg' +import { quoteIdentifier } from './database.mjs' + +const activities = new AsyncLocalStorage() +const TRY_SHARED = 'SELECT pg_try_advisory_lock_shared(hashtext(current_database()), hashtext($1)) AS acquired' +const TRY_SHARED_TX = 'SELECT pg_try_advisory_xact_lock_shared(hashtext(current_database()), hashtext($1)) AS acquired' +const UNLOCK_SHARED = 'SELECT pg_advisory_unlock_shared(hashtext(current_database()), hashtext($1))' +const TRY_EXCLUSIVE = 'SELECT pg_try_advisory_lock(hashtext(current_database()), hashtext($1)) AS acquired' +const EXCLUSIVE = 'SELECT pg_advisory_lock(hashtext(current_database()), hashtext($1))' +const UNLOCK_EXCLUSIVE = 'SELECT pg_advisory_unlock(hashtext(current_database()), hashtext($1))' +const BUSINESS_CLAIMS = [ + ['execution_admissions', 'released_at IS NULL'], + ['execution_jobs', 'ended_at IS NULL'], + ['api_key_admissions', 'released_at IS NULL'], + ['vm_operations', 'released_at IS NULL'], + ['background_task_runs', 'finished_at IS NULL'], + ['session_turns', 'true'], +] + +function failure(code, message, details = {}) { + return Object.assign(new Error(message), { code, ...details }) +} + +export function coordinationSchemaFor(applicationSchema) { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(applicationSchema)) throw new Error('invalid database schema') + return 'vm2api_coord_' + crypto.createHash('sha256').update(applicationSchema).digest('hex').slice(0, 32) +} + +function stateOf(row) { + return { + phase: row.phase, + epoch: Number(row.epoch), + operationId: row.operation_id, + ownerId: row.owner_id, + kind: row.kind, + reason: row.reason, + } +} + +/** One instance per process incarnation. A disconnected incarnation cannot reconnect its old scopes. */ +export class GlobalMaintenanceCoordinator { + constructor(db, { ownerId = crypto.randomUUID(), onLost = null } = {}) { + this.db = db + this.ownerId = ownerId + this.onLost = onLost + this.schema = coordinationSchemaFor(db.schema) + this.qualified = quoteIdentifier(this.schema) + this.activityResource = `vm2api:global-activity:${db.schema}` + this.adminResource = `vm2api:global-maintenance:${db.schema}` + this._client = null + this._healthy = false + this._lost = false + this._closing = false + this._held = false + this._refs = 0 + this._serial = Promise.resolve() + this._onError = (error) => this._lose(error) + this._onEnd = () => this._lose(failure('maintenance_connection_lost', 'Activity connection ended')) + } + + get healthy() { + return this._healthy + } + + _exclusiveLocal(fn) { + const pending = this._serial.then(fn) + this._serial = pending.catch(() => {}) + return pending + } + + _assertHealthy() { + if (!this._healthy || this._lost || this._closing) { + throw failure('maintenance_connection_lost', 'Activity connection is unavailable; restart is required') + } + } + + _lose(error) { + if (this._closing || this._lost) return + this._healthy = false + this._lost = true + this.onLost?.(error) + } + + _connection(label) { + return new pg.Client({ + connectionString: this.db.connectionString, + application_name: `vm2api-${label}`, + connectionTimeoutMillis: 5000, + statement_timeout: 60000, + }) + } + + async _transaction(fn) { + const client = await this.db.pool.connect() + try { + await client.query('BEGIN') + const result = await fn(client) + await client.query('COMMIT') + return result + } catch (error) { + await client.query('ROLLBACK').catch(() => {}) + throw error + } finally { + client.release() + } + } + + async _initialize() { + const q = this.qualified + await this._transaction(async (client) => { + await client.query('SELECT pg_advisory_xact_lock(hashtext(current_database()), hashtext($1))', [ + `vm2api:coordination-bootstrap:${this.db.schema}`, + ]) + const namespace = ( + await client.query( + `SELECT pg_get_userbyid(nspowner) = current_user AS owned + FROM pg_namespace WHERE nspname = $1`, + [this.schema], + ) + ).rows[0] + if (namespace && !namespace.owned) { + throw failure('maintenance_schema_owner', 'Coordination schema belongs to another role') + } + if (namespace) { + const control = (await client.query('SELECT to_regclass($1) AS name', [`${q}.control`])).rows[0].name + if (!control) throw failure('maintenance_schema_unrecognized', 'Coordination schema has no ownership record') + const binding = (await client.query(`SELECT application_schema FROM ${q}.control WHERE id = 1`)).rows[0] + if (binding?.application_schema !== this.db.schema) { + throw failure('maintenance_schema_mismatch', 'Coordination schema belongs to another application schema') + } + } + await client.query(`CREATE SCHEMA IF NOT EXISTS ${q} AUTHORIZATION CURRENT_USER; + REVOKE ALL ON SCHEMA ${q} FROM PUBLIC; + CREATE TABLE IF NOT EXISTS ${q}.control ( + id integer PRIMARY KEY CHECK (id = 1), + application_schema text NOT NULL, + phase text NOT NULL CHECK (phase IN ('open','draining','restoring','rebinding','blocked')), + epoch bigint NOT NULL CHECK (epoch > 0), + operation_id text, + owner_id text, + kind text, + reason text, + changed_at timestamptz NOT NULL DEFAULT clock_timestamp() + ); + CREATE TABLE IF NOT EXISTS ${q}.members ( + owner_id text PRIMARY KEY, + bound_epoch bigint, + started_at timestamptz NOT NULL DEFAULT clock_timestamp(), + stopped_at timestamptz + ); + CREATE TABLE IF NOT EXISTS ${q}.activities ( + id text PRIMARY KEY, + owner_id text NOT NULL, + epoch bigint NOT NULL, + kind text NOT NULL, + request_id text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + completed_at timestamptz, + uncertain_at timestamptz, + reason text + ); + CREATE INDEX IF NOT EXISTS activities_pending ON ${q}.activities (owner_id) WHERE completed_at IS NULL; + REVOKE ALL ON ALL TABLES IN SCHEMA ${q} FROM PUBLIC`) + await client.query( + `INSERT INTO ${q}.control(id,application_schema,phase,epoch) + VALUES (1,$1,'open',1) ON CONFLICT (id) DO NOTHING`, + [this.db.schema], + ) + }) + } + + async start() { + if (this._healthy) return this + if (this._starting) return this._starting + if (this._lost || this._closing) throw failure('maintenance_connection_lost', 'Restart is required') + this._starting = (async () => { + await this._initialize() + const client = this._connection('activity') + client.on('error', this._onError) + client.on('end', this._onEnd) + try { + await client.connect() + this._client = client + await this.db.pool.query(`INSERT INTO ${this.qualified}.members(owner_id) VALUES ($1)`, [this.ownerId]) + if (this._lost) throw failure('maintenance_connection_lost', 'Activity connection ended during startup') + this._healthy = true + return this + } catch (error) { + client.removeListener('error', this._onError) + client.removeListener('end', this._onEnd) + await client.end().catch(() => {}) + this._client = null + throw error + } + })() + try { + return await this._starting + } finally { + this._starting = null + } + } + + async getState() { + const row = (await this.db.pool.query(`SELECT * FROM ${this.qualified}.control WHERE id = 1`)).rows[0] + if (!row || row.application_schema !== this.db.schema) { + throw failure('maintenance_schema_mismatch', 'Coordination ownership record is unavailable') + } + return stateOf(row) + } + + async acknowledgeEpoch(epoch) { + this._assertHealthy() + return this._transaction(async (client) => { + const row = (await client.query(`SELECT * FROM ${this.qualified}.control WHERE id = 1 FOR SHARE`)).rows[0] + if (!Number.isSafeInteger(epoch) || Number(row.epoch) !== epoch || !['open', 'rebinding'].includes(row.phase)) { + throw failure('database_maintenance', 'The loaded application epoch cannot be acknowledged') + } + const updated = await client.query( + `UPDATE ${this.qualified}.members SET bound_epoch = $2 + WHERE owner_id = $1 AND stopped_at IS NULL`, + [this.ownerId, epoch], + ) + if (updated.rowCount !== 1) throw failure('maintenance_connection_lost', 'Pod registration is unavailable') + return stateOf(row) + }) + } + + /** New activities always use another session; session-lock reentry cannot bypass a restore waiter. */ + async admit({ kind = 'request', requestId = null } = {}) { + return this._exclusiveLocal(async () => { + this._assertHealthy() + try { + if (!this._held) { + const result = await this._client.query(TRY_SHARED, [this.activityResource]) + if (!result.rows[0].acquired) throw failure('database_maintenance', 'Global maintenance is in progress') + this._held = true + } + const id = crypto.randomUUID() + const epoch = await this._transaction(async (client) => { + if (!(await client.query(TRY_SHARED_TX, [this.activityResource])).rows[0].acquired) { + throw failure('database_maintenance', 'Global maintenance is waiting for activities to finish') + } + const row = (await client.query(`SELECT * FROM ${this.qualified}.control WHERE id = 1 FOR SHARE`)).rows[0] + if (row.phase !== 'open') throw failure('database_maintenance', 'Global maintenance is in progress') + const member = ( + await client.query( + `SELECT bound_epoch FROM ${this.qualified}.members + WHERE owner_id = $1 AND stopped_at IS NULL`, + [this.ownerId], + ) + ).rows[0] + if (!member || Number(member.bound_epoch) !== Number(row.epoch)) { + throw failure('maintenance_epoch_unbound', 'Pod caches have not loaded the current application epoch') + } + this._assertHealthy() + await client.query( + `INSERT INTO ${this.qualified}.activities(id,owner_id,epoch,kind,request_id) + VALUES ($1,$2,$3,$4,$5)`, + [id, this.ownerId, row.epoch, kind, requestId], + ) + return Number(row.epoch) + }) + this._assertHealthy() + this._refs += 1 + return this._gate(id, epoch) + } catch (error) { + if (!this._refs) await this._releaseHolder() + throw error + } + }) + } + + _gate(id, epoch) { + const scope = { coordinator: this, id, epoch, active: true } + const assertActive = () => { + this._assertHealthy() + if (!scope.active) throw failure('maintenance_activity_closed', 'The activity scope has completed') + } + const gate = { + activityId: id, + ownerId: this.ownerId, + epoch, + assertActive, + run: (fn) => { + assertActive() + return activities.run(scope, fn) + }, + complete: (execution) => this._complete(scope, execution), + markUncertain: (reason) => this._complete(scope, { ended: false, reason }), + } + return gate + } + + assertActivity() { + this._assertHealthy() + const scope = activities.getStore() + if (scope?.coordinator !== this || !scope.active) { + throw failure('maintenance_activity_closed', 'SQL requires a live admitted activity scope') + } + return scope + } + + async _complete(scope, execution) { + return this._exclusiveLocal(async () => { + this._assertHealthy() + const result = await this._transaction(async (client) => { + const row = ( + await client.query( + `SELECT * FROM ${this.qualified}.activities + WHERE id = $1 AND owner_id = $2 FOR UPDATE`, + [scope.id, this.ownerId], + ) + ).rows[0] + if (!row) throw failure('maintenance_activity_missing', 'The durable activity is unavailable') + if (row.completed_at) return { ended: true, uncertain: false } + if (row.uncertain_at) return { ended: false, uncertain: true } + const ended = execution?.ended === true + await client.query( + `UPDATE ${this.qualified}.activities SET + completed_at = CASE WHEN $3 THEN clock_timestamp() ELSE NULL END, + uncertain_at = CASE WHEN $3 THEN NULL ELSE clock_timestamp() END, + reason = $4 WHERE id = $1 AND owner_id = $2`, + [scope.id, this.ownerId, ended, execution?.reason || (ended ? 'completed' : 'execution_unknown')], + ) + return { ended, uncertain: !ended } + }) + if (scope.active) { + scope.active = false + this._refs -= 1 + if (!this._refs) await this._releaseHolder() + } + return result + }) + } + + async _releaseHolder() { + if (!this._held) return + if (!this._lost && this._client) await this._client.query(UNLOCK_SHARED, [this.activityResource]) + this._held = false + } + + async _checkClaims(client) { + const claims = { + activities: Number( + ( + await client.query(`SELECT count(*) FROM ${this.qualified}.activities + WHERE completed_at IS NULL`) + ).rows[0].count, + ), + } + for (const [table, condition] of BUSINESS_CLAIMS) { + const name = `${quoteIdentifier(this.db.schema)}.${quoteIdentifier(table)}` + if (!(await client.query('SELECT to_regclass($1) AS name', [name])).rows[0].name) { + throw failure('maintenance_claims_unavailable', `Cannot verify ${table} before maintenance`) + } + claims[table] = Number((await client.query(`SELECT count(*) FROM ${name} WHERE ${condition}`)).rows[0].count) + } + if (Object.values(claims).some((count) => count > 0)) { + throw failure('maintenance_claims_active', 'Unfinished or uncertain activities prevent maintenance', { claims }) + } + return claims + } + + async beginMaintenance({ kind, signal = null, timeoutMs = 60000 } = {}) { + this._assertHealthy() + if (!['backup', 'restore'].includes(kind)) throw new Error('invalid maintenance kind') + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) throw new Error('invalid maintenance timeout') + if (signal?.aborted) throw failure('maintenance_cancelled', 'Maintenance was cancelled before admission') + const client = this._connection('maintenance') + const operation = new MaintenanceOperation(this, client, kind) + try { + await client.connect() + operation.backendPid = (await client.query('SELECT pg_backend_pid() AS pid')).rows[0].pid + if (!(await client.query(TRY_EXCLUSIVE, [this.adminResource])).rows[0].acquired) { + throw failure('maintenance_busy', 'Another maintenance or schema operation is active') + } + await client.query('BEGIN') + try { + const row = (await client.query(`SELECT * FROM ${this.qualified}.control WHERE id = 1 FOR UPDATE`)).rows[0] + if (row.phase !== 'open') throw failure('database_maintenance', 'A durable maintenance operation is pending') + await client.query( + `UPDATE ${this.qualified}.control SET phase = 'draining', operation_id = $1, + owner_id = $2, kind = $3, reason = NULL, changed_at = clock_timestamp() WHERE id = 1`, + [operation.operationId, this.ownerId, kind], + ) + await client.query('COMMIT') + } catch (error) { + await client.query('ROLLBACK').catch(() => {}) + throw error + } + // DRAINING is committed before waiting. Existing scopes never reacquire this lock for their SQL. + await client.query("SELECT set_config('statement_timeout', $1, false)", [String(timeoutMs)]) + const abort = () => { + void operation.close() + } + signal?.addEventListener('abort', abort, { once: true }) + try { + if (signal?.aborted) throw failure('maintenance_cancelled', 'Maintenance drain was cancelled') + await client.query(EXCLUSIVE, [this.activityResource]) + } finally { + signal?.removeEventListener('abort', abort) + } + operation._assertHealthy() + await client.query("SELECT set_config('statement_timeout', '60000', false)") + await this._checkClaims(client) + return operation + } catch (error) { + await operation.close() + throw error + } + } + + /** Only a still-running owner may cancel its proven pre-restore stage. No crashed-owner takeover. */ + async cancelDrain(operationId) { + this._assertHealthy() + const client = this._connection('maintenance-cancel') + client.on('error', () => {}) + try { + await client.connect() + if (!(await client.query(TRY_EXCLUSIVE, [this.adminResource])).rows[0].acquired) { + throw failure('maintenance_busy', 'The maintenance owner is still active') + } + const result = await client.query( + `UPDATE ${this.qualified}.control SET phase = 'open', operation_id = NULL, + owner_id = NULL, kind = NULL, reason = NULL, changed_at = clock_timestamp() + WHERE id = 1 AND phase = 'draining' AND operation_id = $1 AND owner_id = $2`, + [operationId, this.ownerId], + ) + if (result.rowCount !== 1) + throw failure('maintenance_operation_changed', 'The maintenance operation cannot be cancelled') + return this.getState() + } finally { + await client.end().catch(() => {}) + } + } + + async stop() { + if (this._stopping) return this._stopping + this._stopping = this._exclusiveLocal(async () => { + this._closing = true + this._healthy = false + try { + if (this._client) { + await this.db.pool.query( + `UPDATE ${this.qualified}.members SET stopped_at = clock_timestamp() + WHERE owner_id = $1 AND NOT EXISTS (SELECT 1 FROM ${this.qualified}.activities + WHERE owner_id = $1 AND completed_at IS NULL)`, + [this.ownerId], + ) + } + } finally { + const client = this._client + this._client = null + if (client) { + client.removeListener('error', this._onError) + client.removeListener('end', this._onEnd) + await client.end().catch(() => {}) + } + } + }) + return this._stopping + } +} + +class MaintenanceOperation { + constructor(coordinator, client, kind) { + this.coordinator = coordinator + this.client = client + this.kind = kind + this.operationId = crypto.randomUUID() + this.backendPid = null + this._closed = false + this._lost = false + client.on('error', () => { + this._lost = true + }) + client.on('end', () => { + this._lost = true + }) + } + + _assertHealthy() { + this.coordinator._assertHealthy() + if (this._closed || this._lost) + throw failure('maintenance_connection_lost', 'Maintenance ownership connection ended') + } + + async _transaction(fn) { + this._assertHealthy() + const q = this.coordinator.qualified + await this.client.query('BEGIN') + try { + const row = (await this.client.query(`SELECT * FROM ${q}.control WHERE id = 1 FOR UPDATE`)).rows[0] + if (row.operation_id !== this.operationId || row.owner_id !== this.coordinator.ownerId) { + throw failure('maintenance_operation_changed', 'Maintenance ownership changed') + } + const result = await fn(row) + this._assertHealthy() + await this.client.query('COMMIT') + return result + } catch (error) { + await this.client.query('ROLLBACK').catch(() => {}) + throw error + } + } + + async beginRestore() { + return this._transaction(async (row) => { + if (this.kind !== 'restore' || row.phase !== 'draining') + throw failure('maintenance_operation_changed', 'Restore is not in its pre-restore stage') + await this.coordinator._checkClaims(this.client) + if (!Number.isSafeInteger(Number(row.epoch) + 1)) throw new Error('application epoch exhausted') + const result = await this.client.query(`UPDATE ${this.coordinator.qualified}.control SET phase = 'restoring', + epoch = epoch + 1, changed_at = clock_timestamp() WHERE id = 1 RETURNING *`) + return stateOf(result.rows[0]) + }) + } + + async beginRebind() { + return this._transaction(async (row) => { + if (row.phase !== 'restoring') + throw failure('maintenance_operation_changed', 'Restore has not completed its data replacement stage') + const result = await this.client.query(`UPDATE ${this.coordinator.qualified}.control SET phase = 'rebinding', + changed_at = clock_timestamp() WHERE id = 1 RETURNING *`) + return stateOf(result.rows[0]) + }) + } + + async resume() { + const state = await this._transaction(async (row) => { + if ( + (this.kind === 'restore' && row.phase !== 'rebinding') || + (this.kind === 'backup' && row.phase !== 'draining') + ) { + throw failure('maintenance_operation_changed', 'The maintenance phase does not permit resume') + } + await this.coordinator._checkClaims(this.client) + const pending = ( + await this.client.query( + `SELECT owner_id FROM ${this.coordinator.qualified}.members + WHERE stopped_at IS NULL AND bound_epoch IS DISTINCT FROM $1`, + [row.epoch], + ) + ).rows + if (pending.length) + throw failure('maintenance_rebind_pending', 'Some Pods have not rebound to the restored epoch', { + owners: pending.map((member) => member.owner_id), + }) + const result = await this.client.query(`UPDATE ${this.coordinator.qualified}.control SET phase = 'open', + operation_id = NULL, owner_id = NULL, kind = NULL, reason = NULL, changed_at = clock_timestamp() + WHERE id = 1 RETURNING *`) + return stateOf(result.rows[0]) + }) + await this.client.query(UNLOCK_EXCLUSIVE, [this.coordinator.activityResource]) + return state + } + + async failBlocked(reason) { + return this._transaction(async () => { + const result = await this.client.query( + `UPDATE ${this.coordinator.qualified}.control SET phase = 'blocked', + reason = $1, changed_at = clock_timestamp() WHERE id = 1 RETURNING *`, + [reason || 'maintenance_failed'], + ) + return stateOf(result.rows[0]) + }) + } + + async close() { + if (this._closed) return + this._closed = true + // Closing releases session locks, never the durable latch or unfinished claims. + await this.client.end().catch(() => {}) + } +} diff --git a/test/unit/global-maintenance.test.mjs b/test/unit/global-maintenance.test.mjs index 5f20229c2..a4f377501 100644 --- a/test/unit/global-maintenance.test.mjs +++ b/test/unit/global-maintenance.test.mjs @@ -1,6 +1,7 @@ import test from 'node:test' import assert from 'node:assert/strict' import { setTimeout as delay } from 'node:timers/promises' +import pg from 'pg' import { createDatabase, getDb, quoteIdentifier } from '../../src/lib/db/database.mjs' async function fixture(t, { acknowledge = true } = {}) { @@ -16,7 +17,13 @@ async function fixture(t, { acknowledge = true } = {}) { t.after(async () => { await Promise.allSettled(operations.map((operation) => operation.close())) await Promise.allSettled([first.stop(), second.stop()]) - await db.pool.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(schema)} CASCADE`) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(schema)} CASCADE`) + } finally { + await cleanup.end() + } }) await first.start() await second.start() From 5920a2500b8bbe66fc54614cb487eb8f66533858 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:44:36 +0800 Subject: [PATCH 169/473] test: require a durable activity for startup migrations --- test/unit/global-maintenance.test.mjs | 59 +++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/unit/global-maintenance.test.mjs b/test/unit/global-maintenance.test.mjs index a4f377501..a8c31476f 100644 --- a/test/unit/global-maintenance.test.mjs +++ b/test/unit/global-maintenance.test.mjs @@ -3,6 +3,7 @@ import assert from 'node:assert/strict' import { setTimeout as delay } from 'node:timers/promises' import pg from 'pg' import { createDatabase, getDb, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { runPgTool } from '../../src/lib/admin/backup-core.mjs' async function fixture(t, { acknowledge = true } = {}) { const { GlobalMaintenanceCoordinator, coordinationSchemaFor } = await import( @@ -306,3 +307,61 @@ test('stopping or losing a Pod preserves all of its unfinished durable activitie code: 'maintenance_claims_active', }) }) + +test('startup migration is admitted before caches are bound and finishes before the exclusive drain', async (t) => { + const { db, first, second, operations } = await fixture(t, { acknowledge: false }) + await first.acknowledgeEpoch(1) + const startup = await second.admitBootstrap() + const pending = first.beginMaintenance({ kind: 'backup', timeoutMs: 5000 }) + await waitForPhase(second, 'draining') + await assert.rejects(first.admitBootstrap(), { code: 'database_maintenance' }) + await startup.run(async () => { + second.assertActivity() + await db.exec('CREATE TABLE synthetic_startup (id integer)') + await second.acknowledgeEpoch(startup.epoch) + }) + await startup.complete({ ended: true }) + const operation = await pending + operations.push(operation) + await operation.resume() + await operation.close() + await assert.rejects(second.admitBootstrap(), { code: 'maintenance_bootstrap_bound' }) +}) + +test('business-schema-only pg_dump excludes the persistent coordination latch and activities', async (t) => { + const { db, schema } = await fixture(t) + const dump = await runPgTool( + 'pg_dump', + ['--schema-only', '--no-owner', '--no-privileges', `--schema=${db.schema}`], + db.connectionString, + ) + assert.ok(dump.includes('CREATE TABLE ' + db.schema + '.settings')) + assert.equal(dump.includes(schema), false) + assert.equal(dump.includes('CREATE TABLE activities'), false) +}) + +test('an existing coordination schema cannot be adopted for another application binding', async (t) => { + const { db, peer, schema } = await fixture(t) + await db.pool.query(`UPDATE ${quoteIdentifier(schema)}.control SET application_schema = 'another_tenant' WHERE id=1`) + const { GlobalMaintenanceCoordinator } = await import('../../src/lib/db/global-maintenance.mjs') + const stranger = new GlobalMaintenanceCoordinator(peer, { ownerId: 'stranger' }) + await assert.rejects(stranger.start(), { code: 'maintenance_schema_mismatch' }) + assert.equal( + (await db.pool.query(`SELECT count(*) AS n FROM ${quoteIdentifier(schema)}.members WHERE owner_id='stranger'`)) + .rows[0].n, + 0, + ) +}) + +test('a drain timeout preserves the pre-restore latch and only its original owner can cancel it', async (t) => { + const { first, second } = await fixture(t) + const gate = await second.admit({ kind: 'request' }) + await assert.rejects(first.beginMaintenance({ kind: 'restore', timeoutMs: 30 }), { code: '57014' }) + const state = await second.getState() + assert.equal(state.phase, 'draining') + assert.equal(state.epoch, 1) + await assert.rejects(second.cancelDrain(state.operationId), { code: 'maintenance_operation_changed' }) + await gate.complete({ ended: true }) + await first.cancelDrain(state.operationId) + assert.equal((await second.getState()).phase, 'open') +}) From 7bc6f6a4e9938374722dcfc14c4a476e5d215264 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:46:16 +0800 Subject: [PATCH 170/473] fix: hold a global activity across startup initialization --- src/lib/db/global-maintenance.mjs | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/lib/db/global-maintenance.mjs b/src/lib/db/global-maintenance.mjs index 9cc8de8d6..1f3043138 100644 --- a/src/lib/db/global-maintenance.mjs +++ b/src/lib/db/global-maintenance.mjs @@ -56,6 +56,7 @@ export class GlobalMaintenanceCoordinator { this._closing = false this._held = false this._refs = 0 + this._bootstrapUsed = false this._serial = Promise.resolve() this._onError = (error) => this._lose(error) this._onEnd = () => this._lose(failure('maintenance_connection_lost', 'Activity connection ended')) @@ -215,7 +216,11 @@ export class GlobalMaintenanceCoordinator { this._assertHealthy() return this._transaction(async (client) => { const row = (await client.query(`SELECT * FROM ${this.qualified}.control WHERE id = 1 FOR SHARE`)).rows[0] - if (!Number.isSafeInteger(epoch) || Number(row.epoch) !== epoch || !['open', 'rebinding'].includes(row.phase)) { + if ( + !Number.isSafeInteger(epoch) || + Number(row.epoch) !== epoch || + !['open', 'draining', 'rebinding'].includes(row.phase) + ) { throw failure('database_maintenance', 'The loaded application epoch cannot be acknowledged') } const updated = await client.query( @@ -229,7 +234,16 @@ export class GlobalMaintenanceCoordinator { } /** New activities always use another session; session-lock reentry cannot bypass a restore waiter. */ - async admit({ kind = 'request', requestId = null } = {}) { + admit(metadata = {}) { + return this._admit(metadata, false) + } + + /** Hold across migrations and initial cache loading, before the first epoch acknowledgement. */ + admitBootstrap({ requestId = null } = {}) { + return this._admit({ kind: 'bootstrap', requestId }, true) + } + + async _admit({ kind = 'request', requestId = null }, bootstrap) { return this._exclusiveLocal(async () => { this._assertHealthy() try { @@ -252,7 +266,13 @@ export class GlobalMaintenanceCoordinator { [this.ownerId], ) ).rows[0] - if (!member || Number(member.bound_epoch) !== Number(row.epoch)) { + if (bootstrap && (this._bootstrapUsed || member?.bound_epoch != null)) { + throw failure( + 'maintenance_bootstrap_bound', + 'This Pod incarnation has already initialized its application state', + ) + } + if (!member || (!bootstrap && Number(member.bound_epoch) !== Number(row.epoch))) { throw failure('maintenance_epoch_unbound', 'Pod caches have not loaded the current application epoch') } this._assertHealthy() @@ -265,7 +285,8 @@ export class GlobalMaintenanceCoordinator { }) this._assertHealthy() this._refs += 1 - return this._gate(id, epoch) + if (bootstrap) this._bootstrapUsed = true + return this._gate(id, epoch, kind) } catch (error) { if (!this._refs) await this._releaseHolder() throw error @@ -273,8 +294,8 @@ export class GlobalMaintenanceCoordinator { }) } - _gate(id, epoch) { - const scope = { coordinator: this, id, epoch, active: true } + _gate(id, epoch, kind) { + const scope = { coordinator: this, id, epoch, kind, active: true } const assertActive = () => { this._assertHealthy() if (!scope.active) throw failure('maintenance_activity_closed', 'The activity scope has completed') From 72294ad7de758fdf3ac80e5e49bf0d5956012fdc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:46:27 +0800 Subject: [PATCH 171/473] test: reject scheduled callbacks acquired after stopping --- .../unit/shared-background-producers.test.mjs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/unit/shared-background-producers.test.mjs b/test/unit/shared-background-producers.test.mjs index b026651c6..44736e936 100644 --- a/test/unit/shared-background-producers.test.mjs +++ b/test/unit/shared-background-producers.test.mjs @@ -253,6 +253,45 @@ test('stop prevents an already queued immediate monitor run from sending a new t assert.equal(calls, 0) }) +test('stop cancels a scheduled callback whose durable claim is still being acquired', async (t) => { + const f = await fixture(t) + const entered = deferred() + const finish = deferred() + const claim = f.first.tryClaim.bind(f.first) + f.first.tryClaim = async (plan) => { + const acquired = await claim(plan) + entered.resolve() + await finish.promise + return acquired + } + let tick + t.mock.method(globalThis, 'setInterval', (callback) => { + tick = callback + return { unref() {} } + }) + t.mock.method(globalThis, 'clearInterval', () => {}) + let calls = 0 + const monitor = createCredentialRefreshMonitor({ + sharedTasks: f.first, + listTargets: () => [live], + refreshOne: async () => { + calls++ + return { ok: true } + }, + }) + monitor.start() + tick() + await entered.promise + const stopping = monitor.stop() + finish.resolve() + await stopping + await monitor.flush() + assert.equal(calls, 0) + const row = await f.other.prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.equal(row.unknown_at, null) + assert.ok(row.finished_at) +}) + test('scheduled backups execute once across services without touching business files', async (t) => { const f = await fixture(t) const root = fs.mkdtempSync(path.join(os.tmpdir(), 'synthetic-bg-backup-')) From cf42be3c2ac385c36e9bdf74aa2a9fdda24650e8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:47:45 +0800 Subject: [PATCH 172/473] test: distinguish proven terminal background exceptions --- test/unit/shared-background-tasks.test.mjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/unit/shared-background-tasks.test.mjs b/test/unit/shared-background-tasks.test.mjs index f78f6291c..37a81ecbc 100644 --- a/test/unit/shared-background-tasks.test.mjs +++ b/test/unit/shared-background-tasks.test.mjs @@ -91,7 +91,13 @@ test('explicit terminal failures and known not-started errors permit later plans }), /synthetic validation failure/, ) - const next = await second.tryClaim(plan('four')) + await assert.rejects( + first.run(plan('four'), () => { + throw Object.assign(new Error('synthetic terminal rejection'), { taskTerminal: true }) + }), + /synthetic terminal rejection/, + ) + const next = await second.tryClaim(plan('five')) assert.equal(next.ok, true) await next.complete() }) From 56b3c3cfbfe3af913571bdac801d4549622331bf Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:49:29 +0800 Subject: [PATCH 173/473] test: verify fourteen migrations and one-time state backup restore --- test/unit/backup-service.test.mjs | 8 +++++--- test/unit/database.test.mjs | 9 ++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 4150827d1..6a5c992bc 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,7 +79,7 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; @@ -107,16 +107,18 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'session_turns', 'vm_operations', 'background_task_runs', + 'oauth_pkce_sessions', + 'shell_tickets', ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 13) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 14) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index f0a06ccd7..518376e24 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 38) + assert.equal(rows.length, 41) for (const name of [ 'settings', 'users', @@ -53,6 +53,9 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'execution_jobs', 'session_turns', 'vm_operations', + 'background_task_runs', + 'oauth_pkce_sessions', + 'shell_tickets', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -113,7 +116,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 12) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 14) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -129,7 +132,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 12) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 14) fs.rmSync(dir, { recursive: true, force: true }) }) From f081b9798d72a45e762ef880af65dc2298c73654 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:49:55 +0800 Subject: [PATCH 174/473] feat(k8s): coordinate background producers and discover shared schedules --- src/lib/admin/backup-service.mjs | 53 +++-- src/lib/admin/health-probe.mjs | 107 +++++++++- src/lib/admin/vm-test-schedule.mjs | 188 ++++++++++++++---- src/lib/core/shared-background-tasks.mjs | 2 +- .../repos/vm-test-schedule-results-repo.mjs | 34 ++++ src/lib/oauth/credential-refresh-monitor.mjs | 56 +++++- src/lib/oauth/usage-probe-monitor.mjs | 162 ++++++++++----- 7 files changed, 472 insertions(+), 130 deletions(-) create mode 100644 src/lib/db/repos/vm-test-schedule-results-repo.mjs diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index f1fd2045d..2640513ef 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -8,6 +8,7 @@ import { sha256FileAsync, runPgTool, runProcess, extractArchiveMember } from './ import { SettingsRepo } from '../db/repos/settings-repo.mjs' import { BackupRepo } from '../db/repos/backup-repo.mjs' import { invalidateReadCache } from '../vm/vm-file.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' // tarAvailability / sha256File / the runnable pipeline live in backup-core.mjs; // keep the historical export surface for tests and callers. @@ -22,13 +23,14 @@ export const DEFAULT_SCHEDULE = { } export class BackupService { - constructor({ dataDir, projectRoot, configDir, backupDir, tmpDir, checkIntervalMs = 10 * 60_000 } = {}) { + constructor({ dataDir, projectRoot, configDir, backupDir, tmpDir, checkIntervalMs = 10 * 60_000, sharedTasks } = {}) { this.dataDir = dataDir this.projectRoot = projectRoot this.configDir = configDir || null this.backupDir = backupDir || path.join(dataDir, 'backups') this.tmpDir = tmpDir || path.join(dataDir, 'tmp') this.checkIntervalMs = checkIntervalMs + this.sharedTasks = backgroundTasksFor(sharedTasks) this.maintenanceFile = path.join(dataDir, 'restore-maintenance.json') this.isRestoring = fs.existsSync(this.maintenanceFile) this.isMaintaining = this.isRestoring @@ -576,24 +578,49 @@ export class BackupService { // ---------- scheduler ---------- /** Auto-backup checker: every checkIntervalMs + a catch-up run at boot. */ + runScheduled(generation = this._schedulerGeneration) { + const pending = (async () => { + const now = this.sharedTasks ? await this.sharedTasks.now() : Date.now() + if (!(await this.shouldRunScheduled(now)) || generation !== this._schedulerGeneration) return null + const config = await this.getSchedule() + const backup = async () => { + if (generation !== this._schedulerGeneration) return { notStarted: true, value: null } + if (this.sharedTasks && !(await this.shouldRunScheduled(await this.sharedTasks.now()))) { + return { notStarted: true, value: null } + } + const record = await this.createBackup({ kind: 'scheduled' }) + return { terminal: record?.status === 'ok', value: record } + } + if (!this.sharedTasks) return (await backup()).value + const result = await this.sharedTasks.run( + { taskKey: 'scheduled-backup', intervalMs: Math.round(config.interval_hours * 3600_000) }, + backup, + ) + return result.ran ? result.value : null + })() + this._scheduledChecks.add(pending) + pending.then( + () => this._scheduledChecks.delete(pending), + () => this._scheduledChecks.delete(pending), + ) + return pending + } + startScheduler() { this.stopScheduler() const generation = this._schedulerGeneration const check = () => { if (generation !== this._schedulerGeneration) return - const pending = (async () => { - try { - if ((await this.shouldRunScheduled()) && generation === this._schedulerGeneration) { - const rec = await this.createBackup({ kind: 'scheduled' }) + return this.runScheduled(generation).then( + (rec) => { + if (rec) { console.log(`[backup] scheduled backup completed (${Math.round((rec.size_bytes || 0) / 1048576)} MiB)`) } - } catch (e) { + }, + (e) => { console.warn('[backup] scheduled backup failed:', String(e.message || e)) - } - })() - this._scheduledChecks.add(pending) - pending.finally(() => this._scheduledChecks.delete(pending)) - return pending + }, + ) } this._timer = setInterval(check, this.checkIntervalMs) if (typeof this._timer.unref === 'function') this._timer.unref() @@ -602,7 +629,8 @@ export class BackupService { if (typeof this._bootTimer.unref === 'function') this._bootTimer.unref() } - async shouldRunScheduled(now = Date.now()) { + async shouldRunScheduled(now) { + now ??= this.sharedTasks ? await this.sharedTasks.now() : Date.now() const cfg = await this.getSchedule() if (!cfg.enabled || this._running || this._restoreRunning || this.isMaintaining) return false const last = await this._repo().lastSuccessful({}) @@ -620,5 +648,6 @@ export class BackupService { clearTimeout(this._bootTimer) this._bootTimer = null } + return this.flush().catch(() => {}) } } diff --git a/src/lib/admin/health-probe.mjs b/src/lib/admin/health-probe.mjs index 267417cd4..776f49ccc 100644 --- a/src/lib/admin/health-probe.mjs +++ b/src/lib/admin/health-probe.mjs @@ -11,6 +11,9 @@ import { isValidVmId } from '../vm/vm-file.mjs' import { activeHoursOf, evaluateActiveHours } from '../pool/active-hours.mjs' import { fromClaudeToOpenAIChat, fromClaudeToOpenAICompletions, fromClaudeToResponses } from '../protocol/convert.mjs' import { detectInboundPlatform } from '../protocol/platform-detect.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { getSharedRoutingState } from './shared-routing-config.mjs' export const HEALTH_REAL_HEADER = 'x-kin-health-real' export const HEALTH_VIA_CACHE = 'health-cache' @@ -616,6 +619,9 @@ export function createHealthProbeMonitor(opts = {}) { let lastRunAt = 0 let inflight = null let timer = null + let generation = 0 + let unknown = false + const sharedTasks = backgroundTasksFor(opts.sharedTasks) const nowFn = opts.now || (() => Date.now()) const getConfig = () => config @@ -699,12 +705,14 @@ export function createHealthProbeMonitor(opts = {}) { const probeVm = async (vm) => { try { const result = await opts.runChat(vm, real) + if (!result?.ok && result?.taskTerminal !== true && result?.taskNotStarted !== true) unknown = true if (result?.ok) return { ok: true, result } return { ok: false, error: result?.error?.message || result?.error || 'probe_failed', } } catch (e) { + if (e?.taskTerminal !== true && e?.taskNotStarted !== true) unknown = true return { ok: false, error: String(e?.message || e).slice(0, 300) } } } @@ -764,16 +772,92 @@ export function createHealthProbeMonitor(opts = {}) { } } - const runRealProbe = async () => { + const sharedSnapshot = (platform) => { + const snapshot = snapshots[platform] + if (!snapshot) return null + const usage = {} + for (const key of ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens']) { + const value = snapshot.body?.usage?.[key] + if (Number.isFinite(value) && value >= 0) usage[key] = value + } + const view = getSnapshot(platform) + return { + ok: snapshot.ok, + at: view.at, + vm_id: view.vm_id, + model: view.model, + message_id: view.message_id, + text: view.text, + status: view.status, + error: view.error, + duration_ms: view.duration_ms, + usage, + stop_reason: snapshot.body?.stop_reason || 'end_turn', + stop_sequence: snapshot.body?.stop_sequence ?? null, + } + } + + const acceptSharedSnapshot = (platform, value) => { + if (!value?.at) return + snapshots[platform] = { + ok: value.ok, + at: value.at, + vm_id: value.vm_id, + model: value.model, + text: value.text, + status: value.status, + error: value.error, + duration_ms: value.duration_ms, + body: value.message_id + ? { + id: value.message_id, + type: 'message', + role: 'assistant', + model: value.model, + content: [{ type: 'text', text: value.text || '' }], + usage: value.usage, + stop_reason: value.stop_reason, + stop_sequence: value.stop_sequence, + } + : undefined, + } + } + + const runRealProbe = async ({ scheduled = false } = {}) => { if (inflight) return inflight + const current = generation inflight = (async () => { - const started = nowFn() - lastRunAt = started - await probePlatform('anthropic', config.real, started) - if (config.real.openai_model) { - await probePlatform('openai', { ...config.real, model: config.real.openai_model }, started) - } else { - snapshots.openai = null + if (isKubernetesMode()) { + const routing = getSharedRoutingState(sharedTasks?.repo.db) + if (!routing) + throw Object.assign(new Error('shared routing unavailable'), { code: 'routing_config_unavailable' }) + await routing.refresh() + } + if (scheduled && (current !== generation || !config.enabled)) return getSnapshot('anthropic') + const probe = async () => { + if (scheduled && (current !== generation || !config.enabled)) + return { notStarted: true, value: getSnapshot('anthropic') } + const started = nowFn() + lastRunAt = started + unknown = false + await probePlatform('anthropic', config.real, started) + if (config.real.openai_model) { + await probePlatform('openai', { ...config.real, model: config.real.openai_model }, started) + } else { + snapshots.openai = null + } + return { + terminal: !unknown, + value: getSnapshot('anthropic'), + sharedResult: { anthropic: sharedSnapshot('anthropic'), openai: sharedSnapshot('openai') }, + } + } + if (!sharedTasks) return (await probe()).value + const result = await sharedTasks.run({ taskKey: 'health-probe', intervalMs: config.interval_sec * 1000 }, probe) + if (result.ran) return result.value + if (result.sharedResult) { + acceptSharedSnapshot('anthropic', result.sharedResult.anthropic) + acceptSharedSnapshot('openai', result.sharedResult.openai) } return getSnapshot('anthropic') })().finally(() => { @@ -783,22 +867,25 @@ export function createHealthProbeMonitor(opts = {}) { } const stop = () => { + generation++ if (timer) { clearInterval(timer) timer = null } + return inflight ? Promise.allSettled([inflight]) : Promise.resolve() } const start = ({ immediate = false } = {}) => { stop() if (!config.enabled) return { started: false, reason: 'disabled' } + const current = generation timer = setInterval(() => { - runRealProbe().catch(() => {}) + if (current === generation) runRealProbe({ scheduled: true }).catch(() => {}) }, config.interval_sec * 1000) if (typeof timer.unref === 'function') timer.unref() if (immediate && config.run_on_start) { queueMicrotask(() => { - runRealProbe().catch(() => {}) + if (current === generation) runRealProbe({ scheduled: true }).catch(() => {}) }) } return { started: true, interval_sec: config.interval_sec, run_on_start: config.run_on_start } diff --git a/src/lib/admin/vm-test-schedule.mjs b/src/lib/admin/vm-test-schedule.mjs index 4134a886d..02105eb40 100644 --- a/src/lib/admin/vm-test-schedule.mjs +++ b/src/lib/admin/vm-test-schedule.mjs @@ -5,6 +5,8 @@ import { isValidVmId } from '../vm/vm-file.mjs' import { isCodexVm } from '../vm/vm-kind.mjs' import { getVm, listVms } from '../vm/vm-registry.mjs' import { listTestableModels } from './vm-test-chat.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { VmTestScheduleResultsRepo } from '../db/repos/vm-test-schedule-results-repo.mjs' export const VM_TEST_SCHEDULE_PREFIX = 'vm_test_schedule:' export const VM_TEST_SCHEDULE_DEFAULT_KEY = 'vm_test_schedule_default' @@ -345,13 +347,21 @@ export class VmTestScheduleRunner { runTest, list = () => listVms(projectRoot), now = () => Date.now(), + sharedTasks, } = {}) { this.projectRoot = projectRoot this.settingsFactory = typeof settings === 'function' ? settings : () => settings this.runTest = runTest this.listVms = list this.now = now + this.sharedTasks = backgroundTasksFor(sharedTasks) + this.resultRepo = this.sharedTasks ? new VmTestScheduleResultsRepo(this.sharedTasks.repo.db) : null this.jobs = new Map() + this.jobRevisions = new Map() + this.discoveryGeneration = 0 + this.lifecycleGeneration = 0 + this.discoveries = new Set() + this.discoveryTimer = null this.running = new Set() this.writeFailure = null this.started = false @@ -369,8 +379,7 @@ export class VmTestScheduleRunner { return this.listVms().find((vm) => vm.id === vmId) || getVm(this.projectRoot, vmId) } - _stored(vmId) { - const raw = this._settings().get(keyFor(vmId)) + _stored(vmId, raw = this._settings().get(keyFor(vmId))) { if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null const vm = this._vm(vmId) if (!vm) return null @@ -480,6 +489,7 @@ export class VmTestScheduleRunner { const previous = this._settings().get(keyFor(vmId)) || {} const config = normalizeVmTestSchedule(input, vm) await this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) + this.discoveryGeneration++ if (this.started) this._syncJob(vmId, config) return this.get(vmId) } @@ -518,6 +528,7 @@ export class VmTestScheduleRunner { ...savedResult(previous), revision: randomUUID(), }) + this.discoveryGeneration++ if (this.started) this._syncJob(vmId, config) return this.get(vmId) } @@ -558,6 +569,7 @@ export class VmTestScheduleRunner { const stored = this._settings().get(keyFor(vmId)) if (stored) { await this._settings().set(keyFor(vmId), { ...stored, enabled: false }) + this.discoveryGeneration++ this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) } @@ -582,103 +594,191 @@ export class VmTestScheduleRunner { } else dialog = await defaultDialogParams(vm) const config = normalizeVmTestSchedule({ enabled: full.enabled, ...structure, ...dialog }, vm) await this._settings().set(keyFor(vmId), { ...config, ...savedResult(previous), revision: randomUUID() }) + this.discoveryGeneration++ if (this.started) this._syncJob(vmId, config) } async remove(vmId) { + this.discoveryGeneration++ this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) + this.jobRevisions.delete(vmId) await this._settings().remove?.(keyFor(vmId)) return null } async start() { + const generation = ++this.lifecycleGeneration this.started = true - await this.reload() + clearInterval(this.discoveryTimer) + const discovery = this.reload() + await discovery + if (this.sharedTasks && this.started && generation === this.lifecycleGeneration) { + this.discoveryTimer = setInterval(() => { + if (!this.started) return + if (this.discoveries.size) return Promise.allSettled([...this.discoveries]) + return this.reload().catch((error) => { + this.writeFailure ||= error + }) + }, 1000) + this.discoveryTimer.unref?.() + } } async flush() { - while (this.running.size) await new Promise((resolve) => setTimeout(resolve, 10)) + while (this.running.size || this.discoveries.size) { + if (this.discoveries.size) await Promise.allSettled([...this.discoveries]) + else await new Promise((resolve) => setTimeout(resolve, 10)) + } if (this.writeFailure) throw this.writeFailure } stop() { this.started = false + this.lifecycleGeneration++ + this.discoveryGeneration++ + clearInterval(this.discoveryTimer) + this.discoveryTimer = null for (const job of this.jobs.values()) job.stop() this.jobs.clear() + this.jobRevisions.clear() + return this.flush().catch(() => {}) } - async reload() { - for (const job of this.jobs.values()) job.stop() - this.jobs.clear() - const all = this._settings().all?.() || {} + reload() { + const generation = ++this.discoveryGeneration + const pending = this._reload(generation) + this.discoveries.add(pending) + pending.then( + () => this.discoveries.delete(pending), + () => this.discoveries.delete(pending), + ) + return pending + } + + async _reload(generation) { + const all = this.resultRepo ? await this.resultRepo.listPlans() : this._settings().all?.() || {} + if (generation !== this.discoveryGeneration || !this.started) return + if (!this.sharedTasks) { + for (const job of this.jobs.values()) job.stop() + this.jobs.clear() + this.jobRevisions.clear() + } + const desired = new Map() for (const key of Object.keys(all)) { if (!key.startsWith(VM_TEST_SCHEDULE_PREFIX)) continue const vmId = key.slice(VM_TEST_SCHEDULE_PREFIX.length) const vm = this._vm(vmId) if (!vm) { await this._settings().remove?.(key) + if (generation !== this.discoveryGeneration || !this.started) return continue } - const schedule = this._stored(vmId) - if (schedule?.enabled) this._syncJob(vmId, schedule) + const schedule = this._stored(vmId, all[key]) + if (schedule?.enabled) desired.set(vmId, schedule) + } + for (const [id, job] of this.jobs) { + if (!desired.has(id)) { + job.stop() + this.jobs.delete(id) + this.jobRevisions.delete(id) + } + } + for (const [id, schedule] of desired) { + if (!this.jobs.has(id) || this.jobRevisions.get(id) !== schedule.revision) this._syncJob(id, schedule) } } _syncJob(vmId, schedule) { this.jobs.get(vmId)?.stop() this.jobs.delete(vmId) + this.jobRevisions.delete(vmId) if (!this.started || !schedule.enabled || !this._vm(vmId)) return + const revision = schedule.revision ?? this._settings().get(keyFor(vmId))?.revision const job = new Cron( schedule.cron, { timezone: schedule.timezone, mode: '5-part', catch: true, unref: true }, - async () => await this._run(vmId), + async () => await this._run(vmId, revision), ) this.jobs.set(vmId, job) + this.jobRevisions.set(vmId, revision) } - async _run(vmId) { + async _run(vmId, expectedRevision) { if (!this.started || this.running.has(vmId)) return - if (!this._vm(vmId)) { - await this.remove(vmId) - return - } - const schedule = this._stored(vmId) - if (!schedule?.enabled) return this.running.add(vmId) - const startedAt = this.now() try { - let result - try { - result = await this.runTest(vmId, { - model: schedule.model, - prompt: schedule.prompt, - max_tokens: schedule.max_tokens, - reasoning_effort: schedule.reasoning_effort, - }) - } catch (error) { - result = { ok: false, error } - } + const settings = this._settings() + if (this.sharedTasks) await settings.refresh?.() if (!this.started) return - - const current = this._settings().get(keyFor(vmId)) - if (!current || String(current.revision || '') !== schedule.revision || !this._vm(vmId)) return - const duration = Number(result?.duration_ms) - const lastDuration = - Number.isSafeInteger(duration) && duration >= 0 ? duration : Math.max(0, this.now() - startedAt) - const lastOk = result?.ok === true - try { - await this._settings().set(keyFor(vmId), { - ...current, + if (!this._vm(vmId)) { + await this.remove(vmId) + return + } + const schedule = this._stored(vmId) + if (!schedule?.enabled) return + if (expectedRevision != null && expectedRevision !== schedule.revision) { + this.discoveryGeneration++ + this._syncJob(vmId, schedule) + return + } + const startedAt = this.sharedTasks ? await this.sharedTasks.now() : this.now() + const execute = async () => { + if (!this.started || (this.resultRepo && !(await this.resultRepo.isCurrent(vmId, schedule.revision)))) { + return { notStarted: true } + } + let result + try { + result = await this.runTest(vmId, { + model: schedule.model, + prompt: schedule.prompt, + max_tokens: schedule.max_tokens, + reasoning_effort: schedule.reasoning_effort, + }) + } catch (error) { + result = { ok: false, error, taskTerminal: error?.taskTerminal, taskNotStarted: error?.taskNotStarted } + } + const terminal = result?.ok === true || result?.taskTerminal === true || result?.taskNotStarted === true + if (!this.started) return { terminal } + if (this.sharedTasks) await settings.refresh?.() + const current = settings.get(keyFor(vmId)) + if (!current || String(current.revision || '') !== schedule.revision || !this._vm(vmId)) return { terminal } + const duration = Number(result?.duration_ms) + const lastDuration = + Number.isSafeInteger(duration) && duration >= 0 ? duration : Math.max(0, this.now() - startedAt) + const lastOk = result?.ok === true + const saved = { last_run_at: new Date(startedAt).toISOString(), last_ok: lastOk, last_duration_ms: lastDuration, last_error: lastOk ? null : compactError(result?.error), - }) - } catch (error) { - this.writeFailure ||= error - throw error + } + try { + if (this.resultRepo) { + await this.resultRepo.record(vmId, schedule.revision, saved) + await settings.refresh?.() + } else await settings.set(keyFor(vmId), { ...current, ...saved }) + } catch (error) { + this.writeFailure ||= error + // The completed test is not retried merely because its local summary failed to persist. + if (terminal) error.taskTerminal = true + throw error + } + return { terminal } + } + if (!this.sharedTasks) return await execute() + const cron = new Cron(schedule.cron, { timezone: schedule.timezone, mode: '5-part', paused: true }) + let occurrence + try { + occurrence = cron.previousRuns(1, new Date(startedAt + 1))[0] + } finally { + cron.stop() } + if (!occurrence) return + await this.sharedTasks.run( + { taskKey: `vm-test:${vmId}`, executionId: `${schedule.revision}:${occurrence.toISOString()}` }, + execute, + ) } finally { this.running.delete(vmId) } diff --git a/src/lib/core/shared-background-tasks.mjs b/src/lib/core/shared-background-tasks.mjs index dcd735593..afb1215a9 100644 --- a/src/lib/core/shared-background-tasks.mjs +++ b/src/lib/core/shared-background-tasks.mjs @@ -88,7 +88,7 @@ export class SharedBackgroundTasks { try { outcome = await callback() } catch (error) { - if (error?.taskNotStarted === true) await this._complete(claim) + if (error?.taskNotStarted === true || error?.taskTerminal === true) await this._complete(claim) else await claim.markUnknown() throw error } diff --git a/src/lib/db/repos/vm-test-schedule-results-repo.mjs b/src/lib/db/repos/vm-test-schedule-results-repo.mjs new file mode 100644 index 000000000..ca1e9e2b0 --- /dev/null +++ b/src/lib/db/repos/vm-test-schedule-results-repo.mjs @@ -0,0 +1,34 @@ +/** Only scheduled-test execution metadata may be appended to an unchanged plan. */ +export class VmTestScheduleResultsRepo { + constructor(db) { + this.db = db + } + + async listPlans() { + const rows = await this.db + .prepare('SELECT key, value FROM settings WHERE left(key, length($1)) = $1 ORDER BY key') + .all('vm_test_schedule:') + return Object.fromEntries(rows.map((row) => [row.key, JSON.parse(row.value)])) + } + + async isCurrent(vmId, revision) { + return !!(await this.db + .prepare(`SELECT key FROM settings WHERE key = $1 + AND value::jsonb->>'revision' = $2 AND value::jsonb->'enabled' = 'true'::jsonb`) + .get(`vm_test_schedule:${vmId}`, revision)) + } + + async record(vmId, revision, result) { + const patch = { + last_run_at: result.last_run_at, + last_ok: result.last_ok, + last_duration_ms: result.last_duration_ms, + last_error: result.last_error, + } + const stored = await this.db + .prepare(`UPDATE settings SET value = (value::jsonb || $3::jsonb)::text, + updated_at = clock_timestamp()::text WHERE key = $1 AND value::jsonb->>'revision' = $2`) + .run(`vm_test_schedule:${vmId}`, revision, JSON.stringify(patch)) + return stored.changes > 0 + } +} diff --git a/src/lib/oauth/credential-refresh-monitor.mjs b/src/lib/oauth/credential-refresh-monitor.mjs index 652a236dc..968e9765e 100644 --- a/src/lib/oauth/credential-refresh-monitor.mjs +++ b/src/lib/oauth/credential-refresh-monitor.mjs @@ -7,6 +7,9 @@ import { isLocalEgressProxy } from '../vm/egress.mjs' import { proxyBlockedReason } from '../vm/proxy-policy.mjs' import { hasRefreshPresence, needsRefresh } from './oauth-credentials.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { getSharedRoutingState } from '../admin/shared-routing-config.mjs' export const DEFAULT_CREDENTIAL_REFRESH = Object.freeze({ enabled: true, @@ -98,6 +101,8 @@ export function createCredentialRefreshMonitor(opts = {}) { let timer = null let inflight = null let lastRun = null + let generation = 0 + const sharedTasks = backgroundTasksFor(opts.sharedTasks) const nowFn = opts.now || (() => Date.now()) const listDue = () => { @@ -105,22 +110,50 @@ export function createCredentialRefreshMonitor(opts = {}) { return vms.filter((vm) => isCredentialRefreshTarget(vm) && needsScheduledRefresh(vm, nowFn())) } - const runOnce = async () => { + const runOnce = async ({ scheduled = false } = {}) => { if (inflight) return inflight + const current = generation inflight = (async () => { + if (isKubernetesMode()) { + const routing = getSharedRoutingState(sharedTasks?.repo.db) + if (!routing) + throw Object.assign(new Error('shared routing unavailable'), { code: 'routing_config_unavailable' }) + await routing.refresh() + } + if (scheduled && (current !== generation || !config.enabled)) return lastRun const started = nowFn() const due = listDue() const items = [] if (due.length && typeof opts.refreshOne === 'function') { await mapPool(due, config.concurrency, async (vm) => { try { - const result = await opts.refreshOne(vm, config) - items.push({ - vm_id: vm.id, - schedulable: vm.schedulable !== false, - ok: !!result?.ok, - refresh_class: result?.refresh_class || (result?.ok ? 'already_fresh' : 'failed'), - }) + const refresh = async () => { + if (scheduled && (current !== generation || !config.enabled)) { + return { + notStarted: true, + value: { vm_id: vm.id, ok: false, refresh_class: 'skipped', reason: 'stopped' }, + } + } + const result = await opts.refreshOne(vm, config) + return { + terminal: result?.ok === true || result?.taskTerminal === true || result?.taskNotStarted === true, + value: { + vm_id: vm.id, + schedulable: vm.schedulable !== false, + ok: !!result?.ok, + refresh_class: result?.refresh_class || (result?.ok ? 'already_fresh' : 'failed'), + }, + } + } + if (sharedTasks) { + const result = await sharedTasks.run( + { taskKey: `credential-refresh:${vm.id}`, intervalMs: config.interval_sec * 1000 }, + refresh, + ) + items.push( + result.ran ? result.value : { vm_id: vm.id, ok: false, refresh_class: 'skipped', reason: result.code }, + ) + } else items.push((await refresh()).value) } catch (error) { items.push({ vm_id: vm.id, @@ -146,22 +179,25 @@ export function createCredentialRefreshMonitor(opts = {}) { } const stop = () => { + generation++ if (timer) { clearInterval(timer) timer = null } + return inflight ? Promise.allSettled([inflight]) : Promise.resolve() } const start = ({ immediate = false } = {}) => { stop() if (!config.enabled) return { started: false, reason: 'disabled' } + const current = generation timer = setInterval(() => { - runOnce().catch(() => {}) + if (current === generation) runOnce({ scheduled: true }).catch(() => {}) }, config.interval_sec * 1000) if (typeof timer.unref === 'function') timer.unref() if (immediate && config.run_on_start) { queueMicrotask(() => { - runOnce().catch(() => {}) + if (current === generation) runOnce({ scheduled: true }).catch(() => {}) }) } return { started: true, interval_sec: config.interval_sec, run_on_start: config.run_on_start } diff --git a/src/lib/oauth/usage-probe-monitor.mjs b/src/lib/oauth/usage-probe-monitor.mjs index 20b457629..fc66254f7 100644 --- a/src/lib/oauth/usage-probe-monitor.mjs +++ b/src/lib/oauth/usage-probe-monitor.mjs @@ -14,6 +14,9 @@ import { shouldProbeFable, usageProbeBackoffRemainingMs } from './crs-usage-prob import { hasRefreshPresence } from './oauth-credentials.mjs' import { vmHasProxyPath } from './credential-refresh-monitor.mjs' import { hasLiveExtraSample, hasOfficialUsageSample, parseResetMs } from '../pool/quota-window.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { getSharedRoutingState } from '../admin/shared-routing-config.mjs' export const DEFAULT_USAGE_PROBE = Object.freeze({ enabled: false, interval_sec: 60, @@ -195,6 +198,9 @@ export function createUsageProbeMonitor(opts = {}) { let timer = null let inflight = null let lastRun = null + let generation = 0 + const sharedTasks = backgroundTasksFor(opts.sharedTasks) + const operations = new Set() const nowFn = opts.now || (() => Date.now()) // One /usage hop per VM at a time, shared by the timer and 429 events. const probing = new Map() @@ -202,6 +208,13 @@ export function createUsageProbeMonitor(opts = {}) { const queue = [] let running = 0 + const refreshRouting = async () => { + if (!isKubernetesMode()) return + const routing = getSharedRoutingState(sharedTasks?.repo.db) + if (!routing) throw Object.assign(new Error('shared routing unavailable'), { code: 'routing_config_unavailable' }) + await routing.refresh() + } + const listTargets = () => { const vms = typeof opts.listTargets === 'function' ? opts.listTargets() || [] : [] return vms.filter((vm) => isUsageProbeTarget(vm)) @@ -226,29 +239,51 @@ export function createUsageProbeMonitor(opts = {}) { * Failures keep the old sample (never a 0%), back off exponentially, and * surface in the run snapshot. A success clears the backoff. */ - const probeVm = (vm, reason) => { + const probeVm = (vm, reason, prepare) => { const id = vm.id if (probing.has(id)) return probing.get(id) - const parked = backoff.get(id) - if (parked && parked.until > nowFn()) { - return Promise.resolve({ vm_id: id, ok: false, reason: 'probe_backoff', source: null }) - } const worker = takeWorker() if (!worker) return Promise.resolve({ vm_id: id, ok: false, reason: 'probe_queue_full', source: null }) const run = worker .then(async () => { try { - const result = await withTimeout(Promise.resolve(opts.probeOne(vm)), config.timeout_ms) - const ok = result?.ok !== false - if (ok) backoff.delete(id) - else { - const failures = (backoff.get(id)?.failures || 0) + 1 - backoff.set(id, { - failures, - until: nowFn() + Math.min(STALE_USAGE_PROBE_GAP_MS, PROBE_BACKOFF_BASE_MS * 2 ** (failures - 1)), - }) + const probe = async () => { + const prepared = await prepare?.() + if (prepared?.item) return { notStarted: true, value: prepared.item } + const probeReason = prepared?.reason || reason + const parked = backoff.get(id) + if (parked && parked.until > nowFn()) { + return { notStarted: true, value: { vm_id: id, ok: false, reason: 'probe_backoff', source: null } } + } + // The timeout only stops this caller waiting. Shutdown still observes the original work. + const operation = Promise.resolve().then(() => opts.probeOne(vm)) + operations.add(operation) + operation.then( + () => operations.delete(operation), + () => operations.delete(operation), + ) + const result = await withTimeout(operation, config.timeout_ms) + const payload = result?.data || result?.body?.data || result + const ok = result?.ok !== false && payload?.ok !== false + if (ok) backoff.delete(id) + else { + const failures = (backoff.get(id)?.failures || 0) + 1 + backoff.set(id, { + failures, + until: nowFn() + Math.min(STALE_USAGE_PROBE_GAP_MS, PROBE_BACKOFF_BASE_MS * 2 ** (failures - 1)), + }) + } + return { + terminal: (result != null && ok) || result?.taskTerminal === true || result?.taskNotStarted === true, + value: { vm_id: id, ok, reason: probeReason, source: payload?.source || (ok ? 'oauth-usage' : null) }, + } } - return { vm_id: id, ok, reason, source: result?.source || (ok ? 'oauth-usage' : null) } + if (!sharedTasks) return (await probe()).value + const result = await sharedTasks.run( + { taskKey: `usage-probe:${id}`, intervalMs: config.interval_sec * 1000 }, + probe, + ) + return result.ran ? result.value : { vm_id: id, ok: false, reason: result.code, source: null } } catch (error) { const failures = (backoff.get(id)?.failures || 0) + 1 backoff.set(id, { @@ -266,62 +301,76 @@ export function createUsageProbeMonitor(opts = {}) { } /** Event-driven hop (unknown 429, reset recheck). Same single-flight and bounds as the timer. */ - const probeNow = (vmId, reason = 'event') => { + const probeNow = async (vmId, reason = 'event') => { + await refreshRouting() if (typeof opts.probeOne !== 'function') return Promise.resolve({ vm_id: vmId, ok: false, reason: 'no_prober' }) const vm = listTargets().find((item) => item.id === vmId) if (!vm) return Promise.resolve({ vm_id: vmId, ok: false, reason: 'not_target', source: null }) return probeVm(vm, reason) } - const runOnce = async () => { + const runOnce = async ({ scheduled = false } = {}) => { if (inflight) return inflight + const current = generation inflight = (async () => { + await refreshRouting() + if (scheduled && (current !== generation || !config.enabled)) return lastRun const started = nowFn() const targets = listTargets() - const items = [] - const due = [] + let due = 0 const staleMs = config.stale_sec * 1000 - for (const vm of targets) { - const account = typeof opts.accountForVm === 'function' ? opts.accountForVm(vm) : null - if (typeof opts.reconcile === 'function') { - try { - opts.reconcile(vm, account) - } catch {} + const prepare = async (vm) => { + if (scheduled && (current !== generation || !config.enabled)) { + return { item: { vm_id: vm.id, ok: false, reason: 'stopped', source: null } } } + const account = typeof opts.accountForVm === 'function' ? opts.accountForVm(vm) : null + if (typeof opts.reconcile === 'function') await opts.reconcile(vm, account) let active = false - if (typeof opts.isActive === 'function') { - try { - active = !!opts.isActive(vm, account) - } catch {} - } + if (typeof opts.isActive === 'function') active = !!opts.isActive(vm, account) const decision = isUsageProbeDue(account || {}, { now: nowFn(), staleMs, active }) if (!decision.due || typeof opts.probeOne !== 'function') { - items.push({ - vm_id: vm.id, - ok: true, - reason: 'reconcile_extra', - source: null, - }) - continue + return { item: { vm_id: vm.id, ok: true, reason: 'reconcile_extra', source: null } } } const nextProbeAt = resetProbeAt(account, vm, config, nowFn()) if (nextProbeAt != null && nextProbeAt > nowFn()) { - items.push({ - vm_id: vm.id, - ok: true, - reason: 'reset_probe_delay', - source: null, - next_probe_at: new Date(nextProbeAt).toISOString(), - }) - continue + return { + item: { + vm_id: vm.id, + ok: true, + reason: 'reset_probe_delay', + source: null, + next_probe_at: new Date(nextProbeAt).toISOString(), + }, + } + } + due++ + return { reason: decision.reason } + } + const items = [] + if (sharedTasks) { + let cursor = 0 + await Promise.all( + Array.from({ length: Math.min(config.concurrency, targets.length) }, async () => { + while (cursor < targets.length) { + const index = cursor++ + const vm = targets[index] + items[index] = await probeVm(vm, 'scheduled', () => prepare(vm)) + } + }), + ) + } else { + const probes = [] + for (const vm of targets) { + const prepared = await prepare(vm) + if (prepared.item) items.push(prepared.item) + else probes.push({ vm, reason: prepared.reason }) } - due.push({ vm, reason: decision.reason }) + items.push(...(await Promise.all(probes.map(({ vm, reason }) => probeVm(vm, reason))))) } - items.push(...(await Promise.all(due.map(({ vm, reason }) => probeVm(vm, reason))))) lastRun = { at: new Date(nowFn()).toISOString(), duration_ms: nowFn() - started, - due: due.length, + due, items, } return lastRun @@ -332,22 +381,25 @@ export function createUsageProbeMonitor(opts = {}) { } const stop = () => { + generation++ if (timer) { clearInterval(timer) timer = null } + return flush() } const start = ({ immediate = false } = {}) => { stop() if (!config.enabled) return { started: false, reason: 'disabled' } + const current = generation timer = setInterval(() => { - runOnce().catch(() => {}) + if (current === generation) runOnce({ scheduled: true }).catch(() => {}) }, config.interval_sec * 1000) if (typeof timer.unref === 'function') timer.unref() if (immediate && config.run_on_start) { queueMicrotask(() => { - runOnce().catch(() => {}) + if (current === generation) runOnce({ scheduled: true }).catch(() => {}) }) } return { started: true, interval_sec: config.interval_sec, run_on_start: config.run_on_start } @@ -359,6 +411,12 @@ export function createUsageProbeMonitor(opts = {}) { return config } + const flush = async () => { + while (inflight || probing.size || operations.size) { + await Promise.allSettled([...(inflight ? [inflight] : []), ...probing.values(), ...operations]) + } + } + return { getConfig: () => config, setConfig, @@ -366,9 +424,7 @@ export function createUsageProbeMonitor(opts = {}) { runOnce, probeNow, start, - flush: async () => { - while (inflight || probing.size) await Promise.allSettled([...(inflight ? [inflight] : []), ...probing.values()]) - }, + flush, stop, } } From 09166c4f71336b83f2f83c89151b5e7a945c7a79 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:51:57 +0800 Subject: [PATCH 175/473] test(kernel): reproduce cancellation EOF with live execution occupancy --- test/unit/kernel-cancel-payload.test.mjs | 174 ++++++++++++++++++++++ test/unit/shared-codex-execution.test.mjs | 28 ++++ test/unit/shared-hop-lifecycle.test.mjs | 25 ++++ test/unit/worker-execution.test.mjs | 69 ++++++++- 4 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 test/unit/kernel-cancel-payload.test.mjs diff --git a/test/unit/kernel-cancel-payload.test.mjs b/test/unit/kernel-cancel-payload.test.mjs new file mode 100644 index 000000000..c265598eb --- /dev/null +++ b/test/unit/kernel-cancel-payload.test.mjs @@ -0,0 +1,174 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { fileURLToPath } from 'node:url' +import { setTimeout as delay } from 'node:timers/promises' +import { callGoWorker } from '../../src/lib/transport/go-worker-client.mjs' + +const root = fileURLToPath(new URL('../../', import.meta.url)) +const kernelBin = path.join(root, 'bin/kin-kernel') +const cliBin = path.join(root, 'share/wrap-cli/cli-node') +const payloadTest = process.platform === 'linux' && fs.existsSync(kernelBin) && fs.existsSync(cliBin) ? test : test.skip + +payloadTest( + 'existing kernel payload cancellation can return complete HTTP while execution remains unconfirmed', + { timeout: 15_000 }, + async (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cancel-proof-')) + const socketPath = path.join(dir, 'kernel.sock') + const cliHome = path.join(dir, 'cli-home') + fs.mkdirSync(path.join(cliHome, '.claude'), { recursive: true }) + const credentialPath = path.join(cliHome, '.claude/credentials.json') + fs.writeFileSync(credentialPath, JSON.stringify({ anthropicApiKey: { apiKey: 'synthetic-local-only' } }), { + mode: 0o600, + }) + const tokenPath = path.join(dir, 'internal.token') + const token = 'synthetic-internal-cancel' + fs.writeFileSync(tokenPath, token, { mode: 0o600 }) + let entered + const submitted = new Promise((resolve) => { + entered = resolve + }) + const upstream = http.createServer(async (req, res) => { + for await (const _ of req) { + } + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.write( + `event: message_start\ndata: ${JSON.stringify({ + type: 'message_start', + message: { + id: 'msg_synthetic', + type: 'message', + role: 'assistant', + model: 'claude-sonnet-4-6', + content: [], + stop_reason: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + })}\n\n`, + ) + entered() + }) + await new Promise((resolve) => upstream.listen(0, '127.0.0.1', resolve)) + const upstreamUrl = `http://127.0.0.1:${upstream.address().port}` + const configPath = path.join(dir, 'kernel.json') + fs.writeFileSync( + configPath, + JSON.stringify({ + vm_id: 'vm-cancel-proof', + socket_path: socketPath, + credential_path: credentialPath, + internal_token: token, + provider: 'local_cli', + claude_bin: cliBin, + slots_per_worker: 2, + test_endpoints: true, + anthropic_base_url: upstreamUrl, + }), + { mode: 0o600 }, + ) + const env = { + ...process.env, + CLAUDE_CONFIG_DIR: cliHome, + ANTHROPIC_API_KEY: 'synthetic-local-only', + ANTHROPIC_BASE_URL: upstreamUrl, + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', + DISABLE_TELEMETRY: '1', + DISABLE_ERROR_REPORTING: '1', + NO_PROXY: '127.0.0.1,localhost', + KIN_ENVELOPE_PATH: path.join(dir, 'no-envelope.json'), + } + for (const key of [ + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'ALL_PROXY', + 'http_proxy', + 'https_proxy', + 'all_proxy', + 'ANTHROPIC_AUTH_TOKEN', + 'CLAUDE_CODE_OAUTH_TOKEN', + 'KIN_SOCKS5', + 'KIN_HTTPS_PROXY', + 'KIN_CLAUDE_AI_OAUTH_JSON', + 'KIN_CLAUDE_CODE_OAUTH_TOKEN', + ]) + delete env[key] + const child = spawn(kernelBin, ['--gateway-worker', '--config', configPath], { + cwd: dir, + env, + stdio: 'ignore', + detached: true, + }) + const closed = once(child, 'close') + t.after(async () => { + try { + process.kill(-child.pid, 'SIGKILL') + } catch {} + await closed + upstream.closeAllConnections() + await new Promise((resolve) => upstream.close(resolve)) + fs.rmSync(dir, { recursive: true, force: true }) + }) + const request = (requestPath, body) => + new Promise((resolve, reject) => { + const req = http.request( + { + socketPath, + path: requestPath, + method: body ? 'POST' : 'GET', + headers: { 'x-internal-token': token, 'content-type': 'application/json' }, + timeout: 2000, + }, + (res) => { + let data = '' + res.on('data', (chunk) => { + data += chunk + }) + res.on('end', () => resolve({ status: res.statusCode, body: JSON.parse(data) })) + }, + ) + req.on('error', reject) + req.on('timeout', () => req.destroy(new Error('synthetic request timeout'))) + req.end(body ? JSON.stringify(body) : undefined) + }) + for (let i = 0; !fs.existsSync(socketPath) && i < 120 && child.exitCode === null; i++) await delay(50) + const before = await request('/internal/health') + const pending = callGoWorker({ + exec: { vmId: 'vm-cancel-proof', vm: { runtime: { worker_socket: socketPath, worker_token_file: tokenPath } } }, + envelope: { + body: { + model: 'claude-sonnet-4-6', + max_tokens: 32, + messages: [{ role: 'user', content: 'synthetic held request' }], + }, + stream: false, + }, + requestId: 'synthetic-cancel-proof', + timeoutMs: 5000, + }) + await Promise.race([ + submitted, + pending.then(() => { + throw new Error('request ended before synthetic provider entry') + }), + ]) + const cancelled = await request('/internal/v1/cancel', { request_id: 'synthetic-cancel-proof' }) + assert.equal(cancelled.status, 200) + assert.equal(cancelled.body.cancelled, true) + const result = await pending + assert.equal(result.status, 502) + assert.equal(result.body.error.code, 'provider_error') + assert.equal(result.body.error.message, 'provider error: client disconnected') + // This aggregate observation reproduces the counterexample; it is not a per-job termination signal. + const after = await request('/internal/health') + assert.equal(before.body.ready_slots, 2) + assert.equal(after.body.ready_slots, 1) + assert.equal(result.execution.ended, false) + assert.equal(result.execution.reason, 'kernel_execution_unconfirmed') + }, +) diff --git a/test/unit/shared-codex-execution.test.mjs b/test/unit/shared-codex-execution.test.mjs index 3d71b6937..e0e83a8e2 100644 --- a/test/unit/shared-codex-execution.test.mjs +++ b/test/unit/shared-codex-execution.test.mjs @@ -302,6 +302,34 @@ unixTest('a truncated Codex socket retains its admission and session turn withou assert.equal(fx.messages.length, 1) }) +unixTest('a complete Codex kernel internal error cannot release capacity or start another job', async (t) => { + const fx = await fixture(t, (_job, _req, res) => { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + ok: false, + error: { + type: 'worker_error', + code: 'provider_error', + message: 'provider error: client disconnected', + status: 502, + }, + }), + ) + }) + const result = await fx.run({ runtimeIndex: 0, sessionId: 'kernel-cancelled-conversation' }) + assert.equal(result.status, 502) + assert.equal(result.body.error.code, 'codex_execution_uncertain') + assert.equal(fx.messages.length, 1) + assert.equal((await fx.counts()).vmInflight, 1) + assert.equal((await fx.other.prepare('SELECT ended_at FROM execution_jobs').get()).ended_at, null) + const turn = await fx.other.prepare('SELECT * FROM session_turns').get() + assert.ok(turn.uncertain_at) + const blocked = await fx.run({ runtimeIndex: 1, sessionId: 'kernel-cancelled-conversation' }) + assert.equal(blocked.status, 429) + assert.equal(fx.messages.length, 1) +}) + unixTest('Codex EOF permits an implicit retry only with a new persisted job ID', async (t) => { let executions = 0 const fx = await fixture(t, (_job, _req, res) => { diff --git a/test/unit/shared-hop-lifecycle.test.mjs b/test/unit/shared-hop-lifecycle.test.mjs index 00a0a22f9..d3079164d 100644 --- a/test/unit/shared-hop-lifecycle.test.mjs +++ b/test/unit/shared-hop-lifecycle.test.mjs @@ -213,6 +213,31 @@ unixTest('a truncated shared kernel socket cannot retry or release capacity to a await fx.peerIsBlocked() }) +unixTest('a complete kernel client-disconnected error retains its job, VM and device seat', async (t) => { + const fx = await fixture(t, (_job, _req, res) => { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + ok: false, + error: { + type: 'worker_error', + code: 'provider_error', + message: 'provider error: client disconnected', + status: 502, + }, + }), + ) + }) + const result = await fx.run() + assert.equal(result.execution.ended, false) + assert.equal(result.executionUncertain, true) + assert.equal(fx.messages.length, 1) + assert.equal((await fx.counts()).vmInflight, 1) + assert.equal((await fx.pools[1].executionAdmissions.getSeat(scopedSeatKey)).inflight, 1) + assert.equal((await fx.db.prepare('SELECT ended_at FROM execution_jobs').get()).ended_at, null) + await fx.peerIsBlocked() +}) + for (const [phase, trigger, code, expectedExecutions] of [ ['binding', 'INSERT', 'P6481', 0], ['completion', 'UPDATE OF ended_at', 'P6482', 1], diff --git a/test/unit/worker-execution.test.mjs b/test/unit/worker-execution.test.mjs index d15ced943..c282c799f 100644 --- a/test/unit/worker-execution.test.mjs +++ b/test/unit/worker-execution.test.mjs @@ -5,6 +5,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { callGoWorker, streamGoWorker } from '../../src/lib/transport/go-worker-client.mjs' +import { callCodexKernel, streamCodexKernel } from '../../src/lib/transport/codex-kernel-client.mjs' const unixTest = process.platform === 'win32' ? test.skip : test function deferred() { @@ -28,11 +29,77 @@ async function fixture(t, handler) { await new Promise((resolve) => server.close(resolve)) fs.rmSync(root, { recursive: true, force: true }) }) - return { vmId: 'vm-execution', vm: { runtime: { worker_socket: socket } } } + return { vmId: 'vm-execution', vm: { runtime: { worker_socket: socket, codex_kernel_socket: socket } } } } const envelope = { body: { model: 'claude-test', messages: [{ role: 'user', content: 'hi' }] }, headers: {} } +for (const [name, invoke] of [ + ['HTTP', callGoWorker], + ['SSE', streamGoWorker], + ['Codex HTTP', callCodexKernel], + ['Codex SSE', streamCodexKernel], +]) { + for (const message of [ + 'provider error: client disconnected', + "provider error: client disconnected; You've hit your limit (429)", + 'provider error: internal transport timeout', + 'provider error: failed to read CLI response', + ]) { + unixTest(`${name} complete kernel error '${message}' does not prove execution end`, async (t) => { + const exec = await fixture(t, (_req, res) => { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ ok: false, error: { type: 'worker_error', code: 'provider_error', message, status: 502 } }), + ) + }) + const result = await invoke({ exec, envelope }) + assert.equal(result.execution.started, true) + assert.equal(result.execution.ended, false) + assert.equal(result.execution.reason, 'kernel_execution_unconfirmed') + assert.equal(result.transportError, false, 'the HTTP response itself completed') + }) + } + unixTest(`${name} recognized provider rejection still permits EOF completion`, async (t) => { + const exec = await fixture(t, (_req, res) => { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + error: { type: 'worker_error', code: 'provider_error', message: "provider error: You've hit your limit" }, + }), + ) + }) + const result = await invoke({ exec, envelope }) + assert.equal(result.status, 429) + assert.equal(result.execution.ended, true) + assert.equal(result.execution.reason, 'eof') + }) +} + +for (const [name, invoke] of [ + ['SSE', streamGoWorker], + ['Codex SSE', streamCodexKernel], +]) { + unixTest(`${name} kernel error event followed by EOF keeps termination unknown`, async (t) => { + const exec = await fixture(t, (_req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.end( + `event: error\ndata: ${JSON.stringify({ + type: 'error', + error: { + type: 'worker_error', + code: 'provider_error', + message: 'provider error: client disconnected', + }, + })}\n\n`, + ) + }) + const result = await invoke({ exec, envelope }) + assert.equal(result.execution.ended, false) + assert.equal(result.execution.reason, 'kernel_execution_unconfirmed') + }) +} + for (const [name, invoke] of [ ['HTTP', callGoWorker], ['SSE', streamGoWorker], From 786a8b1f79c8dc230b963574bc5fc482a0f6aa12 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:55:05 +0800 Subject: [PATCH 176/473] test(kernel): preserve ordinary explicit provider rejection completion --- test/unit/worker-execution.test.mjs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/unit/worker-execution.test.mjs b/test/unit/worker-execution.test.mjs index c282c799f..63e122aff 100644 --- a/test/unit/worker-execution.test.mjs +++ b/test/unit/worker-execution.test.mjs @@ -74,6 +74,20 @@ for (const [name, invoke] of [ assert.equal(result.execution.ended, true) assert.equal(result.execution.reason, 'eof') }) + unixTest(`${name} ordinary explicit provider rejection retains EOF completion`, async (t) => { + for (const status of [400, 401, 403, 404, 409, 422, 429, 529]) { + const exec = await fixture(t, (_req, res) => { + res.writeHead(status, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + error: { type: 'worker_error', code: 'provider_error', message: 'provider rejected the request', status }, + }), + ) + }) + const result = await invoke({ exec, envelope }) + assert.equal(result.execution.ended, true, `ordinary ${status} rejection is complete`) + } + }) } for (const [name, invoke] of [ From f3a497ee73aeb9152fbfb3316d7a83e105b7f997 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:56:42 +0800 Subject: [PATCH 177/473] test: reproduce API proxy hiding truncated upstream responses --- api-kernel/internal/server/server_test.go | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/api-kernel/internal/server/server_test.go b/api-kernel/internal/server/server_test.go index da7cd165d..7d50ca3af 100644 --- a/api-kernel/internal/server/server_test.go +++ b/api-kernel/internal/server/server_test.go @@ -3,6 +3,7 @@ package server import ( "bufio" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -112,3 +113,35 @@ func TestForwardStreamsTextDeltasBeforeUpstreamCompletes(t *testing.T) { t.Fatalf("missing second text delta: %q", rest) } } + +func TestForwardDoesNotTurnTruncatedUpstreamIntoCleanEOF(t *testing.T) { + for _, contentType := range []string{"application/json", "text/event-stream"} { + t.Run(contentType, func(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, writer, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer conn.Close() + _, _ = fmt.Fprintf(writer, "HTTP/1.1 200 OK\r\nContent-Type: %s\r\nTransfer-Encoding: chunked\r\n\r\nc\r\npartial body\r\n", contentType) + _ = writer.Flush() // Deliberately omit the final chunk. + })) + defer up.Close() + gateway := httptest.NewServer((&Server{Token: "t", Pool: pool.NewManager()}).Handler()) + defer gateway.Close() + env, _ := json.Marshal(forwardEnv{URL: up.URL, Body: `{}`}) + req, _ := http.NewRequest(http.MethodPost, gateway.URL+"/internal/forward", strings.NewReader(string(env))) + req.Header.Set("X-Kin-Internal-Token", "t") + resp, err := gateway.Client().Do(req) + if err != nil { + return // Aborting before headers also preserves the transport failure. + } + defer resp.Body.Close() + _, err = io.ReadAll(resp.Body) + if err == nil { + t.Fatal("truncated upstream was reported as a clean downstream EOF") + } + }) + } +} From e854273a5d7016c160a24237ef9aeac3df0cf988 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:57:05 +0800 Subject: [PATCH 178/473] fix: preserve upstream body failures through API proxy --- api-kernel/internal/server/server.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api-kernel/internal/server/server.go b/api-kernel/internal/server/server.go index 1adb1c179..eb8df0f92 100644 --- a/api-kernel/internal/server/server.go +++ b/api-kernel/internal/server/server.go @@ -99,11 +99,15 @@ func (s *Server) forward(w http.ResponseWriter, r *http.Request) { if mediaType == "text/event-stream" { if flusher, ok := w.(http.Flusher); ok { flusher.Flush() - _, _ = io.Copy(flushingWriter{writer: w, flusher: flusher}, resp.Body) + if _, err := io.Copy(flushingWriter{writer: w, flusher: flusher}, resp.Body); err != nil { + panic(http.ErrAbortHandler) + } return } } - _, _ = io.Copy(w, resp.Body) + if _, err := io.Copy(w, resp.Body); err != nil { + panic(http.ErrAbortHandler) + } } // Expose only Write so io.Copy cannot bypass flushing through ReaderFrom. From b32641b84aebb3f4beac656e09c59beee8c78270 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 20:57:18 +0800 Subject: [PATCH 179/473] fix(kernel): retain execution occupancy after unconfirmed internal errors --- .../2026-10-21-kubernetes-control-plane.md | 2 +- ...2026-10-21-kubernetes-execution-results.md | 2 +- src/lib/transport/go-worker-client.mjs | 44 +++++++++++++++++-- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/plans/2026-10-21-kubernetes-control-plane.md b/docs/plans/2026-10-21-kubernetes-control-plane.md index 471287072..c7cfc1829 100644 --- a/docs/plans/2026-10-21-kubernetes-control-plane.md +++ b/docs/plans/2026-10-21-kubernetes-control-plane.md @@ -68,7 +68,7 @@ `fb90450d` 已加入账号/VM/Fable/RPM 的共享准入、设备座位与 generation、发送前实际 kernel request ID 绑定，以及未知执行保留和停止重试。设备预算按账号跨 VM 计算。standalone 保留原同步调度路径。详细结果见 [共享执行验证](2026-10-21-kubernetes-execution-results.md)。全量 unit 为 3493 pass、7 fail、10 skip；全量 E2E 为 103 pass、17 fail、1 skip。失败差异和基线依据均在报告中列出。 -在 `fb90450d` 检查点，等待队列为进程内队列，同会话顺序和 Sticky 条件更新尚未实现；后续状态见下文共享快照增量。HTTP/SSE EOF 终态仍依赖正常 kernel 协议，尚无执行端 fencing；控制面死亡后的占位没有自动回收。独占锁和单副本限制继续保留。 +在 `fb90450d` 检查点，等待队列为进程内队列，同会话顺序和 Sticky 条件更新尚未实现；后续状态见下文共享快照增量。普通响应的 HTTP/SSE EOF 终态仍依赖正常 kernel 协议；取消或内部 transport/timeout 等未确认 kernel 错误即使有完整响应也保留未知占位，`ready_slots` 汇总值不能证明单个 job 已结束。尚无执行端 fencing；控制面死亡后的占位没有自动回收。独占锁和单副本限制继续保留。 该检查点之后已实施的快照改造起点为：AccountQuota 的计数改为 PG 原子增量，响应头/probe 改为有数据库观测顺序的字段补丁，身份与并发/RPM 配置使用管理补丁。`panel-api` 的显示检查、Fable probe、手工清理以及 `panel-routes` 的身份修改也会调用整行 `save`，需要同时覆盖。保留已有 bigint 精度和 `countRequest:false` 行为。ProxyPool 的 probe 当前仍调用 `replaceAll` 删除重建代理表，需要拆成目标行配置、健康、绑定及删除操作，并校验 probe 对应的配置版本。 diff --git a/docs/plans/2026-10-21-kubernetes-execution-results.md b/docs/plans/2026-10-21-kubernetes-execution-results.md index f7b3f10ab..03abcca4c 100644 --- a/docs/plans/2026-10-21-kubernetes-execution-results.md +++ b/docs/plans/2026-10-21-kubernetes-execution-results.md @@ -8,7 +8,7 @@ - `execution_seats` 与请求准入在同一事务中分配。设备的 VM、slot index 和 generation 从 PG 读取。有效座位数量按 VM 检查容量，按账号检查预算；同一账号跨 VM 共享预算。未知执行继续持有座位；最后一个已确认请求释放后才开始 grace。 - `execution_jobs` 在发送前将实际 kernel request ID 绑定到持久准入。同一准入不能同时绑定两个未结束作业。释放在锁定准入行后重新检查 pending 作业；迟到的旧作业结果不能释放后来绑定的作业。 - Kubernetes 调度器等待异步准入，进程内 inflight 不覆盖 PG 的容量判断。保留 standalone 同步路径。异步排队处理拒绝、错误、取消后的迟到准入及释放；退出时等待调度器任务。等待队列目前仍属于各进程，未实现全局 FIFO。 -- 工作客户端返回实际 kernel request ID 与 `execution.started/ended/reason`。`ended=true` 由完整 HTTP 响应或 SSE EOF 推断，依赖正常 kernel 协议在结束响应前停止对应执行的约定，尚不是独立的执行端终止证明。超时、连接中断和取消请求的成功返回不能作为执行结束证明。此元数据仅供控制面内部使用。 +- 工作客户端返回实际 kernel request ID 与 `execution.started/ended/reason`。普通响应的 `ended=true` 由完整 HTTP 响应或 SSE EOF 推断，依赖正常 kernel 协议在结束响应前停止对应执行的约定，尚不是独立的执行端终止证明。kernel 内部未确认错误是例外：现有 payload 在取消后可返回完整 HTTP 502 `worker_error/provider_error: client disconnected`，不能据此判定执行结束；此类取消、内部 transport/timeout 及无法识别为明确供应商拒绝的 kernel 错误保留 `ended=false`。超时、连接中断和取消请求的成功返回不能作为执行结束证明。`ready_slots` 是汇总状态，不带 request ID 或 epoch，不能作为单个 job 的结束证明。此元数据仅供控制面内部使用。 - 共享准入下，kernel router 等待作业结果（包括结束未确认）写入 PG。未确认结束时禁止隐式重试。FailoverRunner 同时停止跨账号重试，并保留账号/VM 和 managed key 并发。用量结算与执行结束分别判断。 - 作业绑定或完成记录失败时不继续发送新执行。mock 和预先取消而未提交的请求不完成未注册的作业。 diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index 4a2ac40fc..4529b5f12 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -333,6 +333,39 @@ import { semanticStatusForStreamError } from '../protocol/stream-error-status.mj export { semanticStatusForStreamError } from '../protocol/stream-error-status.mjs' +function kernelExecutionUnconfirmed(body, httpStatus = 0) { + const error = body?.error || body?.response?.error + if (!error || (error.type !== 'worker_error' && error.code !== 'provider_error')) return false + const detail = `${error.code || ''} ${error.message || ''}` + // The existing payload can close its HTTP response while the CLI still owns + // the job. Cancellation/transport failures take precedence over status text. + if ( + /client.*disconnect|cancel|abort|transport|timeout|timed out|internal|failed to (?:read|write)|broken pipe|unexpected eof|connection.*(?:reset|closed)/i.test( + detail, + ) + ) + return true + if (/^(?:needs_refresh|invalid_grant|oauth_revoked|authentication_error)$/.test(String(error.code || ''))) + return false + // A kernel's generic 502 is not an upstream status. Recognized provider + // rejections retain their ordinary EOF semantics. + const status = semanticStatusForStreamError({ + ...body, + status: Number(body?.status) === 502 ? undefined : body?.status, + error: { ...error, status: Number(error.status) === 502 ? undefined : error.status }, + }) + const rejected = (value) => (value >= 400 && value < 500 && value !== 408 && value !== 499) || value === 529 + return !rejected(status) && !rejected(httpStatus) +} + +function completeHttpExecution(execution, body, httpStatus) { + const uncertain = kernelExecutionUnconfirmed(body, httpStatus) + Object.assign(execution, { + ended: !uncertain, + reason: uncertain ? 'kernel_execution_unconfirmed' : 'eof', + }) +} + /** * Kernel `map_kernel` folds every provider failure into 502 `provider_error`. * Restore recognized permission denials and plan-limit / overload statuses. @@ -588,8 +621,8 @@ export async function callGoWorker({ timeoutMs, }) const data = await readAll(response) - Object.assign(execution, { ended: true, reason: 'eof' }) const parsed = parseJson(data) + completeHttpExecution(execution, parsed, response.statusCode) const headers = mergeRateLimitHeaders(publicHeaders(response.headers)) const ok = response.statusCode >= 200 && response.statusCode < 300 && parsed?.type !== 'error' if (ok) rememberThreadTurn(parsed?.id, prepared.thread) @@ -783,12 +816,13 @@ export async function streamGoWorker({ const headers = mergeRateLimitHeaders(publicHeaders(response.headers)) if ((response.statusCode || 0) < 200 || (response.statusCode || 0) >= 300) { const data = await readAll(response, 1024 * 1024) - Object.assign(execution, { ended: true, reason: 'eof' }) + const parsed = parseJson(data) + completeHttpExecution(execution, parsed, response.statusCode) return restoreKernelErrorStatus({ ok: false, status: response.statusCode || 0, via: 'go-worker-stream', - body: parseJson(data), + body: parsed, headers, committed: false, terminalState: headers['x-terminal-state'] || 'error', @@ -799,6 +833,7 @@ export async function streamGoWorker({ } let buffer = '' let lastError = null + let unconfirmedKernelError = null let dataBuf = '' let sseUsage = null let sseModel = null @@ -833,6 +868,7 @@ export async function streamGoWorker({ } const observeSseEvent = (event) => { if (!event) return event + if (kernelExecutionUnconfirmed(event)) unconfirmedKernelError = event if (ttftMs == null && isFirstTokenEvent(event)) ttftMs = Date.now() - startedAt if (event.type === 'response_headers' && event.headers && typeof event.headers === 'object') { sseRateHeaders = { ...sseRateHeaders, ...event.headers } @@ -900,7 +936,6 @@ export async function streamGoWorker({ await emitLine(line) } } - Object.assign(execution, { ended: true, reason: 'eof' }) if (buffer) { applyClaudeSSELineToMessage(buffer, assembler) await emitLine(buffer) @@ -909,6 +944,7 @@ export async function streamGoWorker({ const event = observeSseEvent(takeSseEvent()) if (isDownstreamCommitEvent(event)) await flushCommit() } + completeHttpExecution(execution, unconfirmedKernelError || lastError, response.statusCode) const trailers = mergeRateLimitHeaders(publicHeaders(response.trailers)) const meta = streamMetaFromHeaders({ ...headers, ...trailers }) const assembled = assembler.message From a6abf4b780eaf0cf3668e1750cd5f06fcc74c80a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:02:39 +0800 Subject: [PATCH 180/473] test(api): reproduce missing endpoint execution lifecycle hooks --- test/unit/api-execution-lifecycle.test.mjs | 250 +++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 test/unit/api-execution-lifecycle.test.mjs diff --git a/test/unit/api-execution-lifecycle.test.mjs b/test/unit/api-execution-lifecycle.test.mjs new file mode 100644 index 000000000..5905c25cc --- /dev/null +++ b/test/unit/api-execution-lifecycle.test.mjs @@ -0,0 +1,250 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { runApiInference } from '../../src/lib/pool/api-protocol.mjs' +import { forwardApi } from '../../src/lib/transport/api-kernel-client.mjs' +import { createClaudeMessageAssembler, applyClaudeSSELineToMessage } from '../../src/lib/protocol/convert.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' + +function deferred() { + let resolve + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +function success(res, { stop = true, openai = false } = {}) { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + const events = openai + ? [{ type: stop ? 'response.completed' : 'response.output_text.delta', response: { id: 'resp-test', output: [] } }] + : [ + { + type: 'message_start', + message: { + id: 'msg-test', + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'hello message_stop' }], + }, + }, + ...(stop ? [{ type: 'message_stop' }] : []), + ] + res.end(events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join('')) +} + +async function fixture(t, handle = (_job, _req, res) => success(res)) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-api-execution-')) + const socket = path.join(root, 'run', 'api-kernel.sock') + fs.mkdirSync(path.dirname(socket), { recursive: true }) + const jobs = [] + const server = http.createServer((req, res) => { + void (async () => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + const job = JSON.parse(Buffer.concat(chunks).toString()) + jobs.push(job) + await handle(job, req, res) + })().catch((error) => res.destroy(error)) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(socket, resolve) + }) + t.after(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + fs.rmSync(root, { recursive: true, force: true }) + }) + const cfg = { paths: { data: root } } + return { + cfg, + jobs, + async run(options = {}) { + return runApiInference({ + cfg, + req: { headers: {} }, + res: { headersSent: false, write() {} }, + convertedBody: { model: 'claude-test', messages: [{ role: 'user', content: 'hi' }] }, + inbound: { model: 'claude-test' }, + protocol: 'anthropic.messages', + deliveryMode: 'verified', + clientStream: false, + store: { listRaw: async () => [], setKeyCooldown: async () => {} }, + scheduler: { + reload() {}, + pick: () => ({ + ok: true, + endpoint: { id: 'ep-test', kind: 'claude', protocol: 'anthropic', base_url: 'https://synthetic.invalid' }, + key: { id: 'key-test', api_key: 'synthetic' }, + upstream_model: 'claude-test', + }), + }, + converters: { createClaudeMessageAssembler, applyClaudeSSELineToMessage, writeSSEHeaders() {} }, + ...options, + }) + }, + } +} + +test('API inference awaits start before forwarding and end after complete SSE EOF', async (t) => { + const entered = deferred() + const grant = deferred() + const fx = await fixture(t) + const starts = [] + const ends = [] + const pending = fx.run({ + onExecutionStart: async (execution) => { + starts.push(execution) + entered.resolve() + await grant.promise + }, + onExecutionEnd: async (execution) => { + ends.push({ ...execution }) + await Promise.resolve() + }, + }) + try { + await Promise.race([entered.promise, pending.then(() => assert.fail('start hook was not called'))]) + assert.equal(fx.jobs.length, 0) + grant.resolve() + const result = await pending + assert.equal(starts.length, 1) + assert.equal(ends.length, 1) + assert.equal(fx.jobs[0].request_id, starts[0].requestId) + assert.equal(ends[0].requestId, starts[0].requestId) + assert.equal(ends[0].ended, true) + assert.deepEqual(result.execution, ends[0]) + } finally { + grant.resolve() + await pending + } +}) + +for (const clientStream of [false, true]) { + test(`API SSE missing terminal remains unknown (clientStream=${clientStream})`, async (t) => { + const fx = await fixture(t, (_job, _req, res) => success(res, { stop: false })) + const ended = [] + const result = await fx.run({ clientStream, onExecutionEnd: (execution) => ended.push(execution) }) + assert.equal(result.execution.ended, false) + assert.equal(result.executionUncertain, true) + assert.equal(ended.length, 1) + }) +} + +for (const kind of ['ordinary_rejection', 'truncated_rejection', 'upstream_transport']) { + test(`API ${kind} classifies execution after its body rather than response headers`, async (t) => { + const fx = await fixture(t, (_job, _req, res) => { + res.writeHead(kind === 'upstream_transport' ? 502 : 429, { + 'content-type': 'application/json', + ...(kind === 'truncated_rejection' ? { 'content-length': '1000' } : {}), + }) + const body = JSON.stringify({ error: { code: kind, message: 'synthetic' } }) + if (kind === 'truncated_rejection') { + res.write(body.slice(0, 15)) + setImmediate(() => res.destroy()) + } else res.end(body) + }) + const ended = [] + const result = await fx.run({ onExecutionEnd: (execution) => ended.push(execution) }) + assert.equal(ended.length, 1) + assert.equal(result.execution.ended, kind === 'ordinary_rejection') + assert.equal(!!result.executionUncertain, kind !== 'ordinary_rejection') + }) +} + +test('API abort after headers remains unknown and invokes end once', async (t) => { + const entered = deferred() + const fx = await fixture(t, (_job, _req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.write('data: {"type":"message_start"}\n\n') + entered.resolve() + }) + const controller = new AbortController() + const ended = [] + const pending = fx.run({ signal: controller.signal, onExecutionEnd: (execution) => ended.push(execution) }) + await entered.promise + controller.abort() + const result = await pending + assert.equal(result.execution.ended, false) + assert.equal(result.executionUncertain, true) + assert.equal(ended.length, 1) +}) + +for (const phase of ['start', 'end']) { + test(`API ${phase} hook errors are rethrown unchanged`, async (t) => { + const fx = await fixture(t) + const error = new Error(`synthetic ${phase} error`) + await assert.rejects( + fx.run({ + [phase === 'start' ? 'onExecutionStart' : 'onExecutionEnd']: async () => { + throw error + }, + }), + (actual) => actual === error, + ) + assert.equal(fx.jobs.length, phase === 'start' ? 0 : 1) + }) +} + +test('API no-endpoint rejection has not_started metadata and sends no callbacks', async (t) => { + const fx = await fixture(t) + const result = await fx.run({ + scheduler: { reload() {}, pick: () => ({ ok: false, code: 'no_endpoint' }) }, + onExecutionStart: () => assert.fail('no execution'), + onExecutionEnd: () => assert.fail('no execution'), + }) + assert.deepEqual(result.execution, { requestId: null, started: false, ended: true, reason: 'not_started' }) + assert.equal(fx.jobs.length, 0) +}) + +test('forwardApi headers carry an unfinished execution and never signal completion', async (t) => { + const finish = deferred() + const fx = await fixture(t, async (_job, _req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.flushHeaders() + await finish.promise + res.end('data: {"type":"message_stop"}\n\n') + }) + const upstream = await forwardApi({ cfg: fx.cfg, url: 'https://synthetic.invalid', body: {} }) + try { + assert.equal(upstream.execution.started, true) + assert.equal(upstream.execution.ended, false) + } finally { + finish.resolve() + upstream.resume() + } +}) + +test('API transport-error EOF retains a durable job and blocks independent PostgreSQL admission', async (t) => { + const fx = await fixture(t, (_job, _req, res) => { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end('{"error":{"code":"upstream_transport"}}') + }) + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + const a = new SharedExecutionAdmissions(db) + const b = new SharedExecutionAdmissions(other) + const input = { + accountId: 'api-key-test', + vmId: 'api-endpoint-test', + accountLimit: 1, + vmLimit: 1, + fableLimit: 0, + rpmLimit: 0, + } + const lease = await a.tryReserve(input) + const result = await fx.run({ onExecutionStart: lease.bindExecution, onExecutionEnd: lease.completeExecution }) + assert.equal(result.execution.ended, false) + assert.equal((await lease.release()).released, false) + const blocked = await b.tryReserve(input) + assert.equal(blocked.ok, false) + const job = await other.prepare('SELECT * FROM execution_jobs').get() + assert.equal(job.request_id, fx.jobs[0].request_id) + assert.equal(job.ended_at, null) +}) From c44392d3ceba298bc28dc86cd493f993e5d46087 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:07:35 +0800 Subject: [PATCH 181/473] test: require live activities for database and background work --- test/unit/maintenance-scopes.test.mjs | 126 ++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 test/unit/maintenance-scopes.test.mjs diff --git a/test/unit/maintenance-scopes.test.mjs b/test/unit/maintenance-scopes.test.mjs new file mode 100644 index 000000000..b7434bac5 --- /dev/null +++ b/test/unit/maintenance-scopes.test.mjs @@ -0,0 +1,126 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import pg from 'pg' +import * as database from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator, coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' + +async function fixture(t) { + const db = database.getDb() + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + t.after(async () => { + await coordinator.stop().catch(() => {}) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${database.quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + database.registerActivityCoordinator(db, coordinator) + return { db, coordinator } +} + +test('registered SQL guards reject missing and closed scopes while another activity remains live', async (t) => { + const { db, coordinator } = await fixture(t) + assert.equal(database.getActivityCoordinator(db), coordinator) + await assert.rejects(db.query('SELECT 1'), { code: 'maintenance_activity_closed' }) + const old = await coordinator.admit() + let release + const deferred = new Promise((resolve) => { + release = resolve + }) + const escaped = old.run(async () => { + await deferred + await assert.rejects(db.query('SELECT 1'), { code: 'maintenance_activity_closed' }) + }) + await old.run(() => db.query('SELECT 1')) + await old.complete({ ended: true }) + const fresh = await coordinator.admit() + release() + await escaped + await fresh.run(() => db.query('SELECT 1')) + await fresh.complete({ ended: true }) +}) + +test('a transaction cannot reuse an activity that closed while its JavaScript callback was suspended', async (t) => { + const { db, coordinator } = await fixture(t) + const gate = await coordinator.admit() + let release + let began + const paused = new Promise((resolve) => { + release = resolve + }) + const started = new Promise((resolve) => { + began = resolve + }) + const transaction = gate.run(() => + database.withTransaction(db, async () => { + await db.query('SELECT 1') + began() + await paused + await db.exec("INSERT INTO settings(key,value) VALUES ('closed-scope-write','true')") + }), + ) + transaction.catch(() => {}) + await started + await gate.complete({ ended: true }) + release() + await assert.rejects(transaction, { code: 'maintenance_activity_closed' }) + assert.equal((await db.pool.query("SELECT count(*) AS n FROM settings WHERE key='closed-scope-write'")).rows[0].n, 0) +}) + +test('Kubernetes database startup leaves a live bootstrap scope through cache initialization', async (t) => { + const source = database.getDb() + const schema = source.schema + '_bootstrap' + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + let db + try { + db = await database.createDatabase({ connectionString: source.connectionString, schema }) + } finally { + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + } + t.after(async () => { + await db?.close() + const cleanup = new pg.Client({ connectionString: source.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${database.quoteIdentifier(coordinationSchemaFor(schema))} CASCADE; + DROP SCHEMA IF EXISTS ${database.quoteIdentifier(schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + const coordinator = database.getActivityCoordinator(db) + assert.ok(coordinator) + await assert.rejects(db.query('SELECT 1'), { code: 'maintenance_activity_closed' }) + await db.bootstrapActivity.run(() => db.exec("INSERT INTO settings(key,value) VALUES ('bootstrap-cache','true')")) + await coordinator.acknowledgeEpoch(db.bootstrapActivity.epoch) + await db.bootstrapActivity.complete({ ended: true }) + const gate = await coordinator.admit() + assert.ok(await gate.run(() => db.prepare("SELECT key FROM settings WHERE key='bootstrap-cache'").get())) + await gate.complete({ ended: true }) +}) + +test('shared background work enters an independent activity and retains unknown remote work', async (t) => { + const { db, coordinator } = await fixture(t) + const tasks = new SharedBackgroundTasks(db) + const done = await tasks.run({ taskKey: 'scope', executionId: 'one' }, async () => { + coordinator.assertActivity() + await db.query('SELECT 1') + return { terminal: true, value: 1 } + }) + assert.equal(done.unknown, false) + const unknown = await tasks.run({ taskKey: 'unknown', executionId: 'one' }, () => ({ value: 'transport_error' })) + assert.equal(unknown.unknown, true) + const row = ( + await db.pool.query(`SELECT count(*) AS n FROM ${database.quoteIdentifier(coordinator.schema)}.activities + WHERE completed_at IS NULL AND uncertain_at IS NOT NULL`) + ).rows[0] + assert.equal(row.n, 1) +}) From 51405ef12233b6f829f85d249651063794262739 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:08:11 +0800 Subject: [PATCH 182/473] test: require request activity observers before backend submission --- test/unit/execution-state-hooks.test.mjs | 69 ++++++++++++++++++++++ test/unit/shared-sticky-producers.test.mjs | 40 ++++++++++++- 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 test/unit/execution-state-hooks.test.mjs diff --git a/test/unit/execution-state-hooks.test.mjs b/test/unit/execution-state-hooks.test.mjs new file mode 100644 index 000000000..fc2ba0908 --- /dev/null +++ b/test/unit/execution-state-hooks.test.mjs @@ -0,0 +1,69 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedCodexRuntime } from '../../src/lib/pool/shared-codex-runtime.mjs' + +function fixture() { + const events = [] + const lease = { + async bindExecution() { + events.push('bind') + }, + async completeExecution() { + events.push('complete') + }, + async markUncertain() { + events.push('uncertain') + }, + } + return { events, lease, runtime: new SharedCodexRuntime(getDb()) } +} + +test('shared Codex execution awaits request observers around durable job registration', async () => { + const fx = fixture() + const execution = { requestId: 'observer-job', started: true, ended: true, reason: 'http_body_complete' } + await fx.runtime.invoke( + fx.lease, + async (args) => { + await args.onExecutionStart({ requestId: execution.requestId }) + fx.events.push('job') + await args.onExecutionEnd(execution) + return { execution } + }, + { + onExecutionStart: async () => { + fx.events.push('observer-start') + }, + onExecutionEnd: async () => { + fx.events.push('observer-end') + }, + }, + ) + assert.deepEqual(fx.events, ['observer-start', 'bind', 'job', 'complete', 'observer-end']) +}) + +test('a request observer rejecting Codex submission prevents the actual job', async () => { + const fx = fixture() + const error = new Error('activity already closed') + let submitted = 0 + await assert.rejects( + fx.runtime.invoke( + fx.lease, + async (args) => { + await args.onExecutionStart({ requestId: 'observer-rejected' }) + submitted++ + const execution = { requestId: 'observer-rejected', started: true, ended: true } + await args.onExecutionEnd(execution) + return { execution } + }, + { + onExecutionStart: async () => { + throw error + }, + }, + ), + (actual) => actual === error, + ) + assert.equal(submitted, 0) + assert.equal(fx.events.includes('bind'), false) +}) diff --git a/test/unit/shared-sticky-producers.test.mjs b/test/unit/shared-sticky-producers.test.mjs index 841a83e33..e1b1f1bc3 100644 --- a/test/unit/shared-sticky-producers.test.mjs +++ b/test/unit/shared-sticky-producers.test.mjs @@ -96,12 +96,20 @@ async function fixture(t) { writeCodexKernelConfig() {}, ensureCodexKernel: ensure || (async () => ({ ok: true })), streamCodexKernel: async (args) => { + const execution = { + requestId: `synthetic-observer-${jobs.length + 1}`, + started: true, + ended: true, + reason: 'http_body_complete', + } + await args.onExecutionStart?.({ requestId: execution.requestId }) jobs.push(args) if (hop) { const result = await hop(args) if (result) return result } - return { ok: true, status: 200, terminalState: 'verified', body: { id: 'resp-synthetic' } } + await args.onExecutionEnd?.(execution) + return { ok: true, status: 200, terminalState: 'verified', body: { id: 'resp-synthetic' }, execution } }, }, }) @@ -274,3 +282,33 @@ test('panel cooldown clearing awaits shared sticky deletion before returning', a assert.equal((await pending).ok, true) assert.equal(await fx.peer.persistence.get(fx.key), null) }) + +test('Codex protocol reports actual request execution boundaries to its observer', async (t) => { + const fx = await fixture(t) + const seen = [] + await fx.run({ + onExecutionState: async (req, execution) => { + assert.equal(req.headers['x-session-id'], 'shared-codex-session') + seen.push(execution) + }, + }) + assert.equal(seen.length, 2) + assert.equal(seen[0].started, true) + assert.equal(seen[0].ended, false) + assert.equal(seen[1].ended, true) + assert.equal(seen[0].requestId, seen[1].requestId) +}) + +test('Codex protocol does not send when its activity observer rejects execution', async (t) => { + const fx = await fixture(t) + const error = new Error('activity unavailable') + await assert.rejects( + fx.run({ + onExecutionState: async () => { + throw error + }, + }), + (actual) => actual === error, + ) + assert.equal(fx.jobs.length, 0) +}) From 08998f406eb7a2bf7ac3512d4b20895879cc2829 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:09:39 +0800 Subject: [PATCH 183/473] feat(api): expose durable endpoint execution lifecycle hooks --- src/lib/pool/api-protocol.mjs | 247 +++++++++++++++------ src/lib/transport/api-kernel-client.mjs | 45 +++- test/unit/api-execution-lifecycle.test.mjs | 181 +++++++++++++++ 3 files changed, 401 insertions(+), 72 deletions(-) diff --git a/src/lib/pool/api-protocol.mjs b/src/lib/pool/api-protocol.mjs index f4547d7c0..1e542a10f 100644 --- a/src/lib/pool/api-protocol.mjs +++ b/src/lib/pool/api-protocol.mjs @@ -37,7 +37,24 @@ function joinHeaderMap(headers = {}) { return out } -async function readLines(stream, onLine) { +function apiTransportError(payload) { + const error = payload?.error || payload?.response?.error + return /transport|stream_interrupted|timeout|cancel|abort/i.test(String(error?.code || '')) +} + +function observeApiTerminal(line, lifecycle) { + if (!lifecycle) return + if (lifecycle.protocol === 'openai' && String(line).trim() === 'data: [DONE]') lifecycle.terminal = true + const event = consumeClaudeSSEData(line, lifecycle.sse) + if (!event) return + if (apiTransportError(event)) lifecycle.unknown = true + if (lifecycle.protocol === 'openai') { + if (['response.completed', 'response.done', 'response.failed'].includes(event.type)) lifecycle.terminal = true + } else if (event.type === 'message_stop') lifecycle.terminal = true + if (event.type === 'error' && event.error && !apiTransportError(event)) lifecycle.terminal = true +} + +async function readLines(stream, onLine, lifecycle) { let buf = '' for await (const chunk of stream) { buf += chunk.toString('utf8') @@ -45,30 +62,94 @@ async function readLines(stream, onLine) { while ((idx = buf.indexOf('\n')) >= 0) { const line = buf.slice(0, idx) buf = buf.slice(idx + 1) + observeApiTerminal(line, lifecycle) await onLine(line) } } - if (buf) await onLine(buf) + if (buf) { + observeApiTerminal(buf, lifecycle) + await onLine(buf) + } } -export async function runApiInference({ - req, - res, - cfg, - scheduler, - store, - protocol, - inbound, - convertedBody, - clientStream, - deliveryMode, - signal, - timeoutMs, - personaHideTokens, - cacheTtl, - converters, - requestContext = null, -} = {}) { +export async function runApiInference(options = {}) { + const lifecycle = { + execution: { requestId: null, started: false, ended: true, reason: 'not_started' }, + upstream: null, + startedHook: false, + registered: false, + startFailed: false, + terminal: false, + unknown: false, + sse: { dataBuf: '' }, + } + let result + try { + result = await runApiInferenceBody(options, lifecycle) + } catch (error) { + if (lifecycle.startFailed) throw lifecycle.startError + result = { + ok: false, + status: 502, + via: 'api-kernel', + terminalState: 'transport_error', + transportError: true, + body: { + type: 'error', + error: { + type: 'api_error', + code: error.code || 'api_kernel_transport', + message: String(error.message || error).slice(0, 300), + }, + }, + } + } + const execution = lifecycle.upstream?.execution || lifecycle.execution + if (lifecycle.upstream) { + const complete = lifecycle.upstream.complete === true && lifecycle.upstream.readableEnded === true + const ended = complete && lifecycle.terminal && !lifecycle.unknown && !options.signal?.aborted + Object.assign(execution, { + ended, + reason: ended + ? 'eof' + : options.signal?.aborted + ? 'client_aborted' + : lifecycle.unknown + ? 'upstream_transport' + : !complete + ? execution.reason === 'in_progress' + ? 'transport_error' + : execution.reason + : 'missing_terminal', + }) + } + if (lifecycle.startedHook && (!options.onExecutionStart || lifecycle.registered)) + await options.onExecutionEnd?.({ ...execution }) + return { ...result, execution, ...(!execution.ended ? { executionUncertain: true } : {}) } +} + +async function runApiInferenceBody( + { + req, + res, + cfg, + scheduler, + store, + protocol, + inbound, + convertedBody, + clientStream, + deliveryMode, + signal, + timeoutMs, + personaHideTokens, + cacheTtl, + converters, + requestContext = null, + onExecutionStart, + } = {}, + lifecycle, +) { const canonical = officialMessagesBody(convertedBody) const model = canonical.model scheduler.reload(await store.listRaw()) @@ -87,6 +168,7 @@ export async function runApiInference({ const protocolKind = normalizeProtocol(picked.endpoint.protocol || preset.protocol, preset.kind) const extraHeaders = joinHeaderMap(picked.endpoint.headers) const isOpenAI = protocolKind === 'openai' + lifecycle.protocol = protocolKind if (requestContext && (isOpenAI || picked.upstream_model !== canonical.model)) { return { ok: false, @@ -125,8 +207,23 @@ export async function runApiInference({ proxyUrl: picked.key.proxy_url, signal, timeoutMs, + onExecutionStart: async (execution) => { + lifecycle.startedHook = true + lifecycle.execution = { ...execution, started: false, ended: true, reason: 'not_started' } + try { + await onExecutionStart?.(execution) + lifecycle.registered = true + } catch (error) { + lifecycle.startFailed = true + lifecycle.startError = error + throw error + } + }, }) + lifecycle.upstream = upstream } catch (error) { + if (lifecycle.startFailed) throw lifecycle.startError + if (error.execution) lifecycle.execution = error.execution return { ok: false, status: 502, @@ -166,7 +263,9 @@ export async function runApiInference({ } if (status >= 400) { - const payload = await readApiJson(upstream).catch(() => ({})) + const payload = await readApiJson(upstream) + lifecycle.terminal = true + lifecycle.unknown = apiTransportError(payload) const message = payload?.error?.message || payload?.message || `upstream ${status}` const upstreamCode = String(payload?.error?.code || payload?.code || '').trim() const contentCode = isContentPolicyErrorCode(upstreamCode) ? upstreamCode.toLowerCase() : '' @@ -198,6 +297,7 @@ export async function runApiInference({ res, converters, body, + lifecycle, }) } @@ -206,18 +306,22 @@ export async function runApiInference({ let classifierStopped = false const readUpstreamLines = async (onLine) => { try { - await readLines(upstream, async (line) => { - if (requestContext) { - const event = consumeClaudeSSEData(line, classifierSseState) - if (event?.type === 'error') - classifierError = { - ...event, - error: { ...event.error, ...(event.error?.code ? { upstream_code: event.error.code } : {}) }, - } - if (event?.type === 'message_stop') classifierStopped = true - } - await onLine(line) - }) + await readLines( + upstream, + async (line) => { + if (requestContext) { + const event = consumeClaudeSSEData(line, classifierSseState) + if (event?.type === 'error') + classifierError = { + ...event, + error: { ...event.error, ...(event.error?.code ? { upstream_code: event.error.code } : {}) }, + } + if (event?.type === 'message_stop') classifierStopped = true + } + await onLine(line) + }, + lifecycle, + ) } catch (error) { if (!requestContext) throw error classifierError = { @@ -377,6 +481,7 @@ async function runOpenAIResponsesUpstream({ res, converters, body, + lifecycle, }) { const chunks = [] const anthropicSse = protocol === 'anthropic.messages' ? createAnthropicSseState() : null @@ -389,43 +494,47 @@ async function runOpenAIResponsesUpstream({ let streamedUsage = null const started = Date.now() - await readLines(upstream, async (line) => { - if (conversionError) return - const raw = String(line || '') - const seen = usageFromSseLine(raw) - if (seen) streamedUsage = seen - if (raw.startsWith('data:') && ttftMs == null) ttftMs = Date.now() - started - if (!committed && raw.startsWith('data:')) committed = true - if (!clientStream) { - chunks.push(raw) - if (/response\.(completed|done)/.test(raw) || raw.trim() === 'data: [DONE]') sawTerminal = true - return - } - if (!res.headersSent) converters.writeSSEHeaders(res) - if (protocol === 'openai.responses') { - res.write(raw.endsWith('\n') ? raw : `${raw}\n`) - if (/response\.(completed|done)/.test(raw)) completed = true - return - } - try { - const mapped = - protocol === 'anthropic.messages' - ? responsesSseToAnthropicEvents(raw, anthropicSse) - : responsesSseToChatChunk(raw, 'codex', chatSse) - if (mapped) { - res.write(mapped) - if (mapped.includes('[DONE]') || mapped.includes('event: message_stop\n')) completed = true + await readLines( + upstream, + async (line) => { + if (conversionError) return + const raw = String(line || '') + const seen = usageFromSseLine(raw) + if (seen) streamedUsage = seen + if (raw.startsWith('data:') && ttftMs == null) ttftMs = Date.now() - started + if (!committed && raw.startsWith('data:')) committed = true + if (!clientStream) { + chunks.push(raw) + if (/response\.(completed|done)/.test(raw) || raw.trim() === 'data: [DONE]') sawTerminal = true + return } - } catch (error) { - if (error.code !== 'tool_arguments_mismatch') throw error - conversionError = { type: 'upstream_protocol_error', code: error.code, message: error.message } - res.write( - protocol === 'anthropic.messages' - ? `event: error\ndata: ${JSON.stringify({ type: 'error', error: conversionError })}\n\n` - : `data: ${JSON.stringify({ error: conversionError })}\n\ndata: [DONE]\n\n`, - ) - } - }) + if (!res.headersSent) converters.writeSSEHeaders(res) + if (protocol === 'openai.responses') { + res.write(raw.endsWith('\n') ? raw : `${raw}\n`) + if (/response\.(completed|done)/.test(raw)) completed = true + return + } + try { + const mapped = + protocol === 'anthropic.messages' + ? responsesSseToAnthropicEvents(raw, anthropicSse) + : responsesSseToChatChunk(raw, 'codex', chatSse) + if (mapped) { + res.write(mapped) + if (mapped.includes('[DONE]') || mapped.includes('event: message_stop\n')) completed = true + } + } catch (error) { + if (error.code !== 'tool_arguments_mismatch') throw error + conversionError = { type: 'upstream_protocol_error', code: error.code, message: error.message } + res.write( + protocol === 'anthropic.messages' + ? `event: error\ndata: ${JSON.stringify({ type: 'error', error: conversionError })}\n\n` + : `data: ${JSON.stringify({ error: conversionError })}\n\ndata: [DONE]\n\n`, + ) + } + }, + lifecycle, + ) if (!clientStream) { let assembled diff --git a/src/lib/transport/api-kernel-client.mjs b/src/lib/transport/api-kernel-client.mjs index faa17ccb6..6eb360f75 100644 --- a/src/lib/transport/api-kernel-client.mjs +++ b/src/lib/transport/api-kernel-client.mjs @@ -109,7 +109,7 @@ export function apiKernelSnapshot(cfg) { } } -export function forwardApi({ +export async function forwardApi({ cfg, method = 'POST', url, @@ -118,7 +118,14 @@ export function forwardApi({ proxyUrl = '', signal, timeoutMs = 300_000, + requestId = crypto.randomUUID(), + onExecutionStart, } = {}) { + const execution = { requestId, started: false, ended: true, reason: 'not_started' } + if (signal?.aborted) { + throw Object.assign(new Error('API request aborted before forwarding'), { code: 'ABORT_ERR', execution }) + } + await onExecutionStart?.({ requestId }) return new Promise((resolve, reject) => { const { socketPath, tokenPath } = apiKernelPaths(cfg) const token = readToken(tokenPath) @@ -128,6 +135,7 @@ export function forwardApi({ headers, body: typeof body === 'string' ? body : JSON.stringify(body ?? {}), proxy_url: proxyUrl || '', + request_id: requestId, }) const payload = Buffer.from(envelope) const reqHeaders = { @@ -140,6 +148,8 @@ export function forwardApi({ } const started = Date.now() const attempt = (left) => { + let connected = false + Object.assign(execution, { ended: false, reason: 'connecting' }) const req = http.request( { socketPath, @@ -148,8 +158,23 @@ export function forwardApi({ headers: reqHeaders, signal, }, - (res) => resolve(res), + (res) => { + res.execution = execution + res.once('aborted', () => { + if (execution.reason !== 'timeout') + execution.reason = signal?.aborted ? 'client_aborted' : 'transport_error' + }) + resolve(res) + }, ) + req.once('socket', (socket) => { + const onConnect = () => { + connected = true + Object.assign(execution, { started: true, ended: false, reason: 'in_progress' }) + } + if (socket.connecting) socket.once('connect', onConnect) + else if (!socket.destroyed) onConnect() + }) const timer = setTimeout( () => { req.destroy( @@ -162,11 +187,25 @@ export function forwardApi({ req.once('close', () => clearTimeout(timer)) req.once('error', (err) => { clearTimeout(timer) - const retryable = err?.code === 'ENOENT' || err?.code === 'ECONNREFUSED' + const retryable = !connected && (err?.code === 'ENOENT' || err?.code === 'ECONNREFUSED') if (retryable && left > 0 && !signal?.aborted) { setTimeout(() => attempt(left - 1), 80) return } + Object.assign( + execution, + retryable + ? { started: false, ended: true, reason: 'not_started' } + : { + ended: false, + reason: signal?.aborted + ? 'client_aborted' + : err?.code === 'api_kernel_timeout' + ? 'timeout' + : 'transport_error', + }, + ) + err.execution = execution reject(err) }) req.write(payload) diff --git a/test/unit/api-execution-lifecycle.test.mjs b/test/unit/api-execution-lifecycle.test.mjs index 5905c25cc..8d3b7228a 100644 --- a/test/unit/api-execution-lifecycle.test.mjs +++ b/test/unit/api-execution-lifecycle.test.mjs @@ -248,3 +248,184 @@ test('API transport-error EOF retains a durable job and blocks independent Postg assert.equal(job.request_id, fx.jobs[0].request_id) assert.equal(job.ended_at, null) }) + +test('API error headers alone do not invoke end before the complete body', async (t) => { + const entered = deferred() + const finish = deferred() + const fx = await fixture(t, async (_job, _req, res) => { + res.writeHead(429, { 'content-type': 'application/json' }) + res.write('{"error":') + entered.resolve() + await finish.promise + res.end('{"type":"rate_limit_error"}}') + }) + const ended = [] + const pending = fx.run({ onExecutionEnd: (execution) => ended.push(execution) }) + try { + await entered.promise + assert.equal(ended.length, 0) + finish.resolve() + assert.equal((await pending).execution.ended, true) + assert.equal(ended.length, 1) + } finally { + finish.resolve() + await pending + } +}) + +for (const clientStream of [false, true]) { + for (const terminal of [false, true]) { + test(`OpenAI API terminal=${terminal}, clientStream=${clientStream} observes actual SSE completion`, async (t) => { + const fx = await fixture(t, (_job, _req, res) => success(res, { openai: true, stop: terminal })) + const result = await fx.run({ + clientStream, + protocol: 'openai.responses', + scheduler: { + reload() {}, + pick: () => ({ + ok: true, + endpoint: { id: 'ep-openai', kind: 'openai', protocol: 'openai', base_url: 'https://synthetic.invalid' }, + key: { id: 'key-openai', api_key: 'synthetic' }, + upstream_model: 'gpt-synthetic', + }), + }, + }) + assert.equal(result.execution.ended, terminal) + assert.equal(!!result.executionUncertain, !terminal) + }) + } +} + +for (const phase of ['before headers', 'after headers']) { + test(`API timeout ${phase} retains unknown execution`, async (t) => { + let timer + const fx = await fixture(t, (_job, _req, res) => { + if (phase === 'after headers') { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.write('data: {"type":"message_start"}\n\n') + timer = setTimeout(() => res.end('data: {"type":"message_stop"}\n\n'), 200) + } else timer = setTimeout(() => success(res), 200) + }) + t.after(() => clearTimeout(timer)) + const ended = [] + const result = await fx.run({ timeoutMs: 25, onExecutionEnd: (execution) => ended.push(execution) }) + assert.equal(result.execution.ended, false) + assert.equal(result.executionUncertain, true) + assert.equal(ended.length, 1) + }) +} + +test('API persists the UUID before forward and completes it before releasing global capacity', async (t) => { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + let boundAtWire + const fx = await fixture(t, async (job, _req, res) => { + boundAtWire = await other.prepare('SELECT * FROM execution_jobs WHERE request_id=$1').get(job.request_id) + success(res) + }) + const a = new SharedExecutionAdmissions(db) + const b = new SharedExecutionAdmissions(other) + const input = { + accountId: 'api-key-normal', + vmId: 'api-endpoint-normal', + accountLimit: 1, + vmLimit: 1, + fableLimit: 0, + rpmLimit: 0, + } + const lease = await a.tryReserve(input) + const result = await fx.run({ onExecutionStart: lease.bindExecution, onExecutionEnd: lease.completeExecution }) + assert.ok(boundAtWire) + assert.equal(result.execution.ended, true) + assert.ok((await other.prepare('SELECT ended_at FROM execution_jobs').get()).ended_at) + assert.equal((await lease.release()).released, true) + const next = await b.tryReserve(input) + assert.equal(next.ok, true) + await next.release() +}) + +for (const code of ['rate_limit_error', 'upstream_transport']) { + test(`API SSE error ${code} distinguishes provider rejection from transport uncertainty`, async (t) => { + const fx = await fixture(t, (_job, _req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.end( + `data: ${JSON.stringify({ type: 'error', error: { type: 'api_error', code, status: 429, message: 'synthetic' } })}\n\n`, + ) + }) + const result = await fx.run() + assert.equal(result.execution.ended, code === 'rate_limit_error') + assert.equal(!!result.executionUncertain, code === 'upstream_transport') + }) +} + +test('an API message_stop followed by an HTTP truncation remains unknown', async (t) => { + const fx = await fixture(t, (_job, _req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream', 'content-length': '10000' }) + res.write('data: {"type":"message_stop"}\n\n') + setImmediate(() => res.destroy()) + }) + const result = await fx.run() + assert.equal(result.execution.ended, false) + assert.equal(result.executionUncertain, true) +}) + +test('unconnected API kernel retries retain one logical request ID and invoke start once', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-api-connect-retry-')) + const socket = path.join(root, 'run', 'api-kernel.sock') + fs.mkdirSync(path.dirname(socket), { recursive: true }) + const wireIds = [] + const starts = [] + const server = http.createServer(async (req, res) => { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + wireIds.push(JSON.parse(Buffer.concat(chunks).toString()).request_id) + res.end('{}') + }) + const timer = setTimeout(() => server.listen(socket), 130) + t.after(async () => { + clearTimeout(timer) + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + fs.rmSync(root, { recursive: true, force: true }) + }) + for (let i = 0; i < 2; i++) { + const upstream = await forwardApi({ + cfg: { paths: { data: root } }, + url: 'https://synthetic.invalid', + onExecutionStart: (execution) => starts.push(execution.requestId), + }) + for await (const _ of upstream) { + } + } + assert.equal(starts.length, 2) + assert.equal(wireIds.length, 2) + assert.deepEqual(wireIds, starts) + assert.notEqual(starts[0], starts[1]) +}) + +for (const [phase, event, code, calls] of [ + ['start', 'INSERT', 'P6501', 0], + ['end', 'UPDATE OF ended_at', 'P6502', 1], +]) { + test(`API PostgreSQL ${phase} hook failure is not swallowed`, async (t) => { + const fx = await fixture(t) + const db = getDb() + const lease = await new SharedExecutionAdmissions(db).tryReserve({ + accountId: 'api-fault-key', + vmId: 'api-fault-endpoint', + accountLimit: 1, + vmLimit: 1, + fableLimit: 0, + rpmLimit: 0, + }) + await db.exec(`CREATE FUNCTION reject_api_job() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic api hook failure' USING ERRCODE='${code}'; END; $$; + CREATE TRIGGER reject_api_job BEFORE ${event} ON execution_jobs FOR EACH ROW EXECUTE FUNCTION reject_api_job();`) + await assert.rejects(fx.run({ onExecutionStart: lease.bindExecution, onExecutionEnd: lease.completeExecution }), { + code, + }) + assert.equal(fx.jobs.length, calls) + if (phase === 'end') assert.equal((await lease.release()).released, false) + }) +} From 31bf881f11d4aa7a4bab0fe792665744ed82ce3c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:11:05 +0800 Subject: [PATCH 184/473] test: require HTTP activities and shared maintenance readiness --- test/e2e/maintenance-activity.e2e.test.mjs | 87 ++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 test/e2e/maintenance-activity.e2e.test.mjs diff --git a/test/e2e/maintenance-activity.e2e.test.mjs b/test/e2e/maintenance-activity.e2e.test.mjs new file mode 100644 index 000000000..5f1b57bea --- /dev/null +++ b/test/e2e/maintenance-activity.e2e.test.mjs @@ -0,0 +1,87 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import pg from 'pg' +import { startGateway, api } from '../harness.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { quoteIdentifier } from '../../src/lib/db/database.mjs' + +const env = { VM2API_CONTROL_PLANE_MODE: 'kubernetes', KIN_BACKUP_DISABLED: '1' } + +async function controller(t, gw) { + const coordinator = new GlobalMaintenanceCoordinator(gw.db) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + t.after(async () => { + await coordinator.stop().catch(() => {}) + const cleanup = new pg.Client({ connectionString: gw.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + return coordinator +} + +test('Kubernetes master HTTP inference and panel writes complete durable activities after persistence', async (t) => { + const gw = await startGateway({ env }) + const coordinator = await controller(t, gw) + try { + const routing = await api(gw, 'PUT', '/api/panel/routing', { body: { pool: { queue_max: 11 } } }) + assert.equal(routing.status, 200, routing.text) + const response = await api(gw, 'POST', '/v1/messages', { + body: { + model: 'claude-haiku-4-5-20251001', + max_tokens: 8, + stream: true, + messages: [{ role: 'user', content: 'activity persists before completion' }], + }, + }) + assert.equal(response.status, 200, response.text) + let rows + for (let i = 0; i < 100; i++) { + rows = ( + await gw.db.pool.query( + `SELECT kind, completed_at, uncertain_at FROM ${quoteIdentifier(coordinator.schema)}.activities WHERE kind = 'http'`, + ) + ).rows + if (rows.length >= 2 && rows.every((row) => row.completed_at)) break + await new Promise((resolve) => setTimeout(resolve, 10)) + } + assert.ok(rows.length >= 2) + assert.ok(rows.every((row) => row.completed_at && !row.uncertain_at)) + } finally { + await gw.stop() + } +}) + +test('persisted global DRAINING rejects new panel/master requests and readiness while liveness stays available', async (t) => { + const gw = await startGateway({ env }) + const coordinator = await controller(t, gw) + let operation + let pending + try { + pending = coordinator.beginMaintenance({ kind: 'backup', timeoutMs: 2000 }) + pending.catch(() => {}) + for (let i = 0; i < 100; i++) { + if ((await coordinator.getState()).phase === 'draining') break + await new Promise((resolve) => setTimeout(resolve, 10)) + } + assert.equal((await coordinator.getState()).phase, 'draining') + assert.equal((await fetch(gw.baseUrl + '/readyz')).status, 503) + assert.equal((await fetch(gw.baseUrl + '/healthz')).status, 200) + const panel = await api(gw, 'GET', '/api/panel/proxies') + assert.equal(panel.status, 503, panel.text) + const inference = await api(gw, 'POST', '/v1/messages', { + body: { model: 'claude-haiku-4-5-20251001', messages: [] }, + }) + assert.equal(inference.status, 503, inference.text) + } finally { + operation = await pending?.catch(() => null) + await operation?.close() + const state = await coordinator.getState() + if (state.phase === 'draining') await coordinator.cancelDrain(state.operationId) + await gw.stop() + } +}) From a7d1b16d7d2f679222bd46d211042353a6936d63 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:16:19 +0800 Subject: [PATCH 185/473] test: reproduce missing protocol execution activity forwarding --- .../unit/protocol-execution-observer.test.mjs | 154 ++++++++++++++++++ test/unit/shared-sticky-producers.test.mjs | 3 +- 2 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 test/unit/protocol-execution-observer.test.mjs diff --git a/test/unit/protocol-execution-observer.test.mjs b/test/unit/protocol-execution-observer.test.mjs new file mode 100644 index 000000000..303616a2f --- /dev/null +++ b/test/unit/protocol-execution-observer.test.mjs @@ -0,0 +1,154 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Readable } from 'node:stream' +import { EventEmitter } from 'node:events' +import { createHandleProtocol } from '../../src/lib/protocol/handle-protocol.mjs' +import { readBody } from '../../src/lib/http/respond.mjs' + +async function fixture(t, { clientStream = false, incomplete = false, rejectStart = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-protocol-observer-')) + fs.mkdirSync(path.join(root, 'run')) + const events = [] + let submitted = 0 + const server = http.createServer(async (req, res) => { + for await (const _chunk of req) { + /* consume the internal envelope */ + } + submitted++ + events.push('submitted') + res.writeHead(200, { 'content-type': 'text/event-stream' }) + const data = [ + { + type: 'message_start', + message: { id: 'synthetic', type: 'message', role: 'assistant', content: [], usage: {} }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'synthetic' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 1 } }, + ...(!incomplete ? [{ type: 'message_stop' }] : []), + ] + res.end(data.map((value) => `data: ${JSON.stringify(value)}\n\n`).join('')) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(path.join(root, 'run/api-kernel.sock'), resolve) + }) + t.after(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + fs.rmSync(root, { recursive: true, force: true }) + }) + const key = { id: 'synthetic-key', category: 'api', group_id: 1 } + const startError = new Error('request activity was closed') + const states = [] + const handler = createHandleProtocol({ + cfg: { + limits: { max_body_bytes: 1 << 20, upstream_timeout_ms: 1000 }, + rewrite: { enabled: false }, + intercept: { rules: [] }, + distill: { enabled: false }, + paths: { project: root, data: root }, + }, + json(res, status, body) { + res.statusCode = status + res.body = body + }, + writeSSEHeaders(res) { + res.headersSent = true + }, + readBody, + requireAuth(req) { + req.apiKeyRecord = key + req.apiKeyKind = 'managed' + return true + }, + requestLog: { start: () => ({ request_id: 'synthetic-log-id', t0: Date.now() }) }, + stats: { errors: 0, requests: 0, by_route: {} }, + groupsRepo: { rateMultiplier: () => 1 }, + apiKeyStore: { + reserve: async () => ({ ok: true, shared: true, release: async () => events.push('release') }), + recordUsage: async () => events.push('settled'), + }, + stickyRouter: { resolve: () => null }, + accountQuota: { ingestHeaders() {} }, + routingConfig: { compatibility: { claude_code_only: false } }, + apiEndpointStore: { listRaw: async () => [], setKeyCooldown: async () => {} }, + apiScheduler: { + reload() {}, + pick: () => ({ + ok: true, + endpoint: { + id: 'synthetic-endpoint', + kind: 'claude', + protocol: 'anthropic', + base_url: 'https://synthetic.invalid', + }, + key: { id: 'synthetic-upstream-key', api_key: 'synthetic' }, + upstream_model: 'claude-sonnet-4-6', + }), + }, + async onExecutionState(req, execution) { + assert.equal(req, request) + states.push(execution) + events.push(execution.ended ? 'ended' : 'started') + if (rejectStart && states.length === 1) throw startError + }, + }) + const request = Readable.from([ + Buffer.from( + JSON.stringify({ + model: 'claude-sonnet-4-6', + max_tokens: 20, + stream: clientStream, + messages: [{ role: 'user', content: 'synthetic' }], + }), + ), + ]) + request.headers = { 'user-agent': 'claude-cli/2.1.0 (external, cli)', 'anthropic-version': '2023-06-01' } + const response = new EventEmitter() + response.setHeader = () => {} + response.write = () => {} + response.end = () => { + response.writableEnded = true + } + return { + events, + states, + startError, + submitted: () => submitted, + run: () => handler.handleProtocol(request, response, 'anthropic.messages', '/v1/messages'), + } +} + +for (const clientStream of [false, true]) { + test(`the protocol bridge awaits API execution observers (clientStream=${clientStream})`, async (t) => { + const fx = await fixture(t, { clientStream }) + await fx.run() + assert.equal(fx.states.length, 2) + assert.equal(fx.states[0].started, true) + assert.equal(fx.states[0].ended, false) + assert.equal(fx.states[1].ended, true) + assert.equal(fx.states[0].requestId, fx.states[1].requestId) + assert.ok(fx.events.indexOf('started') < fx.events.indexOf('submitted')) + assert.ok(fx.events.indexOf('ended') < fx.events.indexOf('release')) + }) +} + +test('the protocol bridge cannot submit API work after its activity rejects start', async (t) => { + const fx = await fixture(t, { rejectStart: true }) + await assert.rejects(fx.run(), (error) => error === fx.startError) + assert.equal(fx.submitted(), 0) +}) + +test('an incomplete API execution keeps managed key occupancy after the HTTP handler returns', async (t) => { + const fx = await fixture(t, { incomplete: true }) + await fx.run() + assert.equal(fx.states.length, 2) + assert.equal(fx.states[1].ended, false) + assert.equal(fx.events.includes('release'), false) +}) diff --git a/test/unit/shared-sticky-producers.test.mjs b/test/unit/shared-sticky-producers.test.mjs index e1b1f1bc3..5876ac214 100644 --- a/test/unit/shared-sticky-producers.test.mjs +++ b/test/unit/shared-sticky-producers.test.mjs @@ -72,7 +72,7 @@ async function fixture(t) { key, response, jobs, - run({ ensure, hop, routing = {} } = {}) { + run({ ensure, hop, routing = {}, onExecutionState } = {}) { return handleCodexProtocol({ req, res: response, @@ -84,6 +84,7 @@ async function fixture(t) { projectRoot: root, stickyRouter: router, routing, + onExecutionState, json(_res, status, value) { response.status = status response.body = value From bdda2026dca0ed5905c70514bc3fad94e727a53e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:17:27 +0800 Subject: [PATCH 186/473] feat: connect request activities to actual inference execution boundaries --- src/lib/pool/shared-codex-runtime.mjs | 2 + src/lib/protocol/handle-codex.mjs | 10 +++++ src/lib/protocol/handle-protocol.mjs | 55 +++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/lib/pool/shared-codex-runtime.mjs b/src/lib/pool/shared-codex-runtime.mjs index ce8b940e2..07a235bfc 100644 --- a/src/lib/pool/shared-codex-runtime.mjs +++ b/src/lib/pool/shared-codex-runtime.mjs @@ -114,6 +114,7 @@ export class SharedCodexRuntime { const result = await hop({ ...args, onExecutionStart: async (execution) => { + await args.onExecutionStart?.(execution) await lease.bindExecution(execution) bound = true }, @@ -121,6 +122,7 @@ export class SharedCodexRuntime { await lease.completeExecution(execution) completed = execution?.ended === true if (!completed) await lease.markUncertain() + await args.onExecutionEnd?.(execution) }, }) const notStarted = result?.execution?.started === false && result?.execution?.ended === true && !bound diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index ef0b5e695..15b88ee30 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -588,6 +588,7 @@ export async function handleCodexProtocol({ captureOutbound = false, requestStartedAt = Date.now(), accountGate = null, + onExecutionState = null, }) { const codex = normalizeCodexRouting(routing.codex) const allowed = isCodexProtocolAllowed(protocol, { codex }) @@ -862,6 +863,10 @@ export async function handleCodexProtocol({ : hop, args: { exec: execFor(projectRoot, vm), + onExecutionStart: onExecutionState + ? (execution) => onExecutionState(req, { ...execution, started: true, ended: false }) + : undefined, + onExecutionEnd: onExecutionState ? (execution) => onExecutionState(req, execution) : undefined, signal: gone.signal, body: outboundBody, reqHeaders: req.headers, @@ -1352,6 +1357,7 @@ export async function handleCodexSearch({ projectRoot, stickyRouter = null, ops = {}, + onExecutionState = null, }) { logBag.via = 'codex-search' const reject = (status, type, code, message) => { @@ -1408,6 +1414,10 @@ export async function handleCodexSearch({ refresh: ops.refresh, signal: gone.signal, timeoutMs: ops.searchTimeoutMs, + onExecutionStart: onExecutionState + ? (execution) => onExecutionState(req, { ...execution, started: true, ended: false }) + : undefined, + onExecutionEnd: onExecutionState ? (execution) => onExecutionState(req, execution) : undefined, } last = sharedRuntime ? await sharedRuntime.invoke(admitted.lease, post, args) : await post(args) if (sharedRuntime && (last?.executionUncertain || last?.execution?.ended !== true)) { diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index b3aff4448..40c0d4a74 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -446,12 +446,14 @@ export function createHandleProtocol(deps) { routing = {}, noGoFallback = false, accountObservation = null, + onExecutionStart = candidate.bindExecution, + onExecutionEnd = candidate.completeExecution, }) { const assembler = createClaudeMessageAssembler() const workerResult = await dispatchStreamInference({ exec: candidate.exec, - onExecutionStart: candidate.bindExecution, - onExecutionEnd: candidate.completeExecution, + onExecutionStart, + onExecutionEnd, body, reqHeaders, timeoutMs, @@ -494,6 +496,32 @@ export function createHandleProtocol(deps) { } } + async function observeExecution(req, admission, execution) { + await deps.onExecutionState?.(req, execution) + if (!execution?.requestId) return + admission.pendingExecutions ||= new Set() + if (execution.ended === true) admission.pendingExecutions.delete(execution.requestId) + else if (execution.started === true) { + admission.executionStarted = true + admission.pendingExecutions.add(execution.requestId) + } + if (admission.lease?.shared) admission.lease.executionUncertain = admission.pendingExecutions.size > 0 + } + + function executionHooks(req, admission, candidate = {}) { + if (!deps.onExecutionState && !admission.lease?.shared && !candidate.bindExecution) return {} + return { + onExecutionStart: async (execution) => { + await observeExecution(req, admission, { ...execution, started: true, ended: false }) + await candidate.bindExecution?.(execution) + }, + onExecutionEnd: async (execution) => { + await candidate.completeExecution?.(execution) + await observeExecution(req, admission, execution) + }, + } + } + async function reserveManagedKey(req, res) { if (!req.apiKeyRecord) return { ok: true } const gate = await apiKeyStore.reserve(req.apiKeyRecord) @@ -793,8 +821,13 @@ export function createHandleProtocol(deps) { if (!gate.ok) return admission.lease = gate const runCodex = async (options) => { - admission.executionStarted = true - const output = await handleCodexProtocol(options) + const output = await handleCodexProtocol({ + ...options, + onExecutionState: + deps.onExecutionState || gate.shared + ? (_req, execution) => observeExecution(req, admission, execution) + : undefined, + }) await attachPricing(logBag, { managedKey: req.apiKeyRecord, reservation: gate, @@ -1118,8 +1151,8 @@ export function createHandleProtocol(deps) { logBag.via = 'api-kernel' let result try { - admission.executionStarted = true result = await runApiInference({ + ...executionHooks(req, admission), req, res, cfg, @@ -1522,8 +1555,10 @@ export function createHandleProtocol(deps) { } }, callAttempt: async ({ candidate, body, attemptMeta, deliveryMode: attemptDelivery, signal, onCommit }) => { + const hooks = executionHooks(req, admission, candidate) if (!clientStream) { return await streamAndAssembleClaudeMessage({ + ...hooks, candidate, body, reqHeaders: req.headers, @@ -1570,8 +1605,7 @@ export function createHandleProtocol(deps) { try { const workerResult = await dispatchStreamInference({ exec: candidate.exec, - onExecutionStart: candidate.bindExecution, - onExecutionEnd: candidate.completeExecution, + ...hooks, cacheTtl, preserveCacheBreakpoints, cliHop: attemptMeta?.cliHop === true, @@ -1858,6 +1892,7 @@ export function createHandleProtocol(deps) { logBag.inbound_summary = summarizeBody(body) const gate = await reserveManagedKey(req, res) if (!gate.ok) return + const admission = { lease: gate } try { const output = await handleCodexSearch({ req, @@ -1869,6 +1904,10 @@ export function createHandleProtocol(deps) { routing: getRouting() || {}, projectRoot: cfg.paths.project, stickyRouter, + onExecutionState: + deps.onExecutionState || gate.shared + ? (_req, execution) => observeExecution(req, admission, execution) + : undefined, }) await attachPricing(logBag, { managedKey: req.apiKeyRecord, @@ -1881,7 +1920,7 @@ export function createHandleProtocol(deps) { if (gate.shared && !gate.settled) gate.billingFailed = true throw error } finally { - if (!gate.billingFailed) await gate.release?.() + if (!gate.billingFailed && !gate.executionUncertain) await gate.release?.() } } From 94532fa7eac06b1a0f9a363ed0ef2a9fda41e0b8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:22:42 +0800 Subject: [PATCH 187/473] test: reproduce shared VM physical target alias conflicts --- test/unit/vm-physical-targets.test.mjs | 102 +++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 test/unit/vm-physical-targets.test.mjs diff --git a/test/unit/vm-physical-targets.test.mjs b/test/unit/vm-physical-targets.test.mjs new file mode 100644 index 000000000..7ef678ca6 --- /dev/null +++ b/test/unit/vm-physical-targets.test.mjs @@ -0,0 +1,102 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { containerName } from '../../src/lib/vm/vm-runtime.mjs' + +async function fixture(t) { + const db = getDb() + const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => other.close()) + return { + db, + other, + operations: new SharedVmOperations(db, { ownerId: 'pod-a' }), + otherOperations: new SharedVmOperations(other, { ownerId: 'pod-b' }), + executions: new ExecutionAdmissionsRepo(db), + otherExecutions: new ExecutionAdmissionsRepo(other), + } +} + +function execution(vmId, accountId = 'synthetic-account') { + return { vmId, accountId, accountLimit: 4, vmLimit: 4, rpmLimit: 0, fableLimit: 0 } +} + +test('logical VM aliases cannot obtain separate operations on the same physical container', async (t) => { + const f = await fixture(t) + assert.equal(containerName('vm-synthetic'), containerName('synthetic')) + const first = await f.operations.tryAcquire({ vmId: 'vm-synthetic', nodeId: 'node-a' }) + const second = await f.otherOperations.tryAcquire({ vmId: 'synthetic', nodeId: 'node-b' }) + assert.equal(first.ok, true) + assert.equal(second.ok, false) + assert.equal(second.code, 'vm_operation_active') + await first.release() + const after = await f.otherOperations.tryAcquire({ vmId: 'synthetic' }) + assert.equal(after.ok, true) + await after.release() +}) + +test('an alias operation excludes an execution using the other logical VM identity', async (t) => { + const f = await fixture(t) + const operation = await f.operations.tryAcquire({ vmId: 'vm-synthetic' }) + const admission = await f.otherExecutions.tryReserve(execution('synthetic')) + assert.equal(admission.ok, false) + assert.equal(admission.code, 'vm_operation_active') + await operation.release() +}) + +test('an active execution excludes mutation through a physical alias', async (t) => { + const f = await fixture(t) + const admission = await f.executions.tryReserve(execution('vm-synthetic')) + const operation = await f.otherOperations.tryAcquire({ vmId: 'synthetic' }) + assert.equal(operation.ok, false) + assert.equal(operation.code, 'vm_execution_active') + await admission.release() +}) + +test('different aliases cannot each sell their own VM concurrency budget', async (t) => { + const f = await fixture(t) + const first = await f.executions.tryReserve(execution('vm-synthetic', 'account-a')) + const second = await f.otherExecutions.tryReserve(execution('synthetic', 'account-b')) + assert.equal(second.ok, false) + assert.equal(second.code, 'vm_physical_target_conflict') + assert.equal((await f.otherExecutions.getCounts({ vmId: 'synthetic' })).vmInflight, 1) + await first.release() +}) + +test('physical alias mutation and execution races have exactly one winner', async (t) => { + const f = await fixture(t) + for (let i = 0; i < 5; i++) { + const [operation, admission] = await Promise.all([ + f.operations.tryAcquire({ vmId: `vm-race-${i}` }), + f.otherExecutions.tryReserve(execution(`race-${i}`)), + ]) + assert.equal(Number(operation.ok) + Number(admission.ok), 1) + await operation.release?.() + await admission.release?.() + } +}) + +test('a differently named VM cannot reuse its parent operation through an alias', async (t) => { + const f = await fixture(t) + await f.operations.run('vm-synthetic', async () => { + await assert.rejects( + f.otherOperations.run('synthetic', () => assert.fail('alias callback')), + { + code: 'vm_operation_active', + }, + ) + }) +}) + +test('physical mapping strips the prefix exactly once and does not invent extra aliases', async (t) => { + const f = await fixture(t) + const first = await f.operations.tryAcquire({ vmId: 'vm-vm-synthetic' }) + const different = await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' }) + assert.equal(first.ok, true) + assert.equal(different.ok, true) + assert.notEqual(containerName('vm-vm-synthetic'), containerName('vm-synthetic')) + await first.release() + await different.release() +}) From 34990b6a56fb7b1eae29008fba3bdc9a649f3f71 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:24:09 +0800 Subject: [PATCH 188/473] fix: serialize shared VM aliases on physical container targets --- .../db/repos/execution-admissions-repo.mjs | 21 +++++++++++++------ src/lib/vm/shared-vm-operations.mjs | 9 ++++---- src/lib/vm/vm-physical-target.mjs | 10 +++++++++ src/lib/vm/vm-runtime.mjs | 3 ++- test/unit/vm-physical-targets.test.mjs | 5 +++-- 5 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 src/lib/vm/vm-physical-target.mjs diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 56842b5c2..fb446e7e2 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -1,5 +1,6 @@ import crypto from 'node:crypto' import { withTransaction } from '../database.mjs' +import { physicalContainerName, physicalVmAliases } from '../../vm/vm-physical-target.mjs' function invalid(field) { return Object.assign(new Error(`Invalid execution admission ${field}`), { code: 'invalid_execution_admission' }) @@ -45,7 +46,10 @@ export class ExecutionAdmissionsRepo { }) } if (!Array.isArray(vmIds)) throw invalid('vmIds') - const resources = vmIds.map((id) => 'vm:' + identifier(id, 'vmId')) + const resources = vmIds.flatMap((id) => { + identifier(id, 'vmId') + return [...physicalVmAliases(id).map((alias) => 'vm:' + alias), 'vm-target:' + physicalContainerName(id)] + }) if (accountId != null) resources.push('account:' + identifier(accountId, 'accountId')) for (const resource of [...new Set(resources)].sort()) { await this.db.query('SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))', [ @@ -63,12 +67,12 @@ export class ExecutionAdmissionsRepo { const row = await this.db .prepare(`SELECT count(*) FILTER (WHERE account_id = $1 AND released_at IS NULL) AS account_inflight, - count(*) FILTER (WHERE vm_id = $2 AND released_at IS NULL) AS vm_inflight, + count(*) FILTER (WHERE vm_id = ANY($2::text[]) AND released_at IS NULL) AS vm_inflight, count(*) FILTER (WHERE account_id = $1 AND released_at IS NULL AND model_family = 'fable') AS fable_inflight, count(*) FILTER (WHERE account_id = $1 AND created_at > $3::timestamptz - interval '60 seconds') AS rpm_count, min(created_at) FILTER (WHERE account_id = $1 AND created_at > $3::timestamptz - interval '60 seconds') AS rpm_oldest - FROM execution_admissions WHERE account_id = $1 OR vm_id = $2`) - .get(accountId, vmId, now) + FROM execution_admissions WHERE account_id = $1 OR vm_id = ANY($2::text[])`) + .get(accountId, vmId == null ? [] : physicalVmAliases(vmId), now) return { accountInflight: row.account_inflight, vmInflight: row.vm_inflight, @@ -101,9 +105,14 @@ export class ExecutionAdmissionsRepo { await this.assertRoutingRevision(input.routingRevision) await this.lockResources({ accountId, vmIds: [vmId] }) const operation = await this.db - .prepare('SELECT id FROM vm_operations WHERE vm_id = $1 AND released_at IS NULL') - .get(vmId) + .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') + .get(physicalVmAliases(vmId)) if (operation) return denied('vm_operation_active', 0, vmLimit) + const alias = await this.db + .prepare(`SELECT id FROM execution_admissions + WHERE vm_id = ANY($1::text[]) AND vm_id <> $2 AND released_at IS NULL LIMIT 1`) + .get(physicalVmAliases(vmId), vmId) + if (alias) return denied('vm_physical_target_conflict', 0, vmLimit) const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() await this.db .prepare(`DELETE FROM execution_admissions WHERE account_id = $1 diff --git a/src/lib/vm/shared-vm-operations.mjs b/src/lib/vm/shared-vm-operations.mjs index 696b4317b..21164ea3e 100644 --- a/src/lib/vm/shared-vm-operations.mjs +++ b/src/lib/vm/shared-vm-operations.mjs @@ -2,6 +2,7 @@ import crypto from 'node:crypto' import { AsyncLocalStorage } from 'node:async_hooks' import { withTransaction } from '../db/database.mjs' import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' +import { physicalVmAliases } from './vm-physical-target.mjs' const callbacks = new AsyncLocalStorage() @@ -38,12 +39,12 @@ export class SharedVmOperations { // Identical namespace and VM resource to ExecutionAdmissionsRepo.tryReserve. await this.executions.lockVm(vmId) const active = await this.db - .prepare('SELECT id FROM vm_operations WHERE vm_id = $1 AND released_at IS NULL') - .get(vmId) + .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') + .get(physicalVmAliases(vmId)) if (active) return { ok: false, code: 'vm_operation_active' } const execution = await this.db - .prepare('SELECT id FROM execution_admissions WHERE vm_id = $1 AND released_at IS NULL LIMIT 1') - .get(vmId) + .prepare('SELECT id FROM execution_admissions WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') + .get(physicalVmAliases(vmId)) if (execution) return { ok: false, code: 'vm_execution_active' } const operationId = crypto.randomUUID() await this.db diff --git a/src/lib/vm/vm-physical-target.mjs b/src/lib/vm/vm-physical-target.mjs new file mode 100644 index 000000000..a61754a7f --- /dev/null +++ b/src/lib/vm/vm-physical-target.mjs @@ -0,0 +1,10 @@ +/** Container identity is shared across nodes until a Docker daemon identity is verified. */ +export function physicalContainerName(vmId) { + return `kin-${String(vmId || '').replace(/^vm-/, '')}` +} + +/** Exact inverse of physicalContainerName's single prefix removal. */ +export function physicalVmAliases(vmId) { + const suffix = String(vmId || '').replace(/^vm-/, '') + return suffix.startsWith('vm-') ? [`vm-${suffix}`] : [...new Set([suffix, `vm-${suffix}`])] +} diff --git a/src/lib/vm/vm-runtime.mjs b/src/lib/vm/vm-runtime.mjs index 7f430b6d1..7e654a3d8 100644 --- a/src/lib/vm/vm-runtime.mjs +++ b/src/lib/vm/vm-runtime.mjs @@ -9,6 +9,7 @@ import crypto from 'node:crypto' import fs from 'node:fs' import path from 'node:path' import { slotHost } from './slot-host.mjs' +import { physicalContainerName } from './vm-physical-target.mjs' import { maxRequestBytes, readRoutingConfigFile, streamIdleTimeoutMs } from '../core/config.mjs' import { US_TIMEZONES, validTimezone } from '../core/timezone.mjs' import { runtimeKind, isKvmRuntime, RUNTIME_KVM } from './runtime-kind.mjs' @@ -136,7 +137,7 @@ function sh(argv, opts = {}) { } export function containerName(vmId) { - return `kin-${String(vmId || '').replace(/^vm-/, '')}` + return physicalContainerName(vmId) } export function officialCcUidGid(vmId) { diff --git a/test/unit/vm-physical-targets.test.mjs b/test/unit/vm-physical-targets.test.mjs index 7ef678ca6..f0c46ef68 100644 --- a/test/unit/vm-physical-targets.test.mjs +++ b/test/unit/vm-physical-targets.test.mjs @@ -1,13 +1,14 @@ import test from 'node:test' import assert from 'node:assert/strict' -import { getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { containerName } from '../../src/lib/vm/vm-runtime.mjs' async function fixture(t) { const db = getDb() - const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const other = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) t.after(() => other.close()) return { db, From 9b759d41d42f44e82145842b0eb8f052a1a8ff76 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:24:46 +0800 Subject: [PATCH 189/473] feat: enforce coordinated SQL and request activity scopes --- src/lib/core/shared-background-tasks.mjs | 54 ++++++++-- src/lib/db/database.mjs | 46 +++++++-- src/lib/db/global-maintenance.mjs | 48 +++++++-- src/server.mjs | 122 +++++++++++++++++++++-- test/maintenance-fixture.mjs | 32 ++++++ test/unit/maintenance-scopes.test.mjs | 12 +++ 6 files changed, 283 insertions(+), 31 deletions(-) create mode 100644 test/maintenance-fixture.mjs diff --git a/src/lib/core/shared-background-tasks.mjs b/src/lib/core/shared-background-tasks.mjs index afb1215a9..1586bda0f 100644 --- a/src/lib/core/shared-background-tasks.mjs +++ b/src/lib/core/shared-background-tasks.mjs @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto' -import { getDb } from '../db/database.mjs' +import { getDb, getActivityCoordinator } from '../db/database.mjs' import { BackgroundTaskRunsRepo } from '../db/repos/background-task-runs-repo.mjs' import { isKubernetesMode } from './control-plane-mode.mjs' @@ -8,27 +8,60 @@ const services = new WeakMap() /** Unknown work remains occupied until an explicit recovery protocol proves it ended. */ export class SharedBackgroundTasks { constructor(db, { ownerId = randomUUID() } = {}) { + this.db = db this.repo = new BackgroundTaskRunsRepo(db) this.ownerId = ownerId this.pending = new Set() } - now() { - return this.repo.now() + _coordinator() { + const coordinator = getActivityCoordinator(this.db) + if (!coordinator && isKubernetesMode()) { + throw Object.assign(new Error('background activity coordinator unavailable'), { code: 'maintenance_unavailable' }) + } + return coordinator + } + + async now() { + const activity = await this._coordinator()?.admit({ kind: 'background_clock' }) + try { + return await (activity ? activity.run(() => this.repo.now()) : this.repo.now()) + } finally { + await activity?.complete({ ended: true, reason: 'clock_read_completed' }) + } } async tryClaim(plan) { - const identity = await this.repo.tryClaim({ ...plan, ownerId: this.ownerId }) - if (!identity.ok) return identity + const activity = await this._coordinator()?.admit({ + kind: 'background', + requestId: plan.executionId || plan.taskKey, + }) + const run = (fn) => (activity ? activity.run(fn) : fn()) + let identity + try { + identity = await run(() => this.repo.tryClaim({ ...plan, ownerId: this.ownerId })) + } catch (error) { + await activity?.complete({ ended: true, reason: 'background_claim_not_started' }) + throw error + } + if (!identity.ok) { + await activity?.complete({ ended: true, reason: 'background_claim_rejected' }) + return identity + } let unknown = false let marking = null let completing = null return { ...identity, + run, markUnknown: () => { unknown = true if (!marking) { - const pending = this.markUnknown(identity) + const pending = (async () => { + const result = await run(() => this.markUnknown(identity)) + await activity?.markUncertain('background_task_unknown') + return result + })() marking = pending pending.catch(() => { if (marking === pending) marking = null @@ -42,7 +75,12 @@ export class SharedBackgroundTasks { return { completed: false } } if (!completing) { - const pending = this.complete(identity, options) + const pending = (async () => { + const result = await run(() => this.complete(identity, options)) + if (result.completed) await activity?.complete({ ended: true, reason: 'background_task_completed' }) + else await activity?.markUncertain('background_task_completion_unconfirmed') + return result + })() completing = pending pending.then( (result) => { @@ -86,7 +124,7 @@ export class SharedBackgroundTasks { } let outcome try { - outcome = await callback() + outcome = await claim.run(callback) } catch (error) { if (error?.taskNotStarted === true || error?.taskTerminal === true) await this._complete(claim) else await claim.markUnknown() diff --git a/src/lib/db/database.mjs b/src/lib/db/database.mjs index aee9eba94..ed1e0b4c5 100644 --- a/src/lib/db/database.mjs +++ b/src/lib/db/database.mjs @@ -5,14 +5,27 @@ import crypto from 'node:crypto' import { AsyncLocalStorage } from 'node:async_hooks' import { fileURLToPath } from 'node:url' import pg from 'pg' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' const MIGRATIONS_DIR = fileURLToPath(new URL('./postgres-migrations/', import.meta.url)) const transactions = new AsyncLocalStorage() const maintenance = new AsyncLocalStorage() const instances = new Set() +const activityCoordinators = new WeakMap() let singleton = null let opening = null +export function registerActivityCoordinator(db, coordinator) { + if (coordinator.db !== db || (activityCoordinators.has(db) && activityCoordinators.get(db) !== coordinator)) { + throw new Error('database activity coordinator mismatch') + } + activityCoordinators.set(db, coordinator) +} + +export function getActivityCoordinator(db) { + return activityCoordinators.get(db) || null +} + export function quoteIdentifier(value) { return '"' + String(value).replaceAll('"', '""') + '"' } @@ -73,6 +86,7 @@ export class PostgresDatabase { async query(sql, params = []) { if (this.closed) throw new Error('database is closed') + getActivityCoordinator(this)?.assertActivity() const tx = transactions.getStore() if (this.maintaining && maintenance.getStore() !== this && tx?.db !== this) { throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) @@ -136,6 +150,11 @@ export class PostgresDatabase { } } } finally { + try { + await getActivityCoordinator(this)?.stop() + } catch (error) { + failure ||= error + } this.closed = true await this.pool.end() instances.delete(this) @@ -147,6 +166,7 @@ export class PostgresDatabase { } export async function withTransaction(db, fn) { + getActivityCoordinator(db)?.assertActivity() const outer = transactions.getStore() if (outer?.db === db) return fn(db) if (db.closed) throw new Error('database is closed') @@ -240,16 +260,28 @@ export async function createDatabase(options = {}) { } const db = new PostgresDatabase({ ...options, schema: options.schema || process.env.KIN_DATABASE_SCHEMA || 'public' }) try { - const version = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) - if (version < 170000 || version >= 180000) - throw new Error('vm2api requires PostgreSQL 17 and PostgreSQL 17 backup clients') - await applyMigrations(db, options) - if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'kin.db')) && !options.allowLegacySource) { - const imported = await db.prepare("SELECT value FROM settings WHERE key = 'sqlite_import_done'").get() - if (!imported) throw new Error('existing SQLite data requires db:import:sqlite before PostgreSQL startup') + if (isKubernetesMode()) { + const { GlobalMaintenanceCoordinator } = await import('./global-maintenance.mjs') + const coordinator = new GlobalMaintenanceCoordinator(db) + registerActivityCoordinator(db, coordinator) + await coordinator.start() + db.bootstrapActivity = await coordinator.admitBootstrap() + } + const initialize = async () => { + const version = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) + if (version < 170000 || version >= 180000) + throw new Error('vm2api requires PostgreSQL 17 and PostgreSQL 17 backup clients') + await applyMigrations(db, options) + if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'kin.db')) && !options.allowLegacySource) { + const imported = await db.prepare("SELECT value FROM settings WHERE key = 'sqlite_import_done'").get() + if (!imported) throw new Error('existing SQLite data requires db:import:sqlite before PostgreSQL startup') + } } + if (db.bootstrapActivity) await db.bootstrapActivity.run(initialize) + else await initialize() return db } catch (error) { + await db.bootstrapActivity?.complete({ ended: true, reason: 'database_initialization_failed' }).catch(() => {}) await db.close() throw error } diff --git a/src/lib/db/global-maintenance.mjs b/src/lib/db/global-maintenance.mjs index 1f3043138..a4afdcb71 100644 --- a/src/lib/db/global-maintenance.mjs +++ b/src/lib/db/global-maintenance.mjs @@ -66,6 +66,19 @@ export class GlobalMaintenanceCoordinator { return this._healthy } + async isReady() { + if (!this.healthy || this._closing) return false + const row = ( + await this.db.pool.query( + `SELECT c.phase, c.epoch, m.bound_epoch + FROM ${this.qualified}.control c JOIN ${this.qualified}.members m ON m.owner_id = $1 + WHERE c.id = 1 AND m.stopped_at IS NULL`, + [this.ownerId], + ) + ).rows[0] + return this.healthy && row?.phase === 'open' && Number(row.bound_epoch) === Number(row.epoch) + } + _exclusiveLocal(fn) { const pending = this._serial.then(fn) this._serial = pending.catch(() => {}) @@ -238,6 +251,20 @@ export class GlobalMaintenanceCoordinator { return this._admit(metadata, false) } + /** Producers must explicitly report whether their work ended; a missing result retains the claim. */ + async runActivity(metadata, callback) { + const gate = await this.admit(metadata) + let outcome + try { + outcome = await gate.run(() => callback(gate)) + } catch (error) { + await gate.complete({ ended: error.activityEnded === true, reason: error.code || 'activity_callback_failed' }) + throw error + } + await gate.complete({ ended: outcome?.ended === true, reason: outcome?.reason }) + return outcome?.value + } + /** Hold across migrations and initial cache loading, before the first epoch acknowledgement. */ admitBootstrap({ requestId = null } = {}) { return this._admit({ kind: 'bootstrap', requestId }, true) @@ -295,7 +322,7 @@ export class GlobalMaintenanceCoordinator { } _gate(id, epoch, kind) { - const scope = { coordinator: this, id, epoch, kind, active: true } + const scope = { coordinator: this, id, epoch, kind, active: true, remoteStarted: false } const assertActive = () => { this._assertHealthy() if (!scope.active) throw failure('maintenance_activity_closed', 'The activity scope has completed') @@ -307,7 +334,15 @@ export class GlobalMaintenanceCoordinator { assertActive, run: (fn) => { assertActive() - return activities.run(scope, fn) + return activities.run({ scope, parent: activities.getStore() }, fn) + }, + enter: () => { + assertActive() + activities.enterWith({ scope, parent: activities.getStore() }) + }, + noteExecution: (execution) => { + assertActive() + if (execution?.started === true) scope.remoteStarted = true }, complete: (execution) => this._complete(scope, execution), markUncertain: (reason) => this._complete(scope, { ended: false, reason }), @@ -317,11 +352,12 @@ export class GlobalMaintenanceCoordinator { assertActivity() { this._assertHealthy() - const scope = activities.getStore() - if (scope?.coordinator !== this || !scope.active) { - throw failure('maintenance_activity_closed', 'SQL requires a live admitted activity scope') + for (let context = activities.getStore(); context; context = context.parent) { + const { scope } = context + if (!scope.active) throw failure('maintenance_activity_closed', 'An inherited activity scope has completed') + if (scope.coordinator === this) return scope } - return scope + throw failure('maintenance_activity_closed', 'SQL requires a live admitted activity scope') } async _complete(scope, execution) { diff --git a/src/server.mjs b/src/server.mjs index 04ac8ea26..992ae0d0a 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -71,7 +71,7 @@ import { ensureProxyEgress, proxyEgressReady } from './lib/vm/egress.mjs' import { syncIpv6ProxyEgress } from './lib/vm/proxy-policy-runtime.mjs' import { GATEWAY_CAPABILITIES } from './lib/vm/execution-context.mjs' import { isTelemetryPath, telemetryInterceptResponse } from './lib/identity/telemetry-rewrite.mjs' -import { openDatabase, closeDatabase } from './lib/db/database.mjs' +import { openDatabase, closeDatabase, getActivityCoordinator } from './lib/db/database.mjs' import { runLegacyImport } from './lib/db/legacy-import.mjs' import { initVmDbSync, stopVmWatch, flushVmDbSync, pauseVmDbSync, resumeVmDbSync } from './lib/vm/vm-db-sync.mjs' import { flushPendingWrites } from './lib/db/pending-writes.mjs' @@ -151,13 +151,28 @@ const activeRequests = new Set() let _shuttingDown = false const dataDir = cfg.paths.data || path.join(cfg.paths.root, 'data') const db = await openDatabase({ dataDir }) +const activityCoordinator = getActivityCoordinator(db) +db.bootstrapActivity?.enter() let controlPlaneLock = null let startupAborting = false const lifecycle = new ControlPlaneLifecycle({ - probe: () => db.prepare('SELECT 1').get(), - lockHealthy: () => !kubernetesMode || controlPlaneLock?.healthy === true, + probe: async () => { + if (activityCoordinator) { + if (!(await activityCoordinator.isReady())) throw new Error('global maintenance is not ready') + return db.pool.query('SELECT 1') + } + return db.prepare('SELECT 1').get() + }, + lockHealthy: () => !kubernetesMode || (controlPlaneLock?.healthy === true && activityCoordinator?.healthy === true), isMaintaining: () => backupService.isMaintaining, }) +if (activityCoordinator) { + activityCoordinator.onLost = (error) => { + lifecycle.beginDrain() + if (!lifecycle.initialized) void failStartup(error) + else void shutdown('activity_connection_lost', { failed: true }) + } +} async function failStartup(error) { if (startupAborting) return startupAborting = true @@ -191,6 +206,7 @@ if (kubernetesMode) { await controlPlaneLock.start() } catch (error) { await controlPlaneLock.stop() + await db.bootstrapActivity?.complete({ ended: true, reason: 'startup_exclusive_guard_rejected' }) await closeDatabase() throw Object.assign(new Error(`Control plane startup failed: ${error.code || 'lock_unavailable'}`), { code: error.code || 'lock_unavailable', @@ -945,6 +961,7 @@ const importCommit = createImportCommit({ const { commitImportedOauth, requireSlotProxy, officialCcStatsHandler } = importCommit const protocolHandlers = createHandleProtocol({ + onExecutionState: noteRequestExecution, json, writeSSEHeaders, readBody, @@ -990,12 +1007,20 @@ async function dispatchProtocol(name, req, res, ...args) { error: { type: 'overloaded_error', code: 'maintenance_in_progress', message: 'Database maintenance in progress' }, }) } - activeRequests.add(req) - try { - return await protocolHandlers[name](req, res, ...args) - } finally { - activeRequests.delete(req) - } + return withRequestActivity( + req, + res, + async () => { + const alreadyTracked = activeRequests.has(req) + activeRequests.add(req) + try { + return await protocolHandlers[name](req, res, ...args) + } finally { + if (!alreadyTracked) activeRequests.delete(req) + } + }, + 'websocket_request', + ) } const handleProtocol = async (...args) => await dispatchProtocol('handleProtocol', ...args) @@ -1160,7 +1185,60 @@ const handlePanel = createPanelHandler({ }) const sockets = new Set() -const server = http.createServer(async (req, res) => { +const requestActivities = new WeakMap() + +function noteRequestExecution(req, execution) { + const state = requestActivities.get(req) + if (!state) return + state.gate.noteExecution(execution) + const id = execution?.requestId + if (!id && execution?.started === true) { + state.unknown = true + return + } + if (id) { + const previous = state.executions.get(id) + state.executions.set(id, { + started: execution.started === true || previous?.started === true, + ended: execution.ended === true, + }) + } +} + +async function withRequestActivity(req, res, callback, kind = 'http') { + if (!activityCoordinator || requestActivities.has(req)) return callback() + const gate = await activityCoordinator.admit({ kind }) + const state = { gate, executions: new Map(), unknown: false } + requestActivities.set(req, state) + let responseDone + if (res) { + responseDone = new Promise((resolve) => { + if (res.writableFinished || res.writableEnded || res.destroyed || res.closed) return resolve() + res.once('finish', resolve) + res.once('close', resolve) + }) + } + try { + return await gate.run(async () => { + try { + const value = await callback() + await responseDone + await flushRuntimeWrites() + return value + } catch (error) { + state.unknown = true + throw error + } + }) + } finally { + const ended = + !state.unknown && [...state.executions.values()].every((execution) => !execution.started || execution.ended) + await gate.complete({ ended, reason: ended ? 'handler_persisted' : 'handler_execution_unknown' }) + requestActivities.delete(req) + } +} + +async function handleHttp(req, res) { const handlerFinished = lifecycle.trackHttp(req, res) activeRequests.add(req) try { @@ -1351,6 +1429,24 @@ const server = http.createServer(async (req, res) => { activeRequests.delete(req) handlerFinished() } +} + +const server = http.createServer((req, res) => { + const url = parseRequestUrl(req) + const probe = (req.method === 'GET' || req.method === 'HEAD') && ['/healthz', '/readyz'].includes(url?.pathname) + const pending = probe ? handleHttp(req, res) : withRequestActivity(req, res, () => handleHttp(req, res)) + pending.catch((error) => { + console.warn(`[http] activity failed: ${error.code || error.name}`) + if (!res.headersSent) + json(res, error.code?.startsWith('maintenance_') || error.code === 'database_maintenance' ? 503 : 500, { + error: { + type: 'overloaded_error', + code: error.code || 'request_activity_failed', + message: 'Request activity unavailable', + }, + }) + else res.end() + }) }) server.on('connection', (socket) => { @@ -1515,6 +1611,7 @@ async function shutdown(signal, { failed = false } = {}) { await flushRuntimeWrites() stopAllRustKernels() await clusterManager.stop() + await activityCoordinator?.stop() await controlPlaneLock?.stop() await closeDatabase() })(), @@ -1558,6 +1655,11 @@ async function flushRuntimeWrites() { } await bootApiKernel(cfg) +if (activityCoordinator) { + await flushRuntimeWrites() + await activityCoordinator.acknowledgeEpoch(db.bootstrapActivity.epoch) + await db.bootstrapActivity.complete({ ended: true, reason: 'startup_initialized' }) +} server.listen(cfg.port, cfg.host, async () => { try { diff --git a/test/maintenance-fixture.mjs b/test/maintenance-fixture.mjs new file mode 100644 index 000000000..24d6b6529 --- /dev/null +++ b/test/maintenance-fixture.mjs @@ -0,0 +1,32 @@ +/** Explicit full-lifecycle scopes for tests that exercise coordinated SQL, rather than repository-only fixtures. */ +import { getActivityCoordinator, registerActivityCoordinator } from '../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../src/lib/db/global-maintenance.mjs' + +export async function withDatabaseActivities(databases, callback) { + const gates = [] + for (const db of new Set(databases)) { + let coordinator = getActivityCoordinator(db) + if (!coordinator) { + coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + registerActivityCoordinator(db, coordinator) + } + const state = await coordinator.getState() + if (db.bootstrapActivity) { + await coordinator.acknowledgeEpoch(state.epoch) + await db.bootstrapActivity.complete({ ended: true, reason: 'test_database_initialized' }) + } else { + await coordinator.acknowledgeEpoch(state.epoch) + } + gates.push(await coordinator.admit({ kind: 'test' })) + } + let ended = false + try { + const run = (index) => (index === gates.length ? callback() : gates[index].run(() => run(index + 1))) + const value = await run(0) + ended = true + return value + } finally { + await Promise.all(gates.map((gate) => gate.complete({ ended, reason: ended ? 'test_completed' : 'test_failed' }))) + } +} diff --git a/test/unit/maintenance-scopes.test.mjs b/test/unit/maintenance-scopes.test.mjs index b7434bac5..4c149a108 100644 --- a/test/unit/maintenance-scopes.test.mjs +++ b/test/unit/maintenance-scopes.test.mjs @@ -4,6 +4,7 @@ import pg from 'pg' import * as database from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator, coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' async function fixture(t) { const db = database.getDb() @@ -124,3 +125,14 @@ test('shared background work enters an independent activity and retains unknown ).rows[0] assert.equal(row.n, 1) }) + +test('the explicit activity fixture keeps independent PostgreSQL scopes live together', async (t) => { + const { db } = await fixture(t) + const peer = await database.createDatabase({ connectionString: db.connectionString, schema: db.schema }) + await withDatabaseActivities([db, peer], async () => { + assert.equal((await db.query('SELECT 1 AS n')).rows[0].n, 1) + assert.equal((await peer.query('SELECT 2 AS n')).rows[0].n, 2) + }) + await assert.rejects(peer.query('SELECT 1'), { code: 'maintenance_activity_closed' }) + await peer.close() +}) From bb6f6692e3f1b5ac271f4b00e65a3b4233ddffea Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:25:47 +0800 Subject: [PATCH 190/473] test: reproduce unconfirmed VM termination reported as success --- test/unit/vm-stop-proof.test.mjs | 214 +++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 test/unit/vm-stop-proof.test.mjs diff --git a/test/unit/vm-stop-proof.test.mjs b/test/unit/vm-stop-proof.test.mjs new file mode 100644 index 000000000..9b5afc88c --- /dev/null +++ b/test/unit/vm-stop-proof.test.mjs @@ -0,0 +1,214 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import vmContext from 'node:vm' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { parseLaunchctlPrint } from '../../src/lib/cluster/native-slot.mjs' +import { ClusterError } from '../../src/lib/cluster/ssh-link.mjs' + +function section(file, start, end) { + const source = fs.readFileSync(new URL(file, import.meta.url), 'utf8') + return source.slice(source.indexOf(start), source.indexOf(end)).replaceAll('export ', '') +} + +function localRuntime({ inspectError, running = true, restarting = false, noop = false, replace = false } = {}) { + const commands = [] + let present = true + let replacement = false + const sh = (argv) => { + commands.push(argv) + if (argv[1] === 'inspect') { + if (inspectError) return { ok: false, stderr: inspectError } + const target = argv.at(-1) + if (!present && !(replacement && target === 'kin-synthetic')) { + return { ok: false, stderr: `Error: No such object: ${target}` } + } + const id = replacement && target === 'kin-synthetic' ? 'replacement-id' : 'original-id' + return { + ok: true, + stdout: `${running}|${running ? 123 : 0}|bridge|now|image|host|docker|claude|${restarting}${argv[3].includes('{{.Id}}') ? `|${id}` : ''}`, + } + } + if (!noop) { + running = false + restarting = false + if (argv[1] === 'rm') present = false + if (replace) { + present = false + replacement = true + } + } + return { ok: true, stdout: '' } + } + const source = + section('../../src/lib/vm/vm-runtime.mjs', 'export function inspectContainer(', '/** Label on GPT') + + section('../../src/lib/vm/vm-runtime.mjs', 'export function destroyVmRuntime(', 'export function listRuntimeVms(') + const api = vmContext.runInNewContext(`${source}; ({inspectContainer, stopVmRuntime, destroyVmRuntime})`, { + sh, + PUBLIC_IP: null, + containerName: () => 'kin-synthetic', + isKvmRuntime: () => false, + runtimePatch: (vm, info) => { + vm.runtime = { ...vm.runtime, pid: info?.pid, stopped: !info?.running } + }, + }) + return { ...api, commands, vm: { id: 'vm-synthetic', runtime: {} } } +} + +test('Docker permission or daemon failure is not proof that a container is absent', () => { + for (const inspectError of ['permission denied connecting to Docker', 'Cannot connect to the Docker daemon']) { + const f = localRuntime({ inspectError }) + assert.throws(() => f.inspectContainer('kin-synthetic'), { code: 'container_inspect_failed' }) + const result = f.stopVmRuntime(f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + } +}) + +test('successful stop command requires observed immutable container termination', () => { + const f = localRuntime({ noop: true }) + const result = f.stopVmRuntime(f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) +}) + +test('successful remove command requires observed immutable container absence', () => { + const f = localRuntime({ noop: true }) + const result = f.destroyVmRuntime(f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) +}) + +test('stop and remove use the inspected immutable ID rather than a replaceable name', () => { + for (const action of ['stopVmRuntime', 'destroyVmRuntime']) { + const f = localRuntime() + assert.equal(f[action](f.vm).ok, true) + assert.equal(f.commands.find((argv) => ['stop', 'rm'].includes(argv[1])).at(-1), 'original-id') + } +}) + +test('Docker restart backoff is not treated as an already stopped container', () => { + const f = localRuntime({ restarting: true }) + assert.equal(f.stopVmRuntime(f.vm).action, 'stopped') + assert.equal( + f.commands.some((argv) => argv[1] === 'stop'), + true, + ) +}) + +test('name replacement during stop is unknown and does not mark the VM stopped', () => { + const f = localRuntime({ replace: true }) + const result = f.stopVmRuntime(f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.notEqual(f.vm.runtime.stopped, true) +}) + +test('known absent and already stopped containers remain successful', () => { + assert.equal( + localRuntime({ inspectError: 'Error: No such object: kin-synthetic' }).inspectContainer('kin-synthetic'), + null, + ) + const f = localRuntime({ running: false }) + assert.equal(f.stopVmRuntime(f.vm).action, 'already-stopped') +}) + +test('unconfirmed Docker termination retains the persistent VM operation', async () => { + const operations = new SharedVmOperations(getDb()) + const f = localRuntime({ noop: true }) + const result = await operations.run(f.vm.id, () => f.stopVmRuntime(f.vm)) + assert.equal(result.ok, false) + assert.equal((await operations.tryAcquire({ vmId: f.vm.id })).ok, false) +}) + +test('Codex stop awaits child exit before stopping its slot host', async () => { + const events = [] + let finish + const pending = new Promise((resolve) => { + finish = resolve + }) + const source = section('../../src/lib/vm/slot-runtime.mjs', 'export async function stopSlot(', '/** Destroy the slot') + const stop = vmContext.runInNewContext(`${source}; stopSlot`, { + isCodexVm: () => true, + stopCodexKernel: () => ({ pid: 123 }), + finishCodexRecycle: () => pending, + slotHost: () => ({ + stop: () => { + events.push('host-stop') + return { ok: true } + }, + }), + process: { env: {} }, + }) + const task = stop({ id: 'vm-synthetic' }) + await Promise.resolve() + assert.deepEqual(events, []) + finish({ exited: true }) + assert.equal((await task).ok, true) + assert.deepEqual(events, ['host-stop']) +}) + +function nativeApi(response) { + const source = section( + '../../src/lib/cluster/native-slot.mjs', + 'export async function nativeServiceStatus(', + 'async function bootstrap(', + ) + return vmContext.runInNewContext(`${source}; ({nativeServiceStatus, bootout})`, { + parseLaunchctlPrint, + execCollect: async () => response, + runRemote: async () => '', + shellQuote: (s) => `'${s}'`, + ClusterError, + LAUNCHCTL_TIMEOUT_MS: 100, + BOOTOUT_WAIT_TRIES: 1, + setTimeout: (done) => done(), + }) +} + +test('launchctl permission errors cannot prove native service termination', async () => { + const api = nativeApi({ code: 1, stdout: '', stderr: 'Operation not permitted' }) + await assert.rejects(api.nativeServiceStatus({}, 'gui/501', 'label')) + await assert.rejects(api.bootout({}, 'gui/501', 'label')) +}) + +test('launchctl explicit service-not-found remains a valid absence proof', async () => { + const api = nativeApi({ code: 113, stdout: '', stderr: 'Could not find service' }) + assert.equal((await api.nativeServiceStatus({}, 'gui/501', 'label')).state, 'missing') + await api.bootout({}, 'gui/501', 'label') +}) + +test('remote worker stop without a foreground exit acknowledgement stays unknown', async () => { + const source = section( + '../../src/lib/cluster/remote-slot.mjs', + 'export async function restartRemoteServices(', + 'export async function stopRemoteSlot(', + ) + const restart = vmContext.runInNewContext(`${source}; restartRemoteServices`, { + vmNodeId: () => 'node-synthetic', + nodeSession: async () => ({ client: {}, host: {}, docker: {} }), + remoteSlotDir: () => '/synthetic', + path, + containerName: () => 'kin-synthetic', + migrateRemoteServiceContainers: async () => ({ ok: true, migrated: [] }), + pushSlotFiles: async () => {}, + runtimeKind: () => 'docker', + slotExecCmd: () => ({ Cmd: ['true'] }), + slotExecArgv: () => ['docker', 'exec', 'original-id', 'true'], + inspectContainerOrNull: async () => ({ Id: 'original-id', State: { Running: true } }), + execDetached: async () => {}, + runRemote: async () => { + throw new Error('SSH channel closed before exit') + }, + shellQuote: (s) => `'${s}'`, + setTimeout: (done) => done(), + slotImageSpec: () => ({ ref: 'synthetic' }), + ensureRemoteEgress: async () => ({ ok: true }), + failure: (error) => ({ ok: false, error: error.message }), + }) + const result = await restart({ id: 'vm-synthetic' }, '/synthetic', { telemetryEnabled: false }) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) +}) From dcdd0ee934e4b0d5fd592cd383c6bddcb3be7b03 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:28:10 +0800 Subject: [PATCH 191/473] test: distinguish terminal exit proof from socket closure --- test/fixtures/terminal-execution.mjs | 57 +++++++++++++++++ test/unit/terminal-execution.test.mjs | 91 +++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 test/fixtures/terminal-execution.mjs create mode 100644 test/unit/terminal-execution.test.mjs diff --git a/test/fixtures/terminal-execution.mjs b/test/fixtures/terminal-execution.mjs new file mode 100644 index 000000000..c92282d42 --- /dev/null +++ b/test/fixtures/terminal-execution.mjs @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import { EventEmitter, once } from 'node:events' +import { mock } from 'node:test' +import { WebSocket } from 'ws' + +const moduleUrl = new URL('../../src/lib/cluster/docker-remote.mjs', import.meta.url).href +const actual = await import(moduleUrl) +const tty = new EventEmitter() +tty.write = () => {} +tty.destroy = () => { tty.destroyRequested = true } +const running = process.env.VM2API_TERMINAL_INSPECT_RUNNING === 'true' +mock.module(moduleUrl, { namedExports: { + ...actual, + openExecTty: async () => ({ execId: 'synthetic-exec', stream: tty }), + resizeExec: async () => {}, execDetached: async () => {}, + inspectExec: async () => ({ Running: running, ExitCode: running ? null : 0 }), +} }) +const { createSlotShell } = await import('../../src/lib/vm/slot-shell.mjs') +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'terminal-execution-')) +fs.mkdirSync(path.join(root, 'vms'), { recursive: true }) +fs.writeFileSync(path.join(root, 'vms', 'vm-02.json'), JSON.stringify({ id: 'vm-02' })) +const events = [] +const shell = createSlotShell({ projectRoot: root, onExecutionState: async (_req, execution) => { events.push(execution) } }) +const ticket = await shell.issueTicket('vm-02') +const server = http.createServer() +server.on('upgrade', (req, socket, head) => shell.handleUpgrade(req, socket, head)) +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) +const ws = new WebSocket(`ws://127.0.0.1:${server.address().port}/api/panel/vms/vm-02/shell?ticket=${ticket.ticket}`) +try { + await once(ws, 'open') + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(events[0]?.started, true) + assert.equal(events[0]?.ended, false) + assert.ok(events[0].requestId) + const closed = once(ws, 'close') + if (running) { + ws.close() + await closed + assert.equal(events.some((execution) => execution.ended === true), false) + } + tty.emit('close') + await shell.drain() + await closed + assert.equal(events.at(-1).ended, !running) + assert.equal(events.at(-1).requestId, events[0].requestId) +} finally { + tty.emit('close') + ws.terminate() + await shell.drain() + await new Promise((resolve) => server.close(resolve)) + mock.restoreAll() + fs.rmSync(root, { recursive: true, force: true }) +} diff --git a/test/unit/terminal-execution.test.mjs b/test/unit/terminal-execution.test.mjs new file mode 100644 index 000000000..2bfcb1369 --- /dev/null +++ b/test/unit/terminal-execution.test.mjs @@ -0,0 +1,91 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import { EventEmitter, once } from 'node:events' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { WebSocket } from 'ws' +import { createClusterRoutes } from '../../src/lib/cluster/cluster-routes.mjs' + +async function cluster(t, authorized = true) { + const events = [] + const stream = new EventEmitter() + stream.close = () => {} + let opens = 0 + const routes = createClusterRoutes({ + manager: { + consumeShellTicket: () => authorized, + openShell: async () => { + opens++ + return stream + }, + }, + onExecutionState: async (_req, execution) => { + events.push(execution) + }, + }) + const server = http.createServer() + server.on('upgrade', (req, socket, head) => routes.handleUpgrade(req, socket, head)) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const ws = new WebSocket(`ws://127.0.0.1:${server.address().port}/api/panel/cluster/nodes/node-a/shell?ticket=t`) + ws.on('error', () => {}) + t.after(async () => { + ws.terminate() + await routes.drain() + await new Promise((resolve) => server.close(resolve)) + }) + return { events, stream, ws, opens: () => opens } +} + +test('cluster shell reports a stable execution ID and only SSH exit confirms execution ended', async (t) => { + const f = await cluster(t) + await once(f.ws, 'open') + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(f.events[0]?.started, true) + assert.equal(f.events[0]?.ended, false) + assert.ok(f.events[0].requestId) + f.stream.emit('exit', 0) + f.stream.emit('close') + await once(f.ws, 'close') + assert.equal(f.events.at(-1).ended, true) + assert.equal(f.events.at(-1).requestId, f.events[0].requestId) +}) + +test('cluster channel or WebSocket close without exit never confirms execution ended', async (t) => { + const f = await cluster(t) + await once(f.ws, 'open') + await new Promise((resolve) => setImmediate(resolve)) + f.stream.emit('close') + await once(f.ws, 'close') + assert.equal(f.events[0]?.started, true) + assert.equal(f.events.at(-1)?.ended, false) +}) + +test('cluster shell rejects an invalid ticket with an explicit not-started outcome', async (t) => { + const f = await cluster(t, false) + await new Promise((resolve) => f.ws.once('error', resolve)) + assert.equal(f.opens(), 0) + assert.equal(f.events.length, 1) + assert.equal(f.events[0].started, false) + assert.equal(f.events[0].ended, true) +}) + +for (const running of ['true', 'false']) { + test(`slot shell accepts Docker termination proof only when Running is false (${running})`, () => { + const env = { ...process.env, VM2API_TERMINAL_INSPECT_RUNNING: running } + delete env.NODE_TEST_CONTEXT + const child = spawnSync( + process.execPath, + [ + '--experimental-test-module-mocks', + fileURLToPath(new URL('../fixtures/terminal-execution.mjs', import.meta.url)), + ], + { + env, + encoding: 'utf8', + timeout: 5000, + }, + ) + assert.equal(child.status, 0, child.stdout + child.stderr || child.error?.message) + }) +} From 41ecb9d51ec7dbe0b6d5830692d9b24525be2f59 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:31:17 +0800 Subject: [PATCH 192/473] test: separate repository fixtures from control-plane activity lifetimes --- test/fixtures/shared-auth-worker.mjs | 54 ++++++++++++++++------- test/repository-database.mjs | 17 +++++++ test/unit/pool-scheduler-shared.test.mjs | 3 +- test/unit/shared-auth-state.test.mjs | 3 +- test/unit/shared-codex-execution.test.mjs | 3 +- test/unit/shared-codex-search.test.mjs | 3 +- test/unit/shared-hop-lifecycle.test.mjs | 3 +- test/unit/shared-session-turns.test.mjs | 5 ++- test/unit/shared-vm-operations.test.mjs | 7 +-- test/unit/vm-operation-producers.test.mjs | 3 +- 10 files changed, 73 insertions(+), 28 deletions(-) create mode 100644 test/repository-database.mjs diff --git a/test/fixtures/shared-auth-worker.mjs b/test/fixtures/shared-auth-worker.mjs index 23e0e1956..94a66e450 100644 --- a/test/fixtures/shared-auth-worker.mjs +++ b/test/fixtures/shared-auth-worker.mjs @@ -1,28 +1,48 @@ -import { openDatabase, closeDatabase } from '../../src/lib/db/database.mjs' +import { openDatabase, closeDatabase, getActivityCoordinator } from '../../src/lib/db/database.mjs' import { generateCodexAuthUrl, exchangeCodexAuthCode } from '../../src/lib/oauth/codex-oauth.mjs' import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' process.once('message', async ({ operation, args }) => { try { const db = await openDatabase() - let result - if (operation === 'issueCodex') result = await generateCodexAuthUrl(args) - else if (operation === 'exchangeCodex' || operation === 'claimCodexForever') { - await exchangeCodexAuthCode({ ...args, fetchImpl: async () => { - if (operation === 'claimCodexForever') { - process.send({ event: 'provider_started' }) - await new Promise(() => {}) - } - return { status: 200, json: async () => ({ access_token: 'synthetic', refresh_token: 'synthetic' }) } - } }) - result = { exchanged: true } - } else { - const manager = new ClusterManager({ repo: { db }, dataDir: '/tmp' }) - manager.links.set('node-a', { state: 'ready', client: {} }) - result = operation === 'issueShell' ? await manager.issueShellTicket('node-a') : { - accepted: await manager.consumeShellTicket(args.ticket, args.nodeId), + const run = async () => { + let result + if (operation === 'issueCodex') result = await generateCodexAuthUrl(args) + else if (operation === 'exchangeCodex' || operation === 'claimCodexForever') { + await exchangeCodexAuthCode({ + ...args, + fetchImpl: async () => { + if (operation === 'claimCodexForever') { + process.send({ event: 'provider_started' }) + await new Promise(() => {}) + } + return { status: 200, json: async () => ({ access_token: 'synthetic', refresh_token: 'synthetic' }) } + }, + }) + result = { exchanged: true } + } else { + const manager = new ClusterManager({ repo: { db }, dataDir: '/tmp' }) + manager.links.set('node-a', { state: 'ready', client: {} }) + result = + operation === 'issueShell' + ? await manager.issueShellTicket('node-a') + : { + accepted: await manager.consumeShellTicket(args.ticket, args.nodeId), + } } + return result } + const coordinator = getActivityCoordinator(db) + if (coordinator) { + await coordinator.acknowledgeEpoch(db.bootstrapActivity.epoch) + await db.bootstrapActivity.complete({ ended: true, reason: 'synthetic_controller_initialized' }) + } + const result = coordinator + ? await coordinator.runActivity({ kind: 'synthetic_auth_request' }, async () => ({ + ended: true, + value: await run(), + })) + : await run() process.send({ result }) } catch (error) { process.send({ error: error.code || 'child_failed' }) diff --git a/test/repository-database.mjs b/test/repository-database.mjs new file mode 100644 index 000000000..b13bb10d0 --- /dev/null +++ b/test/repository-database.mjs @@ -0,0 +1,17 @@ +/** + * PostgreSQL connections for repository and scheduler tests. These tests own + * their fixture lifetime and do not boot a control-plane HTTP server. Tests of + * maintenance admission use production createDatabase and real activity scopes. + */ +import { PostgresDatabase, applyMigrations } from '../src/lib/db/database.mjs' + +export async function createRepositoryTestDatabase(options = {}) { + const db = new PostgresDatabase({ ...options, schema: options.schema || process.env.KIN_DATABASE_SCHEMA || 'public' }) + try { + await applyMigrations(db, options) + return db + } catch (error) { + await db.close() + throw error + } +} diff --git a/test/unit/pool-scheduler-shared.test.mjs b/test/unit/pool-scheduler-shared.test.mjs index 5c84b7e8b..ef8773f31 100644 --- a/test/unit/pool-scheduler-shared.test.mjs +++ b/test/unit/pool-scheduler-shared.test.mjs @@ -5,7 +5,8 @@ import os from 'node:os' import path from 'node:path' import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' import { SeatPlanner } from '../../src/lib/pool/seat-planner.mjs' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' function gate() { diff --git a/test/unit/shared-auth-state.test.mjs b/test/unit/shared-auth-state.test.mjs index 24ede63e0..5c87c7774 100644 --- a/test/unit/shared-auth-state.test.mjs +++ b/test/unit/shared-auth-state.test.mjs @@ -5,7 +5,8 @@ import os from 'node:os' import path from 'node:path' import crypto from 'node:crypto' import { fork } from 'node:child_process' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' import { createSlotShell } from '../../src/lib/vm/slot-shell.mjs' import { createClusterRoutes } from '../../src/lib/cluster/cluster-routes.mjs' diff --git a/test/unit/shared-codex-execution.test.mjs b/test/unit/shared-codex-execution.test.mjs index e0e83a8e2..9522df563 100644 --- a/test/unit/shared-codex-execution.test.mjs +++ b/test/unit/shared-codex-execution.test.mjs @@ -6,7 +6,8 @@ import os from 'node:os' import path from 'node:path' import { EventEmitter } from 'node:events' import { setTimeout as delay } from 'node:timers/promises' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' import { handleCodexProtocol, pickCodexCandidates } from '../../src/lib/protocol/handle-codex.mjs' import { resetOpenAIAccountRuntime } from '../../src/lib/pool/openai-account-runtime.mjs' diff --git a/test/unit/shared-codex-search.test.mjs b/test/unit/shared-codex-search.test.mjs index 134e9c2c6..a70b8ab12 100644 --- a/test/unit/shared-codex-search.test.mjs +++ b/test/unit/shared-codex-search.test.mjs @@ -7,7 +7,8 @@ import http from 'node:http' import { EventEmitter } from 'node:events' import { setTimeout as delay } from 'node:timers/promises' import fetch from 'node-fetch' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { handleCodexSearch } from '../../src/lib/protocol/handle-codex.mjs' import { SharedCodexRuntime } from '../../src/lib/pool/shared-codex-runtime.mjs' import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' diff --git a/test/unit/shared-hop-lifecycle.test.mjs b/test/unit/shared-hop-lifecycle.test.mjs index d3079164d..9feb5533b 100644 --- a/test/unit/shared-hop-lifecycle.test.mjs +++ b/test/unit/shared-hop-lifecycle.test.mjs @@ -4,7 +4,8 @@ import http from 'node:http' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' import { dispatchStreamInference } from '../../src/lib/transport/kernel-router.mjs' diff --git a/test/unit/shared-session-turns.test.mjs b/test/unit/shared-session-turns.test.mjs index 10934cf2d..03faffc45 100644 --- a/test/unit/shared-session-turns.test.mjs +++ b/test/unit/shared-session-turns.test.mjs @@ -3,7 +3,8 @@ import assert from 'node:assert/strict' import { setTimeout as delay } from 'node:timers/promises' import { spawn } from 'node:child_process' import { once } from 'node:events' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' function deferred() { @@ -303,7 +304,7 @@ test('terminating the owner process leaves its persistent active turn blocking a '--input-type=module', '-e', ` - import { createDatabase } from './src/lib/db/database.mjs'; + import { createRepositoryTestDatabase as createDatabase } from './test/repository-database.mjs'; import { FailoverRunner } from './src/lib/pool/failover-runner.mjs'; const db = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: process.env.KIN_DATABASE_SCHEMA }); const runner = new FailoverRunner({ scheduler: { diff --git a/test/unit/shared-vm-operations.test.mjs b/test/unit/shared-vm-operations.test.mjs index 8a982f760..8e474882b 100644 --- a/test/unit/shared-vm-operations.test.mjs +++ b/test/unit/shared-vm-operations.test.mjs @@ -3,7 +3,8 @@ import assert from 'node:assert/strict' import { spawn } from 'node:child_process' import { once } from 'node:events' import { createInterface } from 'node:readline' -import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { withVmLock, vmLockBusy } from '../../src/lib/vm/vm-file.mjs' @@ -198,9 +199,9 @@ test('closing the owner PG connection leaves the committed operation unavailable test('SIGKILL of an operation owner does not make its VM available', async (t) => { const observer = await operations(getDb(), 'observer') - const databaseUrl = new URL('../../src/lib/db/database.mjs', import.meta.url).href + const databaseUrl = new URL('../repository-database.mjs', import.meta.url).href const serviceUrl = new URL('../../src/lib/vm/shared-vm-operations.mjs', import.meta.url).href - const script = `import { createDatabase } from ${JSON.stringify(databaseUrl)}; + const script = `import { createRepositoryTestDatabase as createDatabase } from ${JSON.stringify(databaseUrl)}; import { SharedVmOperations } from ${JSON.stringify(serviceUrl)}; const db = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: process.env.KIN_DATABASE_SCHEMA }); const claim = await new SharedVmOperations(db, { ownerId: 'killed-owner' }).tryAcquire({ vmId: 'vm-killed' }); diff --git a/test/unit/vm-operation-producers.test.mjs b/test/unit/vm-operation-producers.test.mjs index 65d119af7..08e4311c4 100644 --- a/test/unit/vm-operation-producers.test.mjs +++ b/test/unit/vm-operation-producers.test.mjs @@ -3,7 +3,8 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { AccountsRepo } from '../../src/lib/db/repos/accounts-repo.mjs' import { createImportCommit } from '../../src/lib/oauth/import-commit.mjs' From 592b90e832ec379ae16c2bede28e396955f8dc98 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:31:42 +0800 Subject: [PATCH 193/473] fix: require observed process termination before VM stop succeeds --- src/lib/cluster/native-slot.mjs | 40 ++++++--- src/lib/cluster/remote-slot.mjs | 34 ++++++-- src/lib/vm/slot-runtime.mjs | 18 +++- src/lib/vm/vm-runtime.mjs | 101 ++++++++++++++++------- test/unit/remote-slot-migration.test.mjs | 9 +- test/unit/vm-recreate.test.mjs | 12 ++- test/unit/vm-stop-proof.test.mjs | 95 ++++++++++++++++++++- 7 files changed, 252 insertions(+), 57 deletions(-) diff --git a/src/lib/cluster/native-slot.mjs b/src/lib/cluster/native-slot.mjs index 15dc90185..3c28be5dd 100644 --- a/src/lib/cluster/native-slot.mjs +++ b/src/lib/cluster/native-slot.mjs @@ -224,22 +224,36 @@ export async function nativeServiceStatus(session, domain, label) { const r = await execCollect(session.client, `launchctl print ${shellQuote(`${domain}/${label}`)}`, { timeoutMs: LAUNCHCTL_TIMEOUT_MS, }) - return parseLaunchctlPrint(`${r.stdout}\n${r.stderr}`, r.code) + const status = parseLaunchctlPrint(`${r.stdout}\n${r.stderr}`, r.code) + if ((r.code !== 0 && status.state !== 'missing') || status.state === 'unknown') { + throw new ClusterError(502, 'launchd_status_unknown', r.stderr || 'Unable to inspect launchd service') + } + return status } /** Remove a job and wait until launchd has really let go of the label (bootout returns early). */ async function bootout(session, domain, label) { const target = shellQuote(`${domain}/${label}`) - await runRemote( - session.client, - [ - `launchctl bootout ${target} >/dev/null 2>&1 || true`, - `i=0; while launchctl print ${target} >/dev/null 2>&1; do`, - ` i=$((i+1)); [ $i -ge ${BOOTOUT_WAIT_TRIES} ] && exit 1; sleep 0.2`, - 'done', - ].join('\n'), - { timeoutMs: LAUNCHCTL_TIMEOUT_MS, code: 'launchd_bootout_failed' }, - ) + const before = await nativeServiceStatus(session, domain, label) + if (before.state === 'missing') return + const result = await execCollect(session.client, `launchctl bootout ${target}`, { timeoutMs: LAUNCHCTL_TIMEOUT_MS }) + if (result.code !== 0 && parseLaunchctlPrint(`${result.stdout}\n${result.stderr}`, result.code).state !== 'missing') { + throw new ClusterError(502, 'launchd_bootout_failed', result.stderr || 'launchctl bootout failed') + } + for (let i = 0; i < BOOTOUT_WAIT_TRIES; i++) { + const current = await nativeServiceStatus(session, domain, label) + if (current.state === 'missing') { + if (!before.pid) return + const process = await execCollect(session.client, `ps -p ${before.pid} -o pid=`, { + timeoutMs: LAUNCHCTL_TIMEOUT_MS, + }) + if (process.code === 1 && !process.stdout.trim() && !process.stderr.trim()) return + if (process.code !== 0) + throw new ClusterError(502, 'launchd_exit_unknown', process.stderr || 'Unable to inspect old launchd process') + } + await new Promise((resolve) => setTimeout(resolve, 200)) + } + throw new ClusterError(502, 'launchd_exit_unconfirmed', 'Old launchd process termination was not confirmed') } async function bootstrap(session, domain, plistPath) { @@ -442,7 +456,7 @@ export async function stopNativeSlot(vm) { if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, stopped: true } return { ok: true, action: was.state === 'missing' ? 'absent' : 'stopped', runtime: vm.runtime || null } } catch (err) { - return failure(err) + return { ...failure(err), vmOperationUncertain: true } } } @@ -460,7 +474,7 @@ export async function destroyNativeSlot(vm) { if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true, removed: true } return { ok: true, action: was.state === 'missing' ? 'absent' : 'removed', runtime: vm.runtime || null } } catch (err) { - return failure(err) + return { ...failure(err), vmOperationUncertain: true } } } diff --git a/src/lib/cluster/remote-slot.mjs b/src/lib/cluster/remote-slot.mjs index 10d057985..80804f7d1 100644 --- a/src/lib/cluster/remote-slot.mjs +++ b/src/lib/cluster/remote-slot.mjs @@ -1202,21 +1202,37 @@ export async function restartRemoteServices(vm, projectRoot, { telemetryEnabled const slotDir = path.join(projectRoot, 'vms', vm.id) const name = vm.runtime?.container || containerName(vm.id) const migration = await migrateRemoteServiceContainers(vm, projectRoot) - if (!migration.ok) return migration + if (!migration.ok) return { ...migration, vmOperationUncertain: true } await pushSlotFiles(vm, slotDir, ['run', 'seed'], session) - const target = { ...vm, runtime: { ...(vm.runtime || {}), container: name, type: runtimeKind(vm) } } - const stop = 'pkill -TERM -x kin-worker 2>/dev/null || true' - await execDetached(session.docker, name, slotExecCmd(target, ['sh', '-c', stop]).Cmd, { - user: slotExecCmd(target, ['sh', '-c', stop]).User, - }) - await new Promise((resolve) => setTimeout(resolve, 150)) + const info = await inspectContainerOrNull(session.docker, name) + if (!info?.Id || info.State?.Running !== true || info.State?.Restarting) { + return { ok: false, code: 'worker_container_unknown', vmOperationUncertain: true } + } + const target = { ...vm, runtime: { ...(vm.runtime || {}), container: info.Id, type: runtimeKind(vm) } } + const stop = [ + 'pids=$(pgrep -x kin-worker 2>&1); rc=$?', + '[ "$rc" -eq 1 ] && [ -z "$pids" ] && exit 0', + '[ "$rc" -eq 0 ] || exit 2', + 'kill -TERM $pids || exit 2', + 'i=0; while :; do', + ' live=$(ps -p "$(echo "$pids" | tr "\\n" "," | sed "s/,$//")" -o pid= 2>&1); rc=$?', + ' [ "$rc" -eq 1 ] && [ -z "$live" ] && exit 0', + ' [ "$rc" -eq 0 ] || exit 2', + ' i=$((i+1)); [ "$i" -ge 25 ] && exit 3; sleep 0.2', + 'done', + ].join('\n') + const argv = slotExecArgv(target, ['sh', '-c', stop], { user: slotExecCmd(target, ['true']).User }) + await runRemote(session.client, ['docker', ...argv].map(shellQuote).join(' '), { timeoutMs: 30_000 }) + const current = await inspectContainerOrNull(session.docker, name) + if (current?.Id !== info.Id) return { ok: false, code: 'worker_container_changed', vmOperationUncertain: true } if (telemetryEnabled === true) await startRemoteTelemetry(session.docker, vm, name) const image = slotImageSpec(projectRoot, vm.kernel || 'ubuntu-24.04') const egress = await ensureRemoteEgress(session, vm, { imageRef: image.ref, binaryDigest: migration.binary_digest, }) - if (!egress.ok) return { ok: false, error: egress.error, code: egress.code || 'egress_update_failed' } + if (!egress.ok) + return { ok: false, error: egress.error, code: egress.code || 'egress_update_failed', vmOperationUncertain: true } return { ok: true, action: telemetryEnabled === true ? 'restarted' : 'stopped', @@ -1225,7 +1241,7 @@ export async function restartRemoteServices(vm, projectRoot, { telemetryEnabled migration: migration.migrated, } } catch (err) { - return failure(err, 'service_update_failed') + return { ...failure(err, 'service_update_failed'), vmOperationUncertain: true } } } diff --git a/src/lib/vm/slot-runtime.mjs b/src/lib/vm/slot-runtime.mjs index 1bca70d00..87f87f415 100644 --- a/src/lib/vm/slot-runtime.mjs +++ b/src/lib/vm/slot-runtime.mjs @@ -10,7 +10,13 @@ import { restartRustKernel, writeKernelConfig, } from '../transport/rust-kernel-supervisor.mjs' -import { ensureCodexKernel, stopCodexKernel, writeCodexKernelConfig } from '../transport/codex-kernel-supervisor.mjs' +import { + ensureCodexKernel, + finishCodexRecycle, + stopCodexKernel, + writeCodexKernelConfig, +} from '../transport/codex-kernel-supervisor.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { runtimeKind, isKvmRuntime } from './runtime-kind.mjs' import { getVm, listVms, isCodexVm, persistVmRuntime } from './vm-registry.mjs' import { resolveInferenceEngine } from './slot-engine.mjs' @@ -62,7 +68,15 @@ export async function startSlotReady(vm, projectRoot, opts = {}) { } export async function stopSlot(vm) { - if (isCodexVm(vm)) stopCodexKernel(vm.id) + if (isCodexVm(vm)) { + const child = stopCodexKernel(vm.id) + if (!child && isKubernetesMode()) { + return { ok: false, code: 'codex_kernel_owner_unknown', vmOperationUncertain: true } + } + if (child && !(await finishCodexRecycle(child)).exited) { + return { ok: false, code: 'codex_kernel_exit_unconfirmed', vmOperationUncertain: true } + } + } return slotHost(vm).stop(vm) } diff --git a/src/lib/vm/vm-runtime.mjs b/src/lib/vm/vm-runtime.mjs index 7e654a3d8..9c7e45f3e 100644 --- a/src/lib/vm/vm-runtime.mjs +++ b/src/lib/vm/vm-runtime.mjs @@ -154,15 +154,28 @@ export function inspectContainer(name) { 'docker', 'inspect', '--format', - '{{.State.Running}}|{{.State.Pid}}|{{.HostConfig.NetworkMode}}|{{.State.StartedAt}}|{{.Config.Image}}|{{.Config.Hostname}}|{{index .Config.Labels "kin.vm.runtime"}}|{{index .Config.Labels "kin.vm.role"}}|{{.State.Restarting}}', + '{{.State.Running}}|{{.State.Pid}}|{{.HostConfig.NetworkMode}}|{{.State.StartedAt}}|{{.Config.Image}}|{{.Config.Hostname}}|{{index .Config.Labels "kin.vm.runtime"}}|{{index .Config.Labels "kin.vm.role"}}|{{.State.Restarting}}|{{.Id}}', name, ]) - if (!r.ok) return null - const [running, pid, networkMode, startedAt, image, hostname, runtime, role, restarting] = r.stdout.split('|') + if (!r.ok) { + if (/^(?:Error(?: response from daemon)?: )?No such (?:object|container): [^\r\n]+$/i.test(r.stderr)) return null + throw Object.assign(new Error(r.stderr || 'docker inspect failed'), { code: 'container_inspect_failed' }) + } + const [running, pid, networkMode, startedAt, image, hostname, runtime, role, restarting, id] = r.stdout.split('|') + if ( + !['true', 'false'].includes(running) || + !['true', 'false'].includes(restarting) || + !id?.trim() || + !/^\d+$/.test(pid) + ) { + throw Object.assign(new Error('Invalid docker inspect state'), { code: 'container_inspect_failed' }) + } // Docker reports Running=true during a restart-policy backoff; nothing can be exec'd there. const crashLooping = restarting === 'true' return { name, + id: id.trim(), + stateRunning: running === 'true', running: running === 'true' && !crashLooping, restarting: crashLooping, pid: Number(pid) || 0, @@ -1158,39 +1171,69 @@ export async function startVmRuntime(vm, projectRoot, { recreate = false, routin /** Explicit factory reset / delete only. Never call from Node deploy. */ export function destroyVmRuntime(vm) { if (!vm?.id) return { ok: false, error: 'vm required' } - const name = containerName(vm.id) - const info = inspectContainer(name) - if (!info) { + try { + const name = containerName(vm.id) + const info = inspectContainer(name) + if (!info) { + if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true, removed: true } + return { ok: true, action: 'absent', runtime: vm.runtime || null } + } + assertContainerIdentity(vm, info) + const r = sh(['docker', 'rm', '-f', info.id], { timeout: 60_000 }) + const remaining = inspectContainer(info.id) + const named = inspectContainer(name) + if (remaining || named) { + return unconfirmedContainer(r.stderr || 'Container removal was not confirmed') + } if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true, removed: true } - return { ok: true, action: 'absent', runtime: vm.runtime || null } - } - const r = sh(['docker', 'rm', '-f', name], { timeout: 60_000 }) - if (!r.ok && inspectContainer(name)) { - return { ok: false, error: r.stderr || 'docker rm failed' } + return { ok: true, action: 'removed', runtime: vm.runtime || null } + } catch (error) { + return unconfirmedContainer(error.message, error.code) } - if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true, removed: true } - return { ok: true, action: 'removed', runtime: vm.runtime || null } } export function stopVmRuntime(vm) { - const name = containerName(vm.id) - const info = inspectContainer(name) - if (!info) { - if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true } - return { ok: true, action: 'absent', runtime: vm.runtime || null } - } - if (!info.running) { - runtimePatch(vm, info) + try { + const name = containerName(vm.id) + const info = inspectContainer(name) + if (!info) { + if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true } + return { ok: true, action: 'absent', runtime: vm.runtime || null } + } + assertContainerIdentity(vm, info) + if (containerStopped(info)) { + runtimePatch(vm, info) + vm.runtime.stopped = true + return { ok: true, action: 'already-stopped', runtime: vm.runtime } + } + const grace = isKvmRuntime(vm) || info.runtime === 'kvm' ? '30' : '3' + const r = sh(['docker', 'stop', '-t', grace, info.id]) + const remaining = inspectContainer(info.id) + const named = inspectContainer(name) + if ((remaining && !containerStopped(remaining)) || (named && (named.id !== info.id || !containerStopped(named)))) { + return unconfirmedContainer(r.stderr || 'Container termination was not confirmed') + } + runtimePatch(vm, named || (remaining ? { ...remaining, name } : null)) vm.runtime.stopped = true - return { ok: true, action: 'already-stopped', runtime: vm.runtime } + vm.runtime.pid = null + return { ok: true, action: 'stopped', runtime: vm.runtime } + } catch (error) { + return unconfirmedContainer(error.message, error.code) + } +} + +function assertContainerIdentity(vm, info) { + if (vm.runtime?.container_id && vm.runtime.container_id !== info.id) { + throw Object.assign(new Error('Container identity changed'), { code: 'container_identity_changed' }) } - const grace = isKvmRuntime(vm) || info.runtime === 'kvm' ? '30' : '3' - const r = sh(['docker', 'stop', '-t', grace, name]) - if (!r.ok) return { ok: false, error: r.stderr || 'docker stop failed' } - runtimePatch(vm, inspectContainer(name)) - vm.runtime.stopped = true - vm.runtime.pid = null - return { ok: true, action: 'stopped', runtime: vm.runtime } +} + +function containerStopped(info) { + return info.stateRunning === false && info.restarting === false && info.pid === 0 +} + +function unconfirmedContainer(error, code = 'container_termination_unconfirmed') { + return { ok: false, error, code, vmOperationUncertain: true } } export function listRuntimeVms() { diff --git a/test/unit/remote-slot-migration.test.mjs b/test/unit/remote-slot-migration.test.mjs index 0b8220555..c045ec97e 100644 --- a/test/unit/remote-slot-migration.test.mjs +++ b/test/unit/remote-slot-migration.test.mjs @@ -189,8 +189,13 @@ test('remote service refresh forwards the migrated binary digest to egress', asy pushSlotFiles: async () => {}, runtimeKind: () => 'docker', slotExecCmd: () => ({ Cmd: ['true'], User: '1000:1000' }), - execDetached: async () => {}, - setTimeout: (done) => done(), + slotExecArgv: (vm) => { + assert.equal(vm.runtime.container, 'immutable-slot-id') + return ['exec', 'immutable-slot-id', 'true'] + }, + inspectContainerOrNull: async () => ({ Id: 'immutable-slot-id', State: { Running: true } }), + runRemote: async (_client, command) => assert.match(command, /immutable-slot-id/), + shellQuote: (s) => `'${s}'`, slotImageSpec: () => ({ ref: 'slot-image' }), ensureRemoteEgress: async (_session, _vm, { binaryDigest }) => { receivedDigest = binaryDigest diff --git a/test/unit/vm-recreate.test.mjs b/test/unit/vm-recreate.test.mjs index 858fa4df1..218846637 100644 --- a/test/unit/vm-recreate.test.mjs +++ b/test/unit/vm-recreate.test.mjs @@ -174,7 +174,17 @@ test('keep rebuild retains the account, fingerprint and credential file', () => fs.rmSync(root, { recursive: true, force: true }) }) -test('destroy missing container is a no-op', () => { +test('destroy missing container is a no-op', (t) => { + const bin = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-missing-docker-')) + const previousPath = process.env.PATH + fs.writeFileSync(path.join(bin, 'docker'), '#!/bin/sh\necho "Error: No such object: synthetic" >&2\nexit 1\n', { + mode: 0o700, + }) + process.env.PATH = bin + t.after(() => { + process.env.PATH = previousPath + fs.rmSync(bin, { recursive: true, force: true }) + }) const r = destroyVmRuntime({ id: 'vm-missing-slot' }) assert.equal(r.ok, true) assert.equal(r.action, 'absent') diff --git a/test/unit/vm-stop-proof.test.mjs b/test/unit/vm-stop-proof.test.mjs index 9b5afc88c..a07b10e93 100644 --- a/test/unit/vm-stop-proof.test.mjs +++ b/test/unit/vm-stop-proof.test.mjs @@ -3,6 +3,7 @@ import assert from 'node:assert/strict' import fs from 'node:fs' import path from 'node:path' import vmContext from 'node:vm' +import { spawnSync } from 'node:child_process' import { getDb } from '../../src/lib/db/database.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { parseLaunchctlPrint } from '../../src/lib/cluster/native-slot.mjs' @@ -150,6 +151,29 @@ test('Codex stop awaits child exit before stopping its slot host', async () => { assert.deepEqual(events, ['host-stop']) }) +test('Codex child exit failure and unknown Kubernetes owner keep the slot host running', async () => { + const source = section('../../src/lib/vm/slot-runtime.mjs', 'export async function stopSlot(', '/** Destroy the slot') + for (const child of [{ pid: 123 }, null]) { + let hostStops = 0 + const stop = vmContext.runInNewContext(`${source}; stopSlot`, { + isCodexVm: () => true, + isKubernetesMode: () => true, + stopCodexKernel: () => child, + finishCodexRecycle: async () => ({ exited: false }), + slotHost: () => ({ + stop: () => { + hostStops++ + return { ok: true } + }, + }), + }) + const result = await stop({ id: 'vm-synthetic' }) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal(hostStops, 0) + } +}) + function nativeApi(response) { const source = section( '../../src/lib/cluster/native-slot.mjs', @@ -158,7 +182,7 @@ function nativeApi(response) { ) return vmContext.runInNewContext(`${source}; ({nativeServiceStatus, bootout})`, { parseLaunchctlPrint, - execCollect: async () => response, + execCollect: async (_client, command) => (typeof response === 'function' ? response(command) : response), runRemote: async () => '', shellQuote: (s) => `'${s}'`, ClusterError, @@ -174,6 +198,33 @@ test('launchctl permission errors cannot prove native service termination', asyn await assert.rejects(api.bootout({}, 'gui/501', 'label')) }) +test('launchd label disappearance alone does not prove the old PID exited', async () => { + for (const process of [ + { code: 0, stdout: '123', stderr: '' }, + { code: 1, stdout: '', stderr: 'Operation not permitted' }, + ]) { + let prints = 0 + const api = nativeApi((command) => { + if (command.startsWith('ps ')) return process + if (command.startsWith('launchctl bootout')) return { code: 0, stdout: '', stderr: '' } + if (prints++ === 0) return { code: 0, stdout: '\tstate = running\n\tpid = 123', stderr: '' } + return { code: 113, stdout: '', stderr: 'Could not find service' } + }) + await assert.rejects(api.bootout({}, 'gui/501', 'label')) + } +}) + +test('launchd confirms both missing label and old PID absence', async () => { + let prints = 0 + const api = nativeApi((command) => { + if (command.startsWith('ps ')) return { code: 1, stdout: '', stderr: '' } + if (command.startsWith('launchctl bootout')) return { code: 0, stdout: '', stderr: '' } + if (prints++ === 0) return { code: 0, stdout: '\tstate = running\n\tpid = 123', stderr: '' } + return { code: 113, stdout: '', stderr: 'Could not find service' } + }) + await api.bootout({}, 'gui/501', 'label') +}) + test('launchctl explicit service-not-found remains a valid absence proof', async () => { const api = nativeApi({ code: 113, stdout: '', stderr: 'Could not find service' }) assert.equal((await api.nativeServiceStatus({}, 'gui/501', 'label')).state, 'missing') @@ -212,3 +263,45 @@ test('remote worker stop without a foreground exit acknowledgement stays unknown assert.equal(result.ok, false) assert.equal(result.vmOperationUncertain, true) }) + +test('remote worker termination script distinguishes live PIDs and inspection failures from exit', async () => { + const source = section( + '../../src/lib/cluster/remote-slot.mjs', + 'export async function restartRemoteServices(', + 'export async function stopRemoteSlot(', + ) + let script + const restart = vmContext.runInNewContext(`${source}; restartRemoteServices`, { + vmNodeId: () => 'node-synthetic', + nodeSession: async () => ({ client: {}, host: {}, docker: {} }), + remoteSlotDir: () => '/synthetic', + path, + containerName: () => 'kin-synthetic', + migrateRemoteServiceContainers: async () => ({ ok: true, migrated: [] }), + pushSlotFiles: async () => {}, + runtimeKind: () => 'docker', + slotExecCmd: () => ({}), + slotExecArgv: (vm, argv) => { + assert.equal(vm.runtime.container, 'immutable-id') + script = argv[2] + return ['exec', 'immutable-id', ...argv] + }, + inspectContainerOrNull: async () => ({ Id: 'immutable-id', State: { Running: true } }), + runRemote: async () => '', + shellQuote: (s) => `'${s}'`, + slotImageSpec: () => ({ ref: 'synthetic' }), + ensureRemoteEgress: async () => ({ ok: true }), + failure: (error) => ({ ok: false, error: error.message }), + }) + assert.equal((await restart({ id: 'vm-synthetic' }, '/synthetic', {})).ok, true) + for (const [ps, expected] of [ + ['printf "123\\n"; return 0', 3], + ['printf "permission denied\\n" >&2; return 1', 2], + ['return 1', 0], + ]) { + // Shell functions replace every process query/signal; no real process is inspected or killed. + const synthetic = `pgrep() { printf '123\\n'; }; kill() { return 0; }; sleep() { :; }; ps() { ${ps}; };\n${script}` + const result = spawnSync('/bin/sh', ['-c', synthetic], { encoding: 'utf8', timeout: 2000 }) + assert.equal(result.status, expected, result.stderr) + } +}) From 05c4abcdf5cfccaa2bf95d9fa17d7407793af7f5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:32:12 +0800 Subject: [PATCH 194/473] test: reproduce background SQL outside live maintenance activities --- .../unit/maintenance-producer-scopes.test.mjs | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 test/unit/maintenance-producer-scopes.test.mjs diff --git a/test/unit/maintenance-producer-scopes.test.mjs b/test/unit/maintenance-producer-scopes.test.mjs new file mode 100644 index 000000000..ef3c2a0ab --- /dev/null +++ b/test/unit/maintenance-producer-scopes.test.mjs @@ -0,0 +1,129 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb, getActivityCoordinator } from '../../src/lib/db/database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' +import { SharedRoutingConfig } from '../../src/lib/admin/shared-routing-config.mjs' +import { RoutingConfigRepo } from '../../src/lib/db/repos/routing-config-repo.mjs' +import { RequestLogStore } from '../../src/lib/admin/request-log.mjs' +import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' +import { initVmDbSync, startVmWatch, stopVmWatch, pauseVmDbSync } from '../../src/lib/vm/vm-db-sync.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' + +async function fixture(t) { + const db = getDb() + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-maintenance-producers-')) + await withDatabaseActivities([db], () => Promise.resolve()) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + return { db, root, coordinator: getActivityCoordinator(db) } +} + +function intervals(t, start) { + let callback + const mock = t.mock.method(globalThis, 'setInterval', (fn) => { + callback = fn + return { unref() {} } + }) + try { + start() + } finally { + mock.mock.restore() + } + assert.equal(typeof callback, 'function') + return callback +} + +test('routing polling refreshes PostgreSQL after its startup activity has closed', async (t) => { + const fx = await fixture(t) + const file = path.join(fx.root, 'routing.json') + fs.writeFileSync(file, JSON.stringify({ version: 1 })) + const applied = [] + const state = new SharedRoutingConfig({ + db: fx.db, + file, + normalize: (x) => x, + merge: (a, b) => ({ ...a, ...b }), + apply: (x) => applied.push(x.version), + }) + await withDatabaseActivities([fx.db], () => state.init()) + const peer = await createRepositoryTestDatabase({ connectionString: fx.db.connectionString, schema: fx.db.schema }) + t.after(() => peer.close()) + await new RoutingConfigRepo(peer).patch(() => ({ version: 2 })) + const tick = intervals(t, () => state.start()) + t.after(() => state.stop()) + tick() + await state.refreshing + assert.equal(state.lastError, undefined) + assert.deepEqual(applied, [1, 2]) +}) + +test('request-log cleanup enters a new activity instead of inheriting a closed startup scope', async (t) => { + const fx = await fixture(t) + const log = new RequestLogStore({ db: fx.db, dataDir: fx.root }) + let cleaned = 0 + log.repo.cleanup = async () => { + await fx.db.query('SELECT 1') + cleaned++ + } + const tick = intervals(t, () => log.startCleanupScheduler()) + t.after(() => log.stopCleanupScheduler()) + tick() + await log.flush() + assert.equal(cleaned, 1) +}) + +test('each scheduled proxy probe receives its own live activity', async (t) => { + const fx = await fixture(t) + const pool = new ProxyPool({ db: fx.db }) + let probes = 0 + pool.probeAll = async () => { + await fx.db.query('SELECT 1') + probes++ + return { ok: true } + } + const tick = intervals(t, () => pool.startScheduler()) + t.after(() => pool.stopScheduler()) + await pool.flush() + await tick() + await pool.flush() + assert.equal(probes, 2) +}) + +test('file-watcher reconciliation can persist after initialization scope closes', async (t) => { + const fx = await fixture(t) + fs.mkdirSync(path.join(fx.root, 'vms')) + const file = path.join(fx.root, 'vms/vm-watch.json') + fs.writeFileSync(file, JSON.stringify({ id: 'vm-watch', note: 'before' })) + await withDatabaseActivities([fx.db], () => initVmDbSync(fx.root, { watch: false })) + let changed + const watcher = t.mock.method(fs, 'watch', (_dir, callback) => { + changed = callback + return { close() {} } + }) + try { + assert.equal(startVmWatch(fx.root), true) + } finally { + watcher.mock.restore() + } + t.after(async () => { + stopVmWatch() + await pauseVmDbSync() + }) + fs.writeFileSync(file, JSON.stringify({ id: 'vm-watch', note: 'after' })) + let tick + const timer = t.mock.method(globalThis, 'setTimeout', (callback) => { + tick = callback + return { unref() {} } + }) + try { + changed() + } finally { + timer.mock.restore() + } + await tick() + const row = await withDatabaseActivities([fx.db], () => new VmsRepo(fx.db).get('vm-watch')) + assert.equal(row.vm.note, 'after') +}) From 4813032721f1fc58ffe2cadb800a0cd5a4e54158 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:35:19 +0800 Subject: [PATCH 195/473] feat: retain terminal activities until execution outcome is known --- src/lib/cluster/cluster-routes.mjs | 54 +++++++++++++++++++++---- src/lib/vm/slot-shell.mjs | 65 ++++++++++++++++++++++++------ src/server.mjs | 49 +++++++++++++++++++--- 3 files changed, 142 insertions(+), 26 deletions(-) diff --git a/src/lib/cluster/cluster-routes.mjs b/src/lib/cluster/cluster-routes.mjs index 655f576e2..999f99a2d 100644 --- a/src/lib/cluster/cluster-routes.mjs +++ b/src/lib/cluster/cluster-routes.mjs @@ -5,6 +5,7 @@ */ import { WebSocketServer } from 'ws' +import { randomUUID } from 'node:crypto' import { containerAction, containerLogs, @@ -34,7 +35,7 @@ function clampInt(v, lo, hi, dflt) { return Math.min(hi, Math.max(lo, n)) } -export function createClusterRoutes({ manager, json, readBody, ok }) { +export function createClusterRoutes({ manager, json, readBody, ok, onExecutionState = null }) { function fail(res, err, respond = json) { const status = err instanceof ClusterError ? err.status : 500 const code = err instanceof ClusterError ? err.code : 'cluster_error' @@ -188,34 +189,58 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { const url = new URL(req.url || '/', 'http://local') const sm = SHELL_RE.exec(url.pathname) if (!sm) return false + const requestId = randomUUID() + const reject = (status, text) => { + if (!onExecutionState) return rejectUpgrade(socket, status, text) + track( + Promise.resolve() + .then(() => + onExecutionState(req, { + requestId, + started: false, + ended: true, + reason: 'not_started', + phase: 'complete', + }), + ) + .then(() => rejectUpgrade(socket, status, text)), + ) + } if (draining) { - rejectUpgrade(socket, 503, 'Service Unavailable') + reject(503, 'Service Unavailable') return true } const id = sm[1] const accept = (authorized) => { - if (!authorized) return rejectUpgrade(socket, 401, 'Unauthorized') - if (draining || socket.destroyed) return rejectUpgrade(socket, 503, 'Service Unavailable') + if (!authorized) return reject(401, 'Unauthorized') + if (draining || socket.destroyed) return reject(503, 'Service Unavailable') const cols = clampInt(url.searchParams.get('cols'), 10, 500, 80) const rows = clampInt(url.searchParams.get('rows'), 5, 200, 24) wss.handleUpgrade(req, socket, head, (ws) => { track(new Promise((resolve) => ws.once('close', resolve))) - track(attachShell(ws, id, { cols, rows })) + track(attachShell(ws, id, { cols, rows }, req, requestId)) }) } const consumed = manager.consumeShellTicket(url.searchParams.get('ticket') || '', id) if (typeof consumed?.then === 'function') { - track(consumed.then(accept, () => rejectUpgrade(socket, 503, 'Service Unavailable'))) + track(consumed.then(accept, () => reject(503, 'Service Unavailable'))) } else accept(consumed) return true } - async function attachShell(ws, id, size) { + async function attachShell(ws, id, size, req, requestId) { // Listen before the channel opens: input/resize sent during the SSH round // trip would otherwise be dropped (ws does not buffer unhandled messages). const pending = [] let stream = null let closed = false + let started = false + let completed = false + const finish = async (ended, reason) => { + if (completed) return + completed = true + await onExecutionState?.(req, { requestId, started, ended, reason, phase: 'complete' }) + } const apply = (msg) => { if (msg?.t === 'd' && typeof msg.d === 'string') stream.write(msg.d) else if (msg?.t === 'r') stream.setWindow(clampInt(msg.r, 5, 200, 24), clampInt(msg.c, 10, 500, 80), 0, 0) @@ -234,22 +259,35 @@ export function createClusterRoutes({ manager, json, readBody, ok }) { ws.on('close', () => { closed = true stream?.close() + if (stream) track(finish(false, 'client_closed')) }) try { + await onExecutionState?.(req, { requestId, started: true, ended: false, reason: 'started', phase: 'start' }) + if (closed || draining) { + await finish(true, 'not_started') + return + } + started = true stream = await manager.openShell(id, size) } catch (err) { + await finish(!started, started ? 'ssh_open_failed' : 'not_started') if (ws.readyState === ws.OPEN) ws.send(JSON.stringify({ t: 'error', message: err.message })) ws.close(1011) return } + stream.on('exit', () => track(finish(true, 'ssh_exit'))) + stream.on('close', () => { + track(finish(false, 'ssh_channel_closed')) + ws.close(1000) + }) if (closed) { stream.close() + await finish(false, 'client_closed') return } stream.on('data', (chunk) => ws.readyState === ws.OPEN && ws.send(chunk)) stream.stderr?.on('data', (chunk) => ws.readyState === ws.OPEN && ws.send(chunk)) stream.on('exit', (code) => ws.readyState === ws.OPEN && ws.send(JSON.stringify({ t: 'exit', code }))) - stream.on('close', () => ws.close(1000)) for (const msg of pending.splice(0)) apply(msg) } diff --git a/src/lib/vm/slot-shell.mjs b/src/lib/vm/slot-shell.mjs index 6498c9162..5e14014b5 100644 --- a/src/lib/vm/slot-shell.mjs +++ b/src/lib/vm/slot-shell.mjs @@ -379,6 +379,7 @@ export function createSlotShell({ authLoginReady = null, upgradeLoginReady = null, db, + onExecutionState = null, } = {}) { const tickets = new Map() const authState = isKubernetesMode() ? new OneTimeAuthRepo(db) : null @@ -457,8 +458,25 @@ export function createSlotShell({ const url = new URL(req.url || '/', 'http://local') const sm = SHELL_RE.exec(url.pathname) if (!sm) return false + const requestId = crypto.randomUUID() + const reject = (status, text) => { + if (!onExecutionState) return rejectUpgrade(socket, status, text) + track( + Promise.resolve() + .then(() => + onExecutionState(req, { + requestId, + started: false, + ended: true, + reason: 'not_started', + phase: 'complete', + }), + ) + .then(() => rejectUpgrade(socket, status, text)), + ) + } if (draining) { - rejectUpgrade(socket, 503, 'Service Unavailable') + reject(503, 'Service Unavailable') return true } const vmId = decodeURIComponent(sm[1]) @@ -469,8 +487,7 @@ export function createSlotShell({ consumeTicket(ticket, vmId, requestedLogin) .then(acceptTicket) .catch((error) => - rejectUpgrade( - socket, + reject( error.code === 'shell_ticket_scope_mismatch' ? 409 : 503, error.code === 'shell_ticket_scope_mismatch' ? 'Conflict' : 'Service Unavailable', ), @@ -482,22 +499,22 @@ export function createSlotShell({ function acceptTicket(entry) { if (!entry) { - rejectUpgrade(socket, 401, 'Unauthorized') + reject(401, 'Unauthorized') return true } const vm = getVm(projectRoot, vmId) if (!vm || isCodexVm(vm)) { - rejectUpgrade(socket, 404, 'Not Found') + reject(404, 'Not Found') return true } if (requestedLogin !== !!entry.login) { - rejectUpgrade(socket, 409, 'Conflict') + reject(409, 'Conflict') return true } const login = !!entry.login const accept = () => { if (draining || socket.destroyed) { - rejectUpgrade(socket, 503, 'Service Unavailable') + reject(503, 'Service Unavailable') return } if (sessions >= MAX_SESSIONS) { @@ -512,7 +529,7 @@ export function createSlotShell({ } wss.handleUpgrade(req, socket, head, (ws) => { track(new Promise((resolve) => ws.once('close', resolve))) - track(attach(ws, vm, size)) + track(attach(ws, vm, size, req, requestId)) }) } if (!login) { @@ -523,8 +540,7 @@ export function createSlotShell({ Promise.resolve() .then(() => (upgradeLoginReady || authLoginReady || assertAuthLoginReady)(projectRoot, vm)) .then(accept, (err) => - rejectUpgrade( - socket, + reject( 409, String(err?.message || 'Conflict') .replace(/[\r\n]+/g, ' ') @@ -536,7 +552,7 @@ export function createSlotShell({ } } - async function attach(ws, vm, size) { + async function attach(ws, vm, size, req, requestId) { const token = crypto.randomBytes(12).toString('hex') liveTokens.add(token) const container = slotContainerName({ vm }) @@ -545,6 +561,13 @@ export function createSlotShell({ let closed = false let released = false let adoptTimer = null + let started = false + let completed = false + const finish = async (ended, reason) => { + if (completed) return + completed = true + await onExecutionState?.(req, { requestId, started, ended, reason, phase: 'complete' }) + } connections.set(ws, () => { clearInterval(adoptTimer) ws.close(1001, 'Control plane is draining') @@ -599,6 +622,13 @@ export function createSlotShell({ }) try { connect = slotHost(vm).dockerApi() + await onExecutionState?.(req, { requestId, started: true, ended: false, reason: 'started', phase: 'start' }) + if (closed || draining) { + await finish(true, 'not_started') + ws.close(1001) + return + } + started = true if (size.login) await preparePanelShellLogin(projectRoot, vm) const launch = panelShellLaunch(slotHost(vm).bins.cli) const env = ['TERM=xterm-256color', `${SESSION_ENV}=${token}`, `${RC_ENV}=${launch.rc}`] @@ -616,6 +646,7 @@ export function createSlotShell({ rows: size.rows, }) } catch (err) { + await finish(!started, started ? 'docker_exec_open_failed' : 'not_started') send(JSON.stringify({ t: 'error', message: `打开终端失败：${err.message}` })) ws.close(1011) return @@ -624,6 +655,8 @@ export function createSlotShell({ // The close handler ran before the session existed: reap here instead. session.stream.destroy() await reap().catch(() => {}) + const info = await inspectExec(connect, session.execId).catch(() => null) + await finish(info?.Running === false, info?.Running === false ? 'docker_exec_stopped' : 'docker_exec_unknown') return } // ConsoleSize on exec start is API >= 1.42; resize covers older daemons. @@ -673,10 +706,18 @@ export function createSlotShell({ } } const info = await inspectExec(connect, session.execId).catch(() => null) + await finish( + info?.Running === false, + info?.Running === false ? 'docker_exec_stopped' : 'docker_exec_unknown', + ) send(JSON.stringify({ t: 'exit', code: info?.ExitCode ?? null })) ws.close(1000) } finally { - completeTerminal() + try { + if (!completed) await finish(false, 'terminal_cleanup_failed') + } finally { + completeTerminal() + } } })(), ) diff --git a/src/server.mjs b/src/server.mjs index 992ae0d0a..39d9db674 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1090,8 +1090,18 @@ if (proxyPool.snapshot().config.ipv6_enabled !== true) { console.warn('[ipv6-policy] exit not blocked', exit.vm_id || exit.proxy_id, exit.error) } } -const clusterRoutes = createClusterRoutes({ manager: clusterManager, json, readBody, ok: panel.ok }) -const slotShell = createSlotShell({ projectRoot: cfg.paths.project, importOauth: commitImportedOauth }) +const clusterRoutes = createClusterRoutes({ + manager: clusterManager, + json, + readBody, + ok: panel.ok, + onExecutionState: noteRequestExecution, +}) +const slotShell = createSlotShell({ + projectRoot: cfg.paths.project, + importOauth: commitImportedOauth, + onExecutionState: noteRequestExecution, +}) for (const vm of listVms(cfg.paths.project)) { if (isCodexVm(vm)) continue try { @@ -1203,6 +1213,7 @@ function noteRequestExecution(req, execution) { ended: execution.ended === true, }) } + if (execution?.phase === 'complete') state.finishExecution?.() } async function withRequestActivity(req, res, callback, kind = 'http') { @@ -1221,7 +1232,7 @@ async function withRequestActivity(req, res, callback, kind = 'http') { try { return await gate.run(async () => { try { - const value = await callback() + const value = await callback(state) await responseDone await flushRuntimeWrites() return value @@ -1465,9 +1476,27 @@ server.on('upgrade', async (req, socket, head) => { socket.end('HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n') return } - if (await responsesWs.handleUpgrade(req, socket, head)) return - if (clusterRoutes.handleUpgrade(req, socket, head)) return - if (!slotShell.handleUpgrade(req, socket, head)) socket.destroy() + try { + await withRequestActivity( + req, + null, + async (state) => { + const terminal = new Promise((resolve) => { + if (state) state.finishExecution = resolve + }) + if (await responsesWs.handleUpgrade(req, socket, head)) return + if (clusterRoutes.handleUpgrade(req, socket, head) || slotShell.handleUpgrade(req, socket, head)) { + if (state) await terminal + return + } + socket.destroy() + }, + 'websocket_upgrade', + ) + } catch (error) { + console.warn(`[websocket] activity failed: ${error.code || error.name}`) + socket.end('HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n') + } }) server.on('clientError', (err, socket) => { @@ -1522,6 +1551,13 @@ process.on('unhandledRejection', (e) => console.error('[unhandled]', e)) async function shutdown(signal, { failed = false } = {}) { if (_shuttingDown) return _shuttingDown = true + let shutdownActivity + if (activityCoordinator?.healthy) { + try { + shutdownActivity = await activityCoordinator.admit({ kind: 'shutdown' }) + shutdownActivity.enter() + } catch {} + } lifecycle.beginDrain() responsesWs.beginDrain() clusterRoutes.beginDrain() @@ -1611,6 +1647,7 @@ async function shutdown(signal, { failed = false } = {}) { await flushRuntimeWrites() stopAllRustKernels() await clusterManager.stop() + await shutdownActivity?.complete({ ended: true, reason: 'shutdown_persistence_completed' }) await activityCoordinator?.stop() await controlPlaneLock?.stop() await closeDatabase() From 4e5cc46870b6b4bbb69583eb123e2db2ab823c64 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:35:51 +0800 Subject: [PATCH 196/473] fix: admit independent activities for background database producers --- src/lib/admin/request-log.mjs | 14 +++++-- src/lib/admin/shared-routing-config.mjs | 12 +++++- src/lib/vm/proxy-pool.mjs | 18 +++++--- src/lib/vm/vm-db-sync.mjs | 26 +++++++++--- .../unit/maintenance-producer-scopes.test.mjs | 42 +++++++++++++++++++ 5 files changed, 97 insertions(+), 15 deletions(-) diff --git a/src/lib/admin/request-log.mjs b/src/lib/admin/request-log.mjs index 7c1d71bb6..2d004cb0f 100644 --- a/src/lib/admin/request-log.mjs +++ b/src/lib/admin/request-log.mjs @@ -18,7 +18,7 @@ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' import { redactSecrets } from '../core/security.mjs' -import { resolveStoreDb } from '../db/database.mjs' +import { resolveStoreDb, getActivityCoordinator } from '../db/database.mjs' import { integerValue } from '../db/integers.mjs' import { UsageLogsRepo } from '../db/repos/usage-logs-repo.mjs' import { classifyRequestError, resolveMutedErrorClasses } from './error-class.mjs' @@ -759,7 +759,9 @@ export class RequestLogStore { debugRetainDays: this.debugRetainDays, maxBytes: this.maxMb > 0 ? this.maxMb * 1024 * 1024 : 0, }) - } catch {} + } catch (error) { + if (getActivityCoordinator(this.db)) throw error + } this._cleanupLegacyFiles(this.retainDays, this.root) this._cleanupLegacyFiles(this.debugRetainDays, path.join(this.root, 'debug')) } @@ -770,7 +772,13 @@ export class RequestLogStore { const run = () => { const pending = (async () => { try { - await this.cleanup() + const coordinator = getActivityCoordinator(this.db) + if (coordinator) { + await coordinator.runActivity({ kind: 'request_log_cleanup' }, async () => ({ + ended: true, + value: await this.cleanup(), + })) + } else await this.cleanup() } catch {} })() this._cleanupPending.add(pending) diff --git a/src/lib/admin/shared-routing-config.mjs b/src/lib/admin/shared-routing-config.mjs index c29afff74..d2e77cec4 100644 --- a/src/lib/admin/shared-routing-config.mjs +++ b/src/lib/admin/shared-routing-config.mjs @@ -1,5 +1,5 @@ import fs from 'node:fs' -import { getDb } from '../db/database.mjs' +import { getDb, getActivityCoordinator } from '../db/database.mjs' import { RoutingConfigRepo } from '../db/repos/routing-config-repo.mjs' import { manageRoutingDocument, clearRoutingDocument, publishRoutingDocument } from '../core/routing-document.mjs' @@ -66,7 +66,15 @@ export class SharedRoutingConfig { if (this.timer) return this.timer = setInterval(() => { if (this.refreshing) return - this.refreshing = this.refresh() + const coordinator = getActivityCoordinator(this.db) + this.refreshing = ( + coordinator + ? coordinator.runActivity({ kind: 'routing_poll' }, async () => ({ + ended: true, + value: await this.refresh(), + })) + : this.refresh() + ) .catch((error) => { this.lastError = error.code || error.message }) diff --git a/src/lib/vm/proxy-pool.mjs b/src/lib/vm/proxy-pool.mjs index 429ad9453..ed578c053 100644 --- a/src/lib/vm/proxy-pool.mjs +++ b/src/lib/vm/proxy-pool.mjs @@ -12,7 +12,7 @@ import { EventEmitter } from 'node:events' import net from 'node:net' import crypto from 'node:crypto' -import { resolveStoreDb, withTransaction } from '../db/database.mjs' +import { resolveStoreDb, withTransaction, getActivityCoordinator } from '../db/database.mjs' import { ProxiesRepo, PROXY_COLUMNS, PROXY_HEALTH_COLUMNS } from '../db/repos/proxies-repo.mjs' import { canBindProxyToVm, normalizeOwnerId, proxyOwnerId, vmOwnerId } from '../admin/resource-owner.mjs' import { validTimezone } from '../core/timezone.mjs' @@ -1617,10 +1617,18 @@ export class ProxyPool extends EventEmitter { if (!this.state.config.enabled) return const min = this.state.config.probe_interval_min || 10 const ms = min * 60 * 1000 - void this.probeAll({ onlyEnabled: true }).catch(() => {}) - this._timer = setInterval(async () => { - this.probeAll({ onlyEnabled: true }).catch(() => {}) - }, ms) + const run = () => { + const coordinator = getActivityCoordinator(this.db) + const pending = coordinator + ? coordinator.runActivity({ kind: 'proxy_probe' }, async () => ({ + ended: true, + value: await this.probeAll({ onlyEnabled: true }), + })) + : this.probeAll({ onlyEnabled: true }) + return this._track(pending).catch(() => {}) + } + void run() + this._timer = setInterval(run, ms) this._timer.unref?.() } diff --git a/src/lib/vm/vm-db-sync.mjs b/src/lib/vm/vm-db-sync.mjs index 484715d7c..6318a99ae 100644 --- a/src/lib/vm/vm-db-sync.mjs +++ b/src/lib/vm/vm-db-sync.mjs @@ -19,7 +19,7 @@ import fs from 'node:fs' import path from 'node:path' -import { isDbOpen, getDb } from '../db/database.mjs' +import { isDbOpen, getDb, getActivityCoordinator } from '../db/database.mjs' import { VmsRepo } from '../db/repos/vms-repo.mjs' import { WriteQueue } from '../pool/write-queue.mjs' import { setVmWriteHook, atomicWriteJson, isVmRecordFile } from './vm-file.mjs' @@ -28,6 +28,7 @@ import { reclaimAssignedVms, repairSlotSmbios, repairVmProxyCliEnabled } from '. let _repo = null let _watcher = null let _watchTimer = null +const _watchPending = new Set() let _projectRoot = null let _writes = new WriteQueue() let _paused = false @@ -97,6 +98,7 @@ export function flushVmDbSync() { export async function pauseVmDbSync() { _paused = true stopVmWatch() + while (_watchPending.size) await Promise.all([..._watchPending]) await flushVmDbSync() } @@ -240,11 +242,25 @@ export function startVmWatch(projectRoot) { if (_watchTimer) clearTimeout(_watchTimer) _watchTimer = setTimeout(async () => { _watchTimer = null - try { - reconcileDeletionsAndChanges(projectRoot).catch((error) => { + const pending = (async () => { + try { + const coordinator = isDbOpen() ? getActivityCoordinator(getDb()) : null + if (coordinator) { + await coordinator.runActivity({ kind: 'vm_file_watch' }, async () => ({ + ended: true, + value: await reconcileDeletionsAndChanges(projectRoot), + })) + } else await reconcileDeletionsAndChanges(projectRoot) + } catch (error) { console.warn(`[vm] database mirror failed: ${error.code || error.name}`) - }) - } catch {} + } + })() + _watchPending.add(pending) + try { + await pending + } finally { + _watchPending.delete(pending) + } }, 500) if (typeof _watchTimer.unref === 'function') _watchTimer.unref() }) diff --git a/test/unit/maintenance-producer-scopes.test.mjs b/test/unit/maintenance-producer-scopes.test.mjs index ef3c2a0ab..bf6af9950 100644 --- a/test/unit/maintenance-producer-scopes.test.mjs +++ b/test/unit/maintenance-producer-scopes.test.mjs @@ -92,6 +92,48 @@ test('each scheduled proxy probe receives its own live activity', async (t) => { assert.equal(probes, 2) }) +test('a draining maintenance operation prevents scheduled proxy probes from starting', async (t) => { + const fx = await fixture(t) + const operation = await fx.coordinator.beginMaintenance({ kind: 'backup' }) + const pool = new ProxyPool({ db: fx.db }) + let probes = 0 + pool.probeAll = async () => { + probes++ + return { ok: true } + } + try { + const tick = intervals(t, () => pool.startScheduler()) + await assert.rejects(pool.flush(), { code: 'database_maintenance' }) + await tick() + await pool.flush() + assert.equal(probes, 0) + } finally { + pool.stopScheduler() + await operation.close() + await fx.coordinator.cancelDrain(operation.operationId) + } +}) + +test('a failed scheduled proxy probe retains an uncertain activity', async (t) => { + const fx = await fixture(t) + const pool = new ProxyPool({ db: fx.db }) + pool.probeAll = async () => { + await fx.db.query('SELECT 1') + throw new Error('synthetic probe termination unknown') + } + intervals(t, () => pool.startScheduler()) + t.after(() => pool.stopScheduler()) + await assert.rejects(pool.flush(), /synthetic probe termination unknown/) + const rows = ( + await fx.db.pool.query( + `SELECT completed_at, uncertain_at FROM ${fx.coordinator.qualified}.activities WHERE kind='proxy_probe'`, + ) + ).rows + assert.equal(rows.length, 1) + assert.equal(rows[0].completed_at, null) + assert.ok(rows[0].uncertain_at) +}) + test('file-watcher reconciliation can persist after initialization scope closes', async (t) => { const fx = await fixture(t) fs.mkdirSync(path.join(fx.root, 'vms')) From e8591d387da5fb1631c7ef2be12c60f9bec09c3e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:38:17 +0800 Subject: [PATCH 197/473] test: reproduce unconfirmed remote container termination --- test/unit/remote-slot-stop-proof.test.mjs | 112 ++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 test/unit/remote-slot-stop-proof.test.mjs diff --git a/test/unit/remote-slot-stop-proof.test.mjs b/test/unit/remote-slot-stop-proof.test.mjs new file mode 100644 index 000000000..a0cba36cd --- /dev/null +++ b/test/unit/remote-slot-stop-proof.test.mjs @@ -0,0 +1,112 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import vmContext from 'node:vm' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' + +function fixture({ noop = false, replace = false, inspectError = false, actionError = false, running = true } = {}) { + const source = fs.readFileSync(new URL('../../src/lib/cluster/remote-slot.mjs', import.meta.url), 'utf8') + const body = source.slice(source.indexOf('export async function stopRemoteSlot(')).replaceAll('export ', '') + let present = true + let replacement = false + const events = [] + const inspect = async (_docker, target) => { + if (inspectError) throw new Error('Docker daemon unavailable') + if (target === 'kin-synthetic' && replacement) return { Id: 'replacement-id', State: { Running: true, Pid: 456 } } + if (!present) return null + return { Id: 'original-id', State: { Running: running, Restarting: false, Pid: running ? 123 : 0 }, HostConfig: {} } + } + const mutate = (id, operation) => { + events.push([operation, id]) + if (!noop) { + running = false + if (operation === 'remove' || replace) present = false + if (replace) replacement = true + } + if (actionError) throw new Error('Docker connection dropped after sending command') + } + const api = vmContext.runInNewContext(`${body}; ({stopRemoteSlot, destroyRemoteSlot})`, { + nodeSession: async () => ({ docker: {} }), + vmNodeId: () => 'node-synthetic', + containerName: () => 'kin-synthetic', + inspectContainerOrNull: inspect, + containerAction: async (_docker, id) => mutate(id, 'stop'), + removeContainer: async (_docker, id) => mutate(id, 'remove'), + isKvmRuntime: () => false, + runtimeLabelOf: () => 'docker', + releaseRemoteEgress: async () => events.push(['egress-release']), + removeRemoteSlotDir: async () => events.push(['files-remove']), + clusterManager: () => ({ dropSlotRelays: async () => events.push(['relays-remove']) }), + failure: (error) => ({ ok: false, error: error.message }), + }) + return { ...api, events, vm: { id: 'vm-synthetic', runtime: {} } } +} + +test('remote stop acknowledgement does not prove a still running container ended', async () => { + const f = fixture({ noop: true }) + const result = await f.stopRemoteSlot(f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.notEqual(f.vm.runtime.stopped, true) +}) + +test('remote removal acknowledgement cannot delete slot files while the container remains', async () => { + const f = fixture({ noop: true }) + const result = await f.destroyRemoteSlot(f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal( + f.events.some(([name]) => name === 'files-remove'), + false, + ) +}) + +test('remote stop and removal address the immutable inspected container ID', async () => { + for (const action of ['stopRemoteSlot', 'destroyRemoteSlot']) { + const f = fixture() + assert.equal((await f[action](f.vm)).ok, true) + assert.equal(f.events[0][1], 'original-id') + } +}) + +test('remote container replacement leaves the mutation unknown and slot files intact', async () => { + for (const action of ['stopRemoteSlot', 'destroyRemoteSlot']) { + const f = fixture({ replace: true }) + const result = await f[action](f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal( + f.events.some(([name]) => name === 'files-remove'), + false, + ) + } +}) + +test('remote Docker inspect failure retains uncertainty', async () => { + const f = fixture({ inspectError: true }) + const result = await f.destroyRemoteSlot(f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.deepEqual(f.events, []) +}) + +test('remote command error can complete only after an independent terminal observation', async () => { + for (const action of ['stopRemoteSlot', 'destroyRemoteSlot']) { + const f = fixture({ actionError: true }) + assert.equal((await f[action](f.vm)).ok, true) + } +}) + +test('remote already stopped container remains successful', async () => { + const f = fixture({ running: false }) + assert.equal((await f.stopRemoteSlot(f.vm)).ok, true) +}) + +test('unconfirmed remote stop keeps its persistent VM operation active', async () => { + const f = fixture({ noop: true }) + const operations = new SharedVmOperations(getDb()) + const result = await operations.run(f.vm.id, () => f.stopRemoteSlot(f.vm)) + assert.equal(result.ok, false) + assert.equal((await operations.tryAcquire({ vmId: f.vm.id })).ok, false) +}) From a05fdcc75bb7231af0fe7c40e33b6c42d4de9bf9 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:38:21 +0800 Subject: [PATCH 198/473] docs(kubernetes): record singleton deployment and later execution fixes --- .../2026-10-21-kubernetes-control-plane.md | 10 +- ...2026-10-21-kubernetes-execution-results.md | 4 +- ...-21-kubernetes-shared-snapshots-results.md | 28 +++- .../shared-snapshots/verification.json | 6 +- .../singleton-6732a864/containerd-image.txt | 2 + .../singleton-6732a864/image-id.txt | 1 + .../singleton-6732a864/image-manifest.json | 131 ++++++++++++++++++ .../kernel-cancel-counterexample.json | 109 +++++++++++++++ .../singleton-6732a864/later-lifecycle.json | 69 +++++++++ .../kubernetes/singleton-6732a864/smoke.jsonl | 3 + .../singleton-6732a864/verification.json | 111 +++++++++++++++ 11 files changed, 463 insertions(+), 11 deletions(-) create mode 100644 docs/plans/evidence/kubernetes/singleton-6732a864/containerd-image.txt create mode 100644 docs/plans/evidence/kubernetes/singleton-6732a864/image-id.txt create mode 100644 docs/plans/evidence/kubernetes/singleton-6732a864/image-manifest.json create mode 100644 docs/plans/evidence/kubernetes/singleton-6732a864/kernel-cancel-counterexample.json create mode 100644 docs/plans/evidence/kubernetes/singleton-6732a864/later-lifecycle.json create mode 100644 docs/plans/evidence/kubernetes/singleton-6732a864/smoke.jsonl create mode 100644 docs/plans/evidence/kubernetes/singleton-6732a864/verification.json diff --git a/docs/plans/2026-10-21-kubernetes-control-plane.md b/docs/plans/2026-10-21-kubernetes-control-plane.md index c7cfc1829..600ca138b 100644 --- a/docs/plans/2026-10-21-kubernetes-control-plane.md +++ b/docs/plans/2026-10-21-kubernetes-control-plane.md @@ -24,7 +24,7 @@ ## 为什么暂不开放多副本推理 -第一阶段发现的 key/user/账号/VM 并发与 RPM、设备座位、同会话顺序、Sticky 和缓存覆盖问题，已有后续共享状态实现。当前仍未完成执行端终止证明与自动接管、后台任务生产接线、一次性认证状态、全局维护、VM/凭据文件共享归属及所有管理入口的协调。共享熔断和 OAuth single-flight 也需继续核实。 +第一阶段发现的 key/user/账号/VM 并发与 RPM、设备座位、同会话顺序、Sticky 和缓存覆盖问题，已有后续共享状态实现。一次性认证状态、后台任务和全局维护已有后续源码及定向验证，但这些增量尚未完成新镜像部署验收。当前仍未完成执行端终止证明与自动接管、VM/凭据文件共享归属及所有管理入口的协调。共享熔断和 OAuth single-flight 也需继续核实。 更关键的是，远端执行取消目前是 best-effort。控制面失联后，不能仅因租约过期就重新分配仍可能执行中的账号/VM 容量。安全的多活需要执行端 fencing 或可确认终止的 orphan 处理协议。活凭据、VM 文件和 Pod 本地 socket 的归属也必须先明确。 @@ -62,7 +62,7 @@ 3. 协调后台任务、全局维护、备份恢复和凭据刷新；完成 routing/VM/凭据文件的共享存储与更新协议。减少每请求全量设置读取的成本。 4. 全部条件具备后再允许两个活动副本，补充 HPA、PodDisruptionBudget 和多节点部署，并验证并发、滚动更新、缩容与故障接管。 -本轮运行证据及未验证项将在增量验证报告中记录。第一阶段的镜像 digest 与 k3s 结果只覆盖当时源码，不能用于证明本轮新代码已部署。 +本轮运行证据及未验证项在增量验证报告中记录。`6732a864` 已有候选镜像和单节点 k3s 的 12 项单活动控制面检查；该镜像仅覆盖该固定源码，不能证明之后的认证、后台任务和生命周期代码已部署。 ## 共享执行准入增量（进行中） @@ -77,4 +77,8 @@ 固定源码 `6732a8648a4a57c6483abc7f685fce4d8cb28946` 已在独立检出完成全量检查：unit 3658 pass、7 fail、10 skip；E2E 104 pass、17 fail、1 skip。全部失败名称及断言文本与上一轮记录一致。AccountQuota/ProxyPool/SchedulingRuntime 字段更新、Sticky PG 条件更新与墓碑、共享同会话 FIFO、Codex 执行准入、PG 路由 revision 及准入事务校验均已接入。后续 Search 和面板路由状态接口另有定向验证。 -结果与范围见 [共享状态验证报告](2026-10-21-kubernetes-shared-snapshots-results.md)。控制面仍为单活动副本。后台任务接线、一次性认证状态、全局维护、所有 VM 管理入口、执行端 fencing 和多节点运行验收继续进行；没有因此关闭 #648。 +该提交另已构建并在单节点 amd64 k3s 验证 12 项单活动控制面检查，命名空间已清理。OCI manifest 为 `sha256:db695e30d1ae3165bc88003f134584761db40dba09f1166e364b7dd9af683d05`，Pod imageID 对应 manifest config `sha256:e1b150ae36e2a80e785eb62b899aa144e4ad05692da05290e757489727451118`。见 [部署证据](evidence/kubernetes/singleton-6732a864/verification.json)。 + +其后的 `b32641b8` 保留取消或内部 provider/transport 错误的未知执行，即使 kernel 返回完整 HTTP；`e854273a` 使 API proxy 暴露 body 拷贝中断。API endpoint 及请求 activity hook 在 `08998f40`、`bdda2026` 有定向源码验证，后者固定归档 75/75 通过。这些修正以及新的认证、后台任务/全局维护代码没有部署到上述镜像。 + +结果与范围见 [共享状态验证报告](2026-10-21-kubernetes-shared-snapshots-results.md)。控制面仍为单活动副本。所有 VM 管理入口协调、执行端 fencing 和多节点运行验收继续进行；单实例检查不是 HA，不满足 #648 的完成条件。 diff --git a/docs/plans/2026-10-21-kubernetes-execution-results.md b/docs/plans/2026-10-21-kubernetes-execution-results.md index 03abcca4c..9b15743cc 100644 --- a/docs/plans/2026-10-21-kubernetes-execution-results.md +++ b/docs/plans/2026-10-21-kubernetes-execution-results.md @@ -39,7 +39,7 @@ 17 项 E2E 失败的断言文本与上一轮已核实基线记录相同，本轮未重新运行原始提交的整套 E2E。例如 Responses 实际返回 400 `protocol_not_allowed`，预期 200；classifier 的 thinking 实际多出 `display: summarized`。未将这些失败改成通过或跳过。 -结构化结果与日志摘要见 [验证元数据](evidence/kubernetes/execution-state/verification.json) 和 [原始 unit 复核](evidence/kubernetes/execution-state/baseline-unit-recheck.json)。上一轮原始 E2E 对照见 [既有基线记录](evidence/kubernetes/shared-state/baseline-e2e.json)。本轮没有重新部署容器，验证 SHA 仅标识被测试的源码。 +结构化结果与日志摘要见 [验证元数据](evidence/kubernetes/execution-state/verification.json) 和 [原始 unit 复核](evidence/kubernetes/execution-state/baseline-unit-recheck.json)。上一轮原始 E2E 对照见 [既有基线记录](evidence/kubernetes/shared-state/baseline-e2e.json)。`fb90450d` 检查点当时没有重新部署；后续 `6732a864` 已完成单实例镜像部署，范围见 [后续部署记录](2026-10-21-kubernetes-shared-snapshots-results.md)。 ## 尚未完成 @@ -50,4 +50,4 @@ - routing、VM/凭据文件共享归属、后台任务唯一执行、全局维护及备份恢复。 - 两个不同 Kubernetes 节点上的活动副本、HPA、PodDisruptionBudget、滚动更新、缩容和故障接管。 -第一阶段镜像与 k3s 证据不覆盖本轮新增源码。本轮没有重新部署，不能用旧镜像 digest 证明当前代码生效。Draft PR #651 的远端分支更新仍等待用户确认 push。 +第一阶段镜像与 k3s 证据只覆盖当时源码。后续 `6732a864` 镜像通过单实例部署检查，但不包含再后的认证、后台任务和生命周期修正；不能用其 digest 证明这些新代码生效。Draft PR #651 的远端分支更新仍等待用户确认 push。 diff --git a/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md b/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md index 4241c1375..6202f4580 100644 --- a/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md +++ b/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md @@ -1,6 +1,6 @@ # Kubernetes 共享状态与请求接线验证（#648，进行中） -固定源码提交：`6732a8648a4a57c6483abc7f685fce4d8cb28946`。本轮在独立检出执行全量检查，未部署当前容器。控制面仍使用独占保护和单活动副本；这些检查尚未满足 #648 的多节点扩缩容与异常退出验收。 +固定源码提交：`6732a8648a4a57c6483abc7f685fce4d8cb28946`。本轮在独立检出执行全量检查，并以该提交构建候选镜像，在单节点 k3s 完成 12 项单活动控制面检查。控制面仍使用独占保护和单活动副本；这些检查尚未满足 #648 的多节点扩缩容与异常退出验收。后续认证、后台任务及执行生命周期增量没有部署到该镜像。 ## 已实现的增量 @@ -28,6 +28,19 @@ 本次没有重新执行原始 `eee8f6e` 的整套测试。7 项 unit 在原始源码上的复核、17 项 E2E 的既有对照来源，见 [上一轮共享执行报告](2026-10-21-kubernetes-execution-results.md)。结构化统计及错误摘要哈希见 [本轮证据](evidence/kubernetes/shared-snapshots/verification.json)。完整测试日志保留于任务临时目录，证据包含其 SHA-256。 +## `6732a864` 的实际单实例部署 + +候选镜像为 `vm2api:kubernetes-6732a864`，运行于本机单节点 amd64 k3s。临时 Dockerfile 仅在开头增加 `ARG BUILDPLATFORM=linux/amd64`，供 legacy builder 解析原有 `FROM --platform=$BUILDPLATFORM`，没有加入功能补丁。测试使用独立 PostgreSQL、合成上游和任务命名空间 `vm2api-k8s-649-77d914a4a9`；该命名空间已清理。 + +- OCI manifest：`sha256:db695e30d1ae3165bc88003f134584761db40dba09f1166e364b7dd9af683d05`。 +- manifest config 与实际 Pod imageID：`sha256:e1b150ae36e2a80e785eb62b899aa144e4ad05692da05290e757489727451118`。 + +Pod imageID 对应 manifest 中的 config digest，不能与 OCI manifest digest 混用。构建制品、containerd 映射和实际 smoke 输出见 [部署证据](evidence/kubernetes/singleton-6732a864/verification.json)。 + +12 项检查全部通过：Pod 就绪、Service DNS/ClusterIP、探针与控制台、Cookie 登录与 PG 指标、API key 创建、托管 key 推理、真实 Go API kernel 的 JSON/SSE、两条请求日志、第二控制面拒绝、Recreate 重启、PG 会话与 key 保留、PVC 文件保留。实际 SSE 响应头在 29.0ms 到达，两段正文分别在 29.1ms 和 1030.0ms 到达，HTTP 在 1216.5ms 结束。本次唯一文本为 `kubernetes-proof-77d914a4a9f4`。 + +这些结果验证单活动控制面的实际容器行为。它们没有验证两个活动副本、跨节点执行、自动扩缩容或故障接管，也不构成高可用（HA）验收。 + ## 后续提交的定向检查 这些结果没有替代后续源码的全量回归： @@ -38,16 +51,23 @@ | `06d841cd5fb5d81366cf0a7684b408fcd7368ac3` | 配置 revision、准入事务和锁顺序：44/44 通过 | | `c5f6ff3d1e3728c74ceb630929bc876298322a2a` | Codex 与 Search 生命周期：145/145 通过 | | `97b4a087ae486814ffdc912f7b662fdeb2e387ff` | 路由面板接口与 VM 变更：34/34 通过 | +| `b32641b84aebb3f4beac656e09c59beee8c78270` | kernel 未确认错误、现有 payload 取消与 Claude/Codex 共享占位：228/228 通过 | +| `08998f406eb7a2bf7ac3512d4b20895879cc2829` | API endpoint 执行 hook、终态与 PG job：72/72 通过 | +| `bdda2026dca0ed5905c70514bc3fad94e727a53e` | 固定源码归档上的请求 activity hook 接线：75/75 通过 | 前两项提交位于全量检查点之前。后两项在其后。Search 直接 HTTP 请求只有在响应体读取完成后才结束作业；401/403 重发前完成旧作业并使用新 ID。响应体截断、取消、超时及 PG 故障保留容量。Search 不增加原协议没有定义的同会话队列。 面板路由更新显式返回已提交 revision 和 application 状态。VM/kernel 投影未执行时返回 pending。PG 提交失败不回退到可能过期的本地配置，也不在提交前切换执行端引擎。 +后续发现普通 EOF 假设的例外：现有 `kin-kernel`/`cli-node` 在合成上游保持 SSE 时，外部 cancel 返回 200 `cancelled:true`，原请求随后可返回完整 HTTP 502 `worker_error/provider_error: client disconnected`。`b32641b8` 对这类取消、内部 transport/timeout 和无法确认的 kernel 错误保留 `ended=false`，停止共享执行重试及容量释放。普通完整响应和明确供应商拒绝保留原有 EOF 语义。`ready_slots` 是汇总状态，不含 request ID 或 epoch，不能证明单个 job 已结束。 + +`e854273a5d7016c160a24237ef9aeac3df0cf988` 修正 Go API proxy 在 `io.Copy` 失败时正常结束响应、隐瞒上游 body 截断的问题。`08998f40` 和 `bdda2026` 接入 API endpoint 及请求 activity 的发送前登记和读取结束确认。取消、超时、截断、`upstream_transport` 或 SSE 缺终态不释放未知执行。源码验证与现有 payload 反例见 [后续生命周期证据](evidence/kubernetes/singleton-6732a864/later-lifecycle.json)。这些修正及新的认证、后台任务/全局维护代码没有部署到 `6732a864` 候选镜像。 + ## 未完成和未验证 -- 后台任务共享执行的生产接线、跨副本一次性 OAuth/终端 ticket 状态仍在后续增量中。 -- 全局维护与备份恢复、VM/凭据文件的权威来源、其他 VM 管理入口、共享熔断和跨 VM 组织兑换等尚未完成。 +- 后台任务共享执行、跨副本一次性 OAuth/终端 ticket、全局维护和请求 activity 的后续源码增量，尚未完成新镜像的部署验收。 +- VM/凭据文件的权威来源、其他 VM 管理入口、共享熔断和跨 VM 组织兑换等仍需继续核实和实施。 - VM/kernel 远端配置投影为 pending。PG 配置提交不证明执行端已更新。 - 当前 HTTP/SSE 结束标记依赖正常执行端协议。执行端 epoch 校验、旧作业终止证明和 owner 死亡后的自动恢复尚未实现。未知占位没有自动回收。 -- 两个不同 Kubernetes 节点上的活动副本、HPA、PodDisruptionBudget、滚动更新、缩容和异常退出接管尚未验证。旧镜像和单节点 k3s 证据不覆盖本轮代码。 +- 两个不同 Kubernetes 节点上的活动副本、HPA、PodDisruptionBudget、滚动更新、缩容和异常退出接管尚未验证。单节点 `6732a864` 镜像的 12 项检查不覆盖其后的源码增量。 - Draft PR #651 的已发布分支仍是早期部署基础。后续本地增量尚未 push；按用户规则等待 push 确认。 diff --git a/docs/plans/evidence/kubernetes/shared-snapshots/verification.json b/docs/plans/evidence/kubernetes/shared-snapshots/verification.json index 4c198382c..84f24c8e8 100644 --- a/docs/plans/evidence/kubernetes/shared-snapshots/verification.json +++ b/docs/plans/evidence/kubernetes/shared-snapshots/verification.json @@ -238,7 +238,7 @@ "Full suites are not all passing; all 7 unit and 17 E2E error texts match comparison records.", "Later targeted checks are not a full-suite verification of later commits.", "Kubernetes control plane remains single active replica with exclusive guard.", - "No current container deployment, multi-node inference, executor fencing, or automatic orphan recovery was verified.", + "The 6732a864 image passed 12 singleton Kubernetes checks; later lifecycle changes, multi-node inference, executor fencing, and automatic orphan recovery were not deployment-verified.", "Background task coordination and one-time OAuth/ticket state are separate ongoing increments.", "VM files/credentials and global backup restore maintenance are not yet coordinated across all entry points.", "Remote VM/kernel routing projection is pending; configuration commit does not prove it was applied." @@ -251,5 +251,7 @@ "diff_check_exit_code": 0, "diff_base": "fb90450d06fd3aaaee8a0a4b52aca13d2f675431", "log_sha256": "6f4fbd68da9299121172e18f8c28ee6f3bbcd5d47b8586e291fd8966d084ed48" - } + }, + "runtime_evidence": "../singleton-6732a864/verification.json", + "later_lifecycle_evidence": "../singleton-6732a864/later-lifecycle.json" } diff --git a/docs/plans/evidence/kubernetes/singleton-6732a864/containerd-image.txt b/docs/plans/evidence/kubernetes/singleton-6732a864/containerd-image.txt new file mode 100644 index 000000000..47b17680e --- /dev/null +++ b/docs/plans/evidence/kubernetes/singleton-6732a864/containerd-image.txt @@ -0,0 +1,2 @@ +REF TYPE DIGEST SIZE PLATFORMS LABELS +docker.io/library/vm2api:kubernetes-6732a864 application/vnd.oci.image.manifest.v1+json sha256:db695e30d1ae3165bc88003f134584761db40dba09f1166e364b7dd9af683d05 260.2 MiB linux/amd64 io.cri-containerd.image=managed diff --git a/docs/plans/evidence/kubernetes/singleton-6732a864/image-id.txt b/docs/plans/evidence/kubernetes/singleton-6732a864/image-id.txt new file mode 100644 index 000000000..efb6601da --- /dev/null +++ b/docs/plans/evidence/kubernetes/singleton-6732a864/image-id.txt @@ -0,0 +1 @@ +sha256:db695e30d1ae3165bc88003f134584761db40dba09f1166e364b7dd9af683d05 \ No newline at end of file diff --git a/docs/plans/evidence/kubernetes/singleton-6732a864/image-manifest.json b/docs/plans/evidence/kubernetes/singleton-6732a864/image-manifest.json new file mode 100644 index 000000000..13e9653fb --- /dev/null +++ b/docs/plans/evidence/kubernetes/singleton-6732a864/image-manifest.json @@ -0,0 +1,131 @@ +{ + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:e1b150ae36e2a80e785eb62b899aa144e4ad05692da05290e757489727451118", + "size": 13680 + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:a8ac7f6c67abc236e4c745052c404112b8fab6fe8ac3a329d1ef3b867ad67c71", + "size": 28232655 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:cea7eb9bbe80eabf8356efb28b3796ab904f36a32049aeb6b435920ca93f9285", + "size": 3314 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:51c53f95c2cae3d00dfc0350297c92dea913fe129ec38b8deb8313e5f3e0297d", + "size": 49938210 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:3cddab114edadc7cf5a7b0b591e59f6312aade3dc4fb68e88aacca5eb1cc1c03", + "size": 1712979 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:0282d618b52cd2eb8fcf041e780cd66b15e7ebae519a4054c5d293eae6b6d34f", + "size": 447 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:36c8b0ef54e52fbae3e76c436bb5415c2b84f77e321e76d0fc7c86152435687f", + "size": 19296556 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:4d6ec1ce9fe3b62293b11d35c3ac21003ea957e00adb6e119c32d5be8b0698aa", + "size": 18319279 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:52ba52ab62570496c1bdbdce8211a96acf3e422d3d5a770853a6e8ed0b7b1625", + "size": 19121283 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:f4847c70b391df969e527f516fc518518ab62f8b403635871d33fa8d6081a6e2", + "size": 113 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:1df9d0cd21a692181e37e9e86b49806aff2b35d3f9602060b04089407111a2f4", + "size": 6266 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:ecac213afa88775a79854aabb7beed041c02414e5ab397a4a4dfb1af97345aa9", + "size": 13018940 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:5ceca8221d8435f95e97f527c70578f4f77b2215102a650ced9b937e9a41d78f", + "size": 1016773 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:4343e178427953941ad17296c97535644fcd7925e65d054017b560c3ca916bec", + "size": 39656 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:d3c3df7a2f366aff050a1fda8d864cf3070114f31ca1b86c4c055a8635aa68cf", + "size": 105021 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:2fa5f69ffa0d80e449f318ddadf16bb56c72ac887fd4e5db58fa536cf5924f0b", + "size": 1653 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:ef3a0bede2c5e76dce83a350e98181ebaf19a0cc95a1b59d81f0c19059bae5a7", + "size": 12331 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:28794b297f2300ecd7677775ea096f99286536f46469091b841236b5ec0d7905", + "size": 2107736 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:0182f5fb5a94f6e862aa3e9f898777fecdff637452feabffb5da10c5767e7df0", + "size": 26478259 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:822fca7a7f6458a7067feed808f3020d773289081cb60e4f220c75b938bead92", + "size": 23967920 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:0d3a52d181e31aa7f19722f15b8b51e025656e5b6a49105250d6f0e3815b5a1f", + "size": 723 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:a9e7bc104fd85600be4afb454308a6381744bd3232f5f0edde0f70595682fe32", + "size": 65868257 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:61e59163742cf0e07f04311a92c2406e9dcdceee6b2339dbcbc353542236ddad", + "size": 3579252 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:a027616dd73f43b09d21bd02e3cb28939d664501a8eae63351cd3effcf68e203", + "size": 2417 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:e21cdcf978b3ffa830b528be0b3da7c6c4494c7ac86f475461f64c2c4c3b3fe8", + "size": 6214 + } + ] +} \ No newline at end of file diff --git a/docs/plans/evidence/kubernetes/singleton-6732a864/kernel-cancel-counterexample.json b/docs/plans/evidence/kubernetes/singleton-6732a864/kernel-cancel-counterexample.json new file mode 100644 index 000000000..a476e7e56 --- /dev/null +++ b/docs/plans/evidence/kubernetes/singleton-6732a864/kernel-cancel-counterexample.json @@ -0,0 +1,109 @@ +[ + { + "phase": "before", + "endpoint": "/internal/health", + "status": 200, + "body": { + "classifier_request_context": true, + "cli_pid": 2436739, + "cli_restarts": 0, + "cli_version": "", + "closed_slots": 0, + "credential_state": "fresh", + "delivery_mode": "realtime", + "engine": "rust", + "healthy": true, + "ok": true, + "provider": "local_cli", + "proxy_configured": false, + "proxy_required": false, + "ready_slots": 2, + "recovering": false, + "runtime_kind": "docker", + "unhealthy_reason": null, + "uptime_seconds": 0, + "version": "0.1.0", + "vm_id": "vm-capability", + "worker_version": "0.1.0" + } + }, + { + "phase": "during", + "endpoint": "/internal/health", + "status": 200, + "body": { + "classifier_request_context": true, + "cli_pid": 2436739, + "cli_restarts": 0, + "cli_version": "", + "closed_slots": 0, + "credential_state": "fresh", + "delivery_mode": "realtime", + "engine": "rust", + "healthy": true, + "ok": true, + "provider": "local_cli", + "proxy_configured": false, + "proxy_required": false, + "ready_slots": 1, + "recovering": false, + "runtime_kind": "docker", + "unhealthy_reason": null, + "uptime_seconds": 1, + "version": "0.1.0", + "vm_id": "vm-capability", + "worker_version": "0.1.0" + } + }, + { + "phase": "cancel", + "endpoint": "/internal/v1/cancel", + "status": 200, + "body": { + "cancelled": true, + "ok": true + } + }, + { + "phase": "immediately-after-cancel", + "endpoint": "/internal/health", + "status": 200, + "body": { + "classifier_request_context": true, + "cli_pid": 2436739, + "cli_restarts": 0, + "cli_version": "", + "closed_slots": 0, + "credential_state": "fresh", + "delivery_mode": "realtime", + "engine": "rust", + "healthy": true, + "ok": true, + "provider": "local_cli", + "proxy_configured": false, + "proxy_required": false, + "ready_slots": 1, + "recovering": false, + "runtime_kind": "docker", + "unhealthy_reason": null, + "uptime_seconds": 1, + "version": "0.1.0", + "vm_id": "vm-capability", + "worker_version": "0.1.0" + } + }, + { + "phase": "job-result", + "endpoint": "/internal/v1/messages", + "status": 502, + "body": { + "ok": false, + "error": { + "type": "worker_error", + "code": "provider_error", + "message": "provider error: client disconnected", + "status": 502 + } + } + } +] diff --git a/docs/plans/evidence/kubernetes/singleton-6732a864/later-lifecycle.json b/docs/plans/evidence/kubernetes/singleton-6732a864/later-lifecycle.json new file mode 100644 index 000000000..6078643c8 --- /dev/null +++ b/docs/plans/evidence/kubernetes/singleton-6732a864/later-lifecycle.json @@ -0,0 +1,69 @@ +{ + "scope": "later source validation; none of these changes was included in the deployed 6732a864 image", + "directed_checks": [ + { + "commit": "b32641b84aebb3f4beac656e09c59beee8c78270", + "scope": "kernel internal errors, existing payload cancellation, Claude/Codex shared capacity", + "totals": { + "tests": 228, + "pass": 228, + "fail": 0, + "skipped": 0, + "cancelled": 0 + }, + "log_sha256": "5256f142d405fd7dca2f71db408077d7d201dbcd8fb36ad19592479687347317" + }, + { + "commit": "08998f406eb7a2bf7ac3512d4b20895879cc2829", + "scope": "API endpoint start/end hooks and PostgreSQL execution ledger", + "totals": { + "tests": 72, + "pass": 72, + "fail": 0, + "skipped": 0, + "cancelled": 0 + }, + "log_sha256": "d7e9fe905a756c78478e189c465bbf791d742a7b0e647bdec1f14a0c1082d827" + }, + { + "commit": "bdda2026dca0ed5905c70514bc3fad94e727a53e", + "scope": "request activities connected to actual inference execution; fixed source archive", + "totals": { + "tests": 75, + "pass": 75, + "fail": 0, + "skipped": 0, + "cancelled": 0 + }, + "log_sha256": "1dead2fb600b3fff9db28469c887cb43e65ab1e3cdc4b7521bd1bf45f2bca6e0" + } + ], + "go_api_body_copy_fix": { + "commit": "e854273a5d7016c160a24237ef9aeac3df0cf988", + "change": "Abort the downstream HTTP response when io.Copy of an upstream body fails; Go package checks reported passing by task owner.", + "runtime_deployment_verified": false + }, + "kernel_counterexample": { + "artifact": "kernel-cancel-counterexample.json", + "sha256": "7dc1df4660af181540d2e317f297db87109c1608f7e38ba7bd85eb7f18ac17b1", + "payload_sha256": { + "bin/kin-kernel": "8f1347ccffa6ac66d2a6625fded59ea2b9d46c3499f4f192d77cc0510a636a68", + "share/wrap-cli/cli-node": "b5054cb8494670354e61f3e0a0f7b2c3b768832c841140a4e22407fec24ab009" + }, + "observation": "cancel returned 200/cancelled:true and the original request returned full HTTP 502 worker_error/provider_error client disconnected; aggregate ready_slots was still 1.", + "limitation": "ready_slots has no request ID or epoch and cannot prove an individual job ended.", + "before": { + "source_head": "f081b9798d72a45e762ef880af65dc2298c73654", + "test_changes_committed_as": "09166c4f", + "tests": 62, + "pass": 41, + "fail": 21, + "log_sha256": "dfe8c2240556f0f6a6f00138d26e5067db9d90b19fa4c31b0df5c3485be189db" + } + }, + "remaining_limitations": [ + "No independent per-job fencing or unknown-job recovery proof.", + "No deployment validation of later auth/background/API lifecycle changes.", + "Issue 648 remains incomplete; singleton deployment is not HA." + ] +} diff --git a/docs/plans/evidence/kubernetes/singleton-6732a864/smoke.jsonl b/docs/plans/evidence/kubernetes/singleton-6732a864/smoke.jsonl new file mode 100644 index 000000000..057191b8b --- /dev/null +++ b/docs/plans/evidence/kubernetes/singleton-6732a864/smoke.jsonl @@ -0,0 +1,3 @@ +{"event": "pod-ready", "namespace": "vm2api-k8s-649-77d914a4a9", "image_id": "sha256:e1b150ae36e2a80e785eb62b899aa144e4ad05692da05290e757489727451118"} +{"event": "sse-timing", "headers_ms": 29.0, "headers": {"content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache, no-transform", "transfer-encoding": "chunked", "x-accel-buffering": "no"}, "frames": [{"type": "message_start", "arrived_ms": 29.1}, {"type": "content_block_start", "arrived_ms": 29.1}, {"type": "content_block_delta", "arrived_ms": 29.1}, {"type": "content_block_delta", "arrived_ms": 1030.0}, {"type": "content_block_stop", "arrived_ms": 1030.0}, {"type": "message_delta", "arrived_ms": 1030.1}, {"type": "message_stop", "arrived_ms": 1030.1}], "deltas": [{"text": "kubernetes-pro", "arrived_ms": 29.1}, {"text": "of-77d914a4a9f4", "arrived_ms": 1030.0}], "ended_ms": 1216.5} +{"ok": true, "namespace": "vm2api-k8s-649-77d914a4a9", "image_id": "sha256:e1b150ae36e2a80e785eb62b899aa144e4ad05692da05290e757489727451118", "proof": "kubernetes-proof-77d914a4a9f4", "checks": ["pod-ready", "service-dns-and-clusterip", "health-ready-console", "panel-cookie-login-and-postgres-metrics", "api-key-create", "managed-api-key-inference", "go-api-kernel-json-and-sse", "two-request-logs", "second-control-plane-rejected", "recreate-restart", "postgres-session-and-api-key-persist", "pvc-persists"]} diff --git a/docs/plans/evidence/kubernetes/singleton-6732a864/verification.json b/docs/plans/evidence/kubernetes/singleton-6732a864/verification.json new file mode 100644 index 000000000..bbadbfc56 --- /dev/null +++ b/docs/plans/evidence/kubernetes/singleton-6732a864/verification.json @@ -0,0 +1,111 @@ +{ + "source_commit": "6732a8648a4a57c6483abc7f685fce4d8cb28946", + "scope": "single active control plane on single-node amd64 k3s; not HA or issue 648 acceptance", + "image_reference": "docker.io/library/vm2api:kubernetes-6732a864", + "oci_manifest_digest": "sha256:db695e30d1ae3165bc88003f134584761db40dba09f1166e364b7dd9af683d05", + "config_digest": "sha256:e1b150ae36e2a80e785eb62b899aa144e4ad05692da05290e757489727451118", + "pod_image_id": "sha256:e1b150ae36e2a80e785eb62b899aa144e4ad05692da05290e757489727451118", + "namespace": "vm2api-k8s-649-77d914a4a9", + "namespace_cleanup": { + "state": "deleted", + "source": "task owner confirmation" + }, + "build_adjustment": "Temporary Dockerfile only prepended ARG BUILDPLATFORM=linux/amd64 for the legacy builder; no feature patch.", + "checks": [ + "pod-ready", + "service-dns-and-clusterip", + "health-ready-console", + "panel-cookie-login-and-postgres-metrics", + "api-key-create", + "managed-api-key-inference", + "go-api-kernel-json-and-sse", + "two-request-logs", + "second-control-plane-rejected", + "recreate-restart", + "postgres-session-and-api-key-persist", + "pvc-persists" + ], + "checks_passed": 12, + "smoke_exit_code": 0, + "sse_timing": { + "event": "sse-timing", + "headers_ms": 29.0, + "headers": { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache, no-transform", + "transfer-encoding": "chunked", + "x-accel-buffering": "no" + }, + "frames": [ + { + "type": "message_start", + "arrived_ms": 29.1 + }, + { + "type": "content_block_start", + "arrived_ms": 29.1 + }, + { + "type": "content_block_delta", + "arrived_ms": 29.1 + }, + { + "type": "content_block_delta", + "arrived_ms": 1030.0 + }, + { + "type": "content_block_stop", + "arrived_ms": 1030.0 + }, + { + "type": "message_delta", + "arrived_ms": 1030.1 + }, + { + "type": "message_stop", + "arrived_ms": 1030.1 + } + ], + "deltas": [ + { + "text": "kubernetes-pro", + "arrived_ms": 29.1 + }, + { + "text": "of-77d914a4a9f4", + "arrived_ms": 1030.0 + } + ], + "ended_ms": 1216.5 + }, + "artifacts": { + "smoke.jsonl": { + "sha256": "d29c550db3419dae39bf902d743f975b8590d5f6e23b1b748543e465b6425b5d" + }, + "image-manifest.json": { + "sha256": "db695e30d1ae3165bc88003f134584761db40dba09f1166e364b7dd9af683d05" + }, + "containerd-image.txt": { + "sha256": "e5a5d2392522e2b7ba523eb8db249eceae2a39ae54f57882a55cc3b92acf8551" + }, + "image-id.txt": { + "sha256": "63a179914da1d2b7e7d97c9bb03570b29a0fad3d1a9a0af680e8a9918894a517" + } + }, + "build_log_sha256": "31d12b596e274c70bf0a7775b7130b3ee98eef53ca480a1beaac91e67847033a", + "not_deployed": [ + "later PKCE and shell-ticket persistence", + "later background-task and global-maintenance integration", + "b32641b8 kernel execution EOF correction", + "e854273a Go API body-copy interruption correction", + "08998f40 API lifecycle hooks", + "bdda2026 request activity hooks" + ], + "not_verified": [ + "two active controllers on different Kubernetes nodes", + "HPA, rolling updates, scale-down and failover", + "node-side fencing and immutable remote execution termination proof", + "real remote VM/provider inference", + "production load and cross-node storage" + ] +} From ab47c57618a93350604f9b9084a2b92dca9b4692 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:39:11 +0800 Subject: [PATCH 199/473] test: reproduce cluster callbacks using closed startup activities --- test/unit/cluster-maintenance-scopes.test.mjs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 test/unit/cluster-maintenance-scopes.test.mjs diff --git a/test/unit/cluster-maintenance-scopes.test.mjs b/test/unit/cluster-maintenance-scopes.test.mjs new file mode 100644 index 000000000..750fa27ee --- /dev/null +++ b/test/unit/cluster-maintenance-scopes.test.mjs @@ -0,0 +1,76 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' +import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' +import { SshLink } from '../../src/lib/cluster/ssh-link.mjs' +import { SocketRelay } from '../../src/lib/cluster/socket-relay.mjs' + +async function fixture(t) { + const db = getDb() + const calls = [] + const sql = async (name) => { + await db.query('SELECT 1') + calls.push(name) + } + const manager = new ClusterManager({ + repo: { + db, + list: async () => [{ id: 'node-synthetic', host: 'synthetic.invalid', port: 22, username: 'synthetic' }], + getSecrets: async () => { + await sql('target') + return { password: 'synthetic' } + }, + }, + dataDir: '/tmp', + logger: { log() {}, warn() {} }, + onLinkState: async () => sql('state'), + onNodesChanged: async () => sql('nodes'), + }) + manager.remoteHost = async () => sql('host') + manager._checkHealth = async () => sql('health') + manager._ensureGate = async () => sql('gate') + t.mock.method(SshLink.prototype, 'start', () => {}) + t.mock.method(SshLink.prototype, 'stop', () => {}) + t.mock.method(SocketRelay.prototype, 'start', async () => {}) + t.mock.method(SocketRelay.prototype, 'stop', async () => {}) + let tick + const interval = t.mock.method(globalThis, 'setInterval', (callback) => { + tick = callback + return { unref() {} } + }) + try { + await withDatabaseActivities([db], () => manager.start()) + } finally { + interval.mock.restore() + } + t.after(() => manager.stop()) + return { manager, calls, tick, link: manager.links.get('node-synthetic') } +} + +test('cluster health timers run in independent activities after startup', async (t) => { + const fx = await fixture(t) + await fx.tick() + await fx.manager.stop() + assert.deepEqual(fx.calls, ['health']) +}) + +test('SSH reconnect target reads enter a live activity', async (t) => { + const fx = await fixture(t) + const target = await fx.link.resolveTarget() + assert.equal(target.host, 'synthetic.invalid') + assert.deepEqual(fx.calls, ['target']) +}) + +test('link ready waits for host, health, gate and state callback activities', async (t) => { + const fx = await fixture(t) + fx.link.emit('state', 'ready') + await fx.manager.stop() + assert.deepEqual([...fx.calls].sort(), ['gate', 'health', 'host', 'state']) +}) + +test('node change hooks await asynchronous persistence in a fresh activity', async (t) => { + const fx = await fixture(t) + await fx.manager._nodesChanged() + assert.deepEqual(fx.calls, ['nodes']) +}) From ce7d1bd864dddceae4747210c51be1f327593132 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:39:20 +0800 Subject: [PATCH 200/473] test: retain VM ownership when a recorded container loses its name --- test/unit/remote-slot-stop-proof.test.mjs | 24 ++++++++++++++++++++++- test/unit/vm-stop-proof.test.mjs | 20 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/test/unit/remote-slot-stop-proof.test.mjs b/test/unit/remote-slot-stop-proof.test.mjs index a0cba36cd..6338ea746 100644 --- a/test/unit/remote-slot-stop-proof.test.mjs +++ b/test/unit/remote-slot-stop-proof.test.mjs @@ -5,7 +5,14 @@ import vmContext from 'node:vm' import { getDb } from '../../src/lib/db/database.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' -function fixture({ noop = false, replace = false, inspectError = false, actionError = false, running = true } = {}) { +function fixture({ + noop = false, + replace = false, + inspectError = false, + actionError = false, + running = true, + nameAbsent = false, +} = {}) { const source = fs.readFileSync(new URL('../../src/lib/cluster/remote-slot.mjs', import.meta.url), 'utf8') const body = source.slice(source.indexOf('export async function stopRemoteSlot(')).replaceAll('export ', '') let present = true @@ -13,6 +20,7 @@ function fixture({ noop = false, replace = false, inspectError = false, actionEr const events = [] const inspect = async (_docker, target) => { if (inspectError) throw new Error('Docker daemon unavailable') + if (target === 'kin-synthetic' && nameAbsent) return null if (target === 'kin-synthetic' && replacement) return { Id: 'replacement-id', State: { Running: true, Pid: 456 } } if (!present) return null return { Id: 'original-id', State: { Running: running, Restarting: false, Pid: running ? 123 : 0 }, HostConfig: {} } @@ -103,6 +111,20 @@ test('remote already stopped container remains successful', async () => { assert.equal((await f.stopRemoteSlot(f.vm)).ok, true) }) +test('a missing name cannot hide a still present recorded immutable remote container', async () => { + for (const action of ['stopRemoteSlot', 'destroyRemoteSlot']) { + const f = fixture({ nameAbsent: true, noop: true }) + f.vm.runtime.container_id = 'original-id' + const result = await f[action](f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal( + f.events.some(([name]) => name === 'files-remove'), + false, + ) + } +}) + test('unconfirmed remote stop keeps its persistent VM operation active', async () => { const f = fixture({ noop: true }) const operations = new SharedVmOperations(getDb()) diff --git a/test/unit/vm-stop-proof.test.mjs b/test/unit/vm-stop-proof.test.mjs index a07b10e93..cd6ab9435 100644 --- a/test/unit/vm-stop-proof.test.mjs +++ b/test/unit/vm-stop-proof.test.mjs @@ -14,7 +14,14 @@ function section(file, start, end) { return source.slice(source.indexOf(start), source.indexOf(end)).replaceAll('export ', '') } -function localRuntime({ inspectError, running = true, restarting = false, noop = false, replace = false } = {}) { +function localRuntime({ + inspectError, + running = true, + restarting = false, + noop = false, + replace = false, + nameAbsent = false, +} = {}) { const commands = [] let present = true let replacement = false @@ -23,6 +30,7 @@ function localRuntime({ inspectError, running = true, restarting = false, noop = if (argv[1] === 'inspect') { if (inspectError) return { ok: false, stderr: inspectError } const target = argv.at(-1) + if (nameAbsent && target === 'kin-synthetic') return { ok: false, stderr: `Error: No such object: ${target}` } if (!present && !(replacement && target === 'kin-synthetic')) { return { ok: false, stderr: `Error: No such object: ${target}` } } @@ -116,6 +124,16 @@ test('known absent and already stopped containers remain successful', () => { assert.equal(f.stopVmRuntime(f.vm).action, 'already-stopped') }) +test('a missing name cannot hide a still present recorded immutable local container', () => { + for (const action of ['stopVmRuntime', 'destroyVmRuntime']) { + const f = localRuntime({ nameAbsent: true, noop: true }) + f.vm.runtime.container_id = 'original-id' + const result = f[action](f.vm) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + } +}) + test('unconfirmed Docker termination retains the persistent VM operation', async () => { const operations = new SharedVmOperations(getDb()) const f = localRuntime({ noop: true }) From f62ae214c8ec7bb6624a85b87ab91a41756961af Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:39:43 +0800 Subject: [PATCH 201/473] test: require trusted maintenance handoff and private backup claims --- test/unit/maintenance-handoff.test.mjs | 154 +++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 test/unit/maintenance-handoff.test.mjs diff --git a/test/unit/maintenance-handoff.test.mjs b/test/unit/maintenance-handoff.test.mjs new file mode 100644 index 000000000..15768d74c --- /dev/null +++ b/test/unit/maintenance-handoff.test.mjs @@ -0,0 +1,154 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import pg from 'pg' +import { getDb, quoteIdentifier, registerActivityCoordinator } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { runPgTool } from '../../src/lib/admin/backup-core.mjs' + +async function fixture(t) { + const db = getDb() + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + registerActivityCoordinator(db, coordinator) + const operations = [] + t.after(async () => { + await Promise.allSettled(operations.map((operation) => operation.close())) + await coordinator.stop().catch(() => {}) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + return { db, coordinator, operations, q: quoteIdentifier(coordinator.schema) } +} + +test('trusted current HTTP activity transfers to maintenance without waiting for itself and resumes for response writes', async (t) => { + const { db, coordinator, operations, q } = await fixture(t) + const request = await coordinator.admit({ kind: 'http' }) + const operation = await request.run(() => + coordinator.beginMaintenanceFromCurrentActivity({ kind: 'backup', timeoutMs: 1000 }), + ) + operations.push(operation) + const history = ( + await db.pool.query(`SELECT source_activity_id, ended_at FROM ${q}.operations WHERE id=$1`, [operation.operationId]) + ).rows[0] + assert.equal(history.source_activity_id, request.activityId) + assert.equal(history.ended_at, null) + await operation.runAdmin(() => db.query('SELECT 1')) + await operation.resume() + await operation.close() + await request.run(() => db.exec("INSERT INTO settings(key,value) VALUES ('after-maintenance-response','true')")) + await request.complete({ ended: true }) + assert.ok( + (await db.pool.query(`SELECT ended_at FROM ${q}.operations WHERE id=$1`, [operation.operationId])).rows[0].ended_at, + ) +}) + +test('scheduled backup claims and deduplication stay outside the business snapshot', async (t) => { + const { db, coordinator, q } = await fixture(t) + let calls = 0 + const plan = { taskKey: 'backup', executionId: 'synthetic-backup-1' } + const result = await coordinator.runScheduledMaintenance(plan, async (operation) => { + calls++ + await operation.runAdmin(async () => { + assert.equal( + (await db.query('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL')).rows[0].n, + 0, + ) + const dump = await runPgTool( + 'pg_dump', + [ + '--data-only', + '--no-owner', + '--no-privileges', + `--schema=${db.schema}`, + `--table=${db.schema}.background_task_runs`, + ], + db.connectionString, + ) + assert.equal(dump.includes('synthetic-backup-1'), false) + }) + await operation.resume() + return { ended: true, value: { backupId: 'synthetic-backup-1' } } + }) + assert.deepEqual(result, { ran: true, value: { backupId: 'synthetic-backup-1' } }) + const duplicate = await coordinator.runScheduledMaintenance(plan, () => { + calls++ + throw new Error('must not run') + }) + assert.equal(duplicate.ran, false) + assert.deepEqual(duplicate.sharedResult, { backupId: 'synthetic-backup-1' }) + assert.equal(calls, 1) + assert.equal((await db.pool.query(`SELECT count(*) AS n FROM ${q}.operations WHERE task_key='backup'`)).rows[0].n, 1) +}) + +test('handoff rejects missing current scope, started remote work and unfinished SQL', async (t) => { + const { db, coordinator } = await fixture(t) + await assert.rejects(coordinator.beginMaintenanceFromCurrentActivity({ kind: 'restore' }), { + code: 'maintenance_activity_closed', + }) + const remote = await coordinator.admit({ kind: 'http' }) + remote.noteExecution({ requestId: 'remote', started: true, ended: false }) + await remote.run(() => + assert.rejects(coordinator.beginMaintenanceFromCurrentActivity({ kind: 'restore' }), { + code: 'maintenance_handoff_started', + }), + ) + await remote.complete({ ended: true }) + const writing = await coordinator.admit({ kind: 'http' }) + await writing.run(async () => { + const pending = db.query('SELECT pg_sleep(0.05)') + await assert.rejects(coordinator.beginMaintenanceFromCurrentActivity({ kind: 'restore' }), { + code: 'maintenance_persistence_pending', + }) + await pending + }) + await writing.complete({ ended: true }) + assert.equal((await coordinator.getState()).phase, 'open') +}) + +test('a transferred restore owner death retains its phase and private unfinished history', async (t) => { + const { db, coordinator, operations, q } = await fixture(t) + const request = await coordinator.admit({ kind: 'http' }) + const operation = await request.run(() => + coordinator.beginMaintenanceFromCurrentActivity({ kind: 'restore', timeoutMs: 1000 }), + ) + operations.push(operation) + await operation.beginRestore() + await db.pool.query('SELECT pg_terminate_backend($1)', [operation.backendPid]) + await new Promise((resolve) => setTimeout(resolve, 10)) + assert.equal((await coordinator.getState()).phase, 'restoring') + assert.equal( + (await db.pool.query(`SELECT ended_at FROM ${q}.operations WHERE id=$1`, [operation.operationId])).rows[0].ended_at, + null, + ) + await assert.rejects(coordinator.admit({ kind: 'http' }), { code: 'database_maintenance' }) + await assert.rejects( + async () => request.run(() => db.query('SELECT 1')), + { code: 'maintenance_activity_closed' }, + ) +}) + +test('handoff never excludes another active or unknown background claim', async (t) => { + const { db, coordinator } = await fixture(t) + await db.pool.query( + "INSERT INTO background_task_runs(task_key,execution_id,owner_id,unknown_at) VALUES ('other','one','other-pod',clock_timestamp())", + ) + const request = await coordinator.admit({ kind: 'http' }) + await request.run(() => + assert.rejects(coordinator.beginMaintenanceFromCurrentActivity({ kind: 'backup', timeoutMs: 1000 }), (error) => { + assert.equal(error.code, 'maintenance_claims_active') + assert.equal(error.claims.background_task_runs, 1) + return true + }), + ) + const state = await coordinator.getState() + assert.equal(state.phase, 'draining') + await coordinator.cancelDrain(state.operationId) + await request.run(() => db.query('SELECT 1')) + await request.complete({ ended: true }) +}) From 75f69a85d73064d36ba9d6d733d3b7e2c95d67dc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:40:51 +0800 Subject: [PATCH 202/473] fix: verify remote container termination before removing VM state --- src/lib/cluster/remote-slot.mjs | 59 ++++++++++++++++++++++++++++---- src/lib/vm/vm-runtime.mjs | 5 +-- test/unit/vm-stop-proof.test.mjs | 21 ++++++++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/src/lib/cluster/remote-slot.mjs b/src/lib/cluster/remote-slot.mjs index 80804f7d1..79099a44f 100644 --- a/src/lib/cluster/remote-slot.mjs +++ b/src/lib/cluster/remote-slot.mjs @@ -1249,17 +1249,37 @@ export async function stopRemoteSlot(vm) { try { const session = await nodeSession(vmNodeId(vm)) const name = containerName(vm.id) - const info = await inspectContainerOrNull(session.docker, name) + const info = await inspectRemoteStopTarget(session, vm, name) if (!info) { if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true } return { ok: true, action: 'absent', runtime: vm.runtime || null } } + if (remoteContainerStopped(info)) { + vm.runtime = { ...(vm.runtime || {}), pid: null, stopped: true } + return { ok: true, action: 'already-stopped', runtime: vm.runtime } + } const t = isKvmRuntime(vm) || runtimeLabelOf(info) === 'kvm' ? 30 : undefined - await containerAction(session.docker, name, 'stop', { t }) + let commandError + try { + await containerAction(session.docker, info.Id, 'stop', { t }) + } catch (error) { + commandError = error + } + const remaining = await inspectContainerOrNull(session.docker, info.Id) + const named = await inspectContainerOrNull(session.docker, name) + if ( + (remaining && !remoteContainerStopped(remaining)) || + (named && (named.Id !== info.Id || !remoteContainerStopped(named))) + ) { + return { + ...failure(commandError || new Error('Container termination was not confirmed')), + vmOperationUncertain: true, + } + } vm.runtime = { ...(vm.runtime || {}), pid: null, stopped: true } return { ok: true, action: 'stopped', runtime: vm.runtime } } catch (err) { - return failure(err) + return { ...failure(err), vmOperationUncertain: true } } } @@ -1268,9 +1288,22 @@ export async function destroyRemoteSlot(vm) { try { const session = await nodeSession(vmNodeId(vm)) const name = containerName(vm.id) - const info = await inspectContainerOrNull(session.docker, name) + const info = await inspectRemoteStopTarget(session, vm, name) if (info) { - await removeContainer(session.docker, name) + let commandError + try { + await removeContainer(session.docker, info.Id) + } catch (error) { + commandError = error + } + const remaining = await inspectContainerOrNull(session.docker, info.Id) + const named = await inspectContainerOrNull(session.docker, name) + if (remaining || named) { + return { + ...failure(commandError || new Error('Container removal was not confirmed')), + vmOperationUncertain: true, + } + } await releaseRemoteEgress(session, info.HostConfig?.NetworkMode) } await removeRemoteSlotDir(vm, session) @@ -1278,6 +1311,20 @@ export async function destroyRemoteSlot(vm) { if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true, removed: true } return { ok: true, action: info ? 'removed' : 'absent', runtime: vm.runtime || null } } catch (err) { - return failure(err) + return { ...failure(err), vmOperationUncertain: true } } } + +async function inspectRemoteStopTarget(session, vm, name) { + const info = + (await inspectContainerOrNull(session.docker, name)) || + (vm.runtime?.container_id ? await inspectContainerOrNull(session.docker, vm.runtime.container_id) : null) + if (info && (!info.Id || (vm.runtime?.container_id && vm.runtime.container_id !== info.Id))) { + throw new Error('Container identity changed') + } + return info +} + +function remoteContainerStopped(info) { + return info.State?.Running === false && info.State?.Restarting === false && info.State?.Pid === 0 +} diff --git a/src/lib/vm/vm-runtime.mjs b/src/lib/vm/vm-runtime.mjs index 9c7e45f3e..551890bcc 100644 --- a/src/lib/vm/vm-runtime.mjs +++ b/src/lib/vm/vm-runtime.mjs @@ -1173,7 +1173,7 @@ export function destroyVmRuntime(vm) { if (!vm?.id) return { ok: false, error: 'vm required' } try { const name = containerName(vm.id) - const info = inspectContainer(name) + const info = inspectContainer(name) || (vm.runtime?.container_id ? inspectContainer(vm.runtime.container_id) : null) if (!info) { if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true, removed: true } return { ok: true, action: 'absent', runtime: vm.runtime || null } @@ -1195,12 +1195,13 @@ export function destroyVmRuntime(vm) { export function stopVmRuntime(vm) { try { const name = containerName(vm.id) - const info = inspectContainer(name) + const info = inspectContainer(name) || (vm.runtime?.container_id ? inspectContainer(vm.runtime.container_id) : null) if (!info) { if (vm.runtime) vm.runtime = { ...vm.runtime, pid: null, ip: null, stopped: true } return { ok: true, action: 'absent', runtime: vm.runtime || null } } assertContainerIdentity(vm, info) + info.name = name if (containerStopped(info)) { runtimePatch(vm, info) vm.runtime.stopped = true diff --git a/test/unit/vm-stop-proof.test.mjs b/test/unit/vm-stop-proof.test.mjs index cd6ab9435..cd2c743ec 100644 --- a/test/unit/vm-stop-proof.test.mjs +++ b/test/unit/vm-stop-proof.test.mjs @@ -216,6 +216,27 @@ test('launchctl permission errors cannot prove native service termination', asyn await assert.rejects(api.bootout({}, 'gui/501', 'label')) }) +test('native stop returns uncertainty and keeps its durable VM claim when launchd inspection fails', async () => { + const api = nativeApi({ code: 1, stdout: '', stderr: 'Operation not permitted' }) + const source = section( + '../../src/lib/cluster/native-slot.mjs', + 'export async function stopNativeSlot(', + '/** Explicit delete / factory reset: both jobs', + ) + const stop = vmContext.runInNewContext(`${source}; stopNativeSlot`, { + ...api, + nativeSession: async () => ({}), + launchdDomain: async () => 'gui/501', + nativeLabels: () => ({ kernel: 'kernel', telemetry: 'telemetry' }), + failure: (error) => ({ ok: false, error: error.message }), + }) + const operations = new SharedVmOperations(getDb()) + const result = await operations.run('vm-synthetic', () => stop({ id: 'vm-synthetic', runtime: {} })) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal((await operations.tryAcquire({ vmId: 'vm-synthetic' })).ok, false) +}) + test('launchd label disappearance alone does not prove the old PID exited', async () => { for (const process of [ { code: 0, stdout: '123', stderr: '' }, From da35b899c8b1428d432a91ccd7e1f4126b953d3f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:41:02 +0800 Subject: [PATCH 203/473] fix(api): preserve preparation failures before execution hooks --- src/lib/pool/api-protocol.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/pool/api-protocol.mjs b/src/lib/pool/api-protocol.mjs index 1e542a10f..48a4c8e06 100644 --- a/src/lib/pool/api-protocol.mjs +++ b/src/lib/pool/api-protocol.mjs @@ -88,6 +88,7 @@ export async function runApiInference(options = {}) { result = await runApiInferenceBody(options, lifecycle) } catch (error) { if (lifecycle.startFailed) throw lifecycle.startError + if (!lifecycle.startedHook) throw error result = { ok: false, status: 502, From 7260ef35f87c918d399953f0e5814e616aa0fe9f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:41:50 +0800 Subject: [PATCH 204/473] fix: await cluster persistence callbacks within activity scopes --- src/server.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.mjs b/src/server.mjs index 39d9db674..09abb52f1 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1069,12 +1069,12 @@ const clusterManager = new ClusterManager({ } }, onNodesChanged: () => { - if (!backupService.isMaintaining) syncNodeLocalEgress() + if (!backupService.isMaintaining) return syncNodeLocalEgress() }, // A node's local row follows its link: re-probe as soon as the link settles. onLinkState: (nodeId, state) => { if (!backupService.isMaintaining && (state === 'ready' || state === 'error' || state === 'backoff')) { - proxyPool.probeLocalNode(nodeId).catch(() => {}) + return proxyPool.probeLocalNode(nodeId) } }, }) @@ -1082,7 +1082,7 @@ localEgressCluster = clusterManager bindPlacement({ manager: clusterManager, projectRoot: cfg.paths.project }) await startNodeEgressSocks({ clientFor: (nodeId) => clusterManager.client(nodeId) }) await clusterManager.start() -clusterManager.restoreSlotRelays() +await clusterManager.restoreSlotRelays() await syncNodeLocalEgress() if (proxyPool.snapshot().config.ipv6_enabled !== true) { const exits = await syncIpv6ProxyEgress(cfg.paths.project, proxyPool) From a92ce1b31f0e1bbe424f285219e42374aa6c7154 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:42:34 +0800 Subject: [PATCH 205/473] fix: scope and await asynchronous cluster database work --- src/lib/cluster/cluster-manager.mjs | 68 ++++++++++++++++++++--------- test/unit/distill-detect.test.mjs | 2 +- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/src/lib/cluster/cluster-manager.mjs b/src/lib/cluster/cluster-manager.mjs index e3c475a8c..5acd86425 100644 --- a/src/lib/cluster/cluster-manager.mjs +++ b/src/lib/cluster/cluster-manager.mjs @@ -5,7 +5,7 @@ */ import crypto from 'node:crypto' -import { getDb, withTransaction } from '../db/database.mjs' +import { getDb, withTransaction, getActivityCoordinator } from '../db/database.mjs' import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { OneTimeAuthRepo } from '../db/repos/one-time-auth-repo.mjs' import fs from 'node:fs' @@ -159,6 +159,8 @@ export class ClusterManager { this.tickets = new Map() this.authState = isKubernetesMode() ? authState || new OneTimeAuthRepo(repo.db || getDb()) : null this._healthTimer = null + this._backgroundTasks = new Set() + this._stopping = false } async reload() { @@ -180,20 +182,25 @@ export class ClusterManager { } async start() { + this._stopping = false await this.reload() for (const node of this.nodes.values()) this._attach(node) this._healthTimer = setInterval(() => { - for (const id of this.links.keys()) this._checkHealth(id) + for (const id of this.links.keys()) { + void this._background('cluster_health', () => this._checkHealth(id)).catch(() => {}) + } }, HEALTH_INTERVAL_MS) this._healthTimer.unref?.() } async stop() { + this._stopping = true clearInterval(this._healthTimer) this._healthTimer = null for (const link of this.links.values()) link.stop() const relays = [...this.bridges.values(), ...[...this.slotRelays.values()].flatMap((r) => [r.kernel, r.worker])] await Promise.all(relays.map((r) => r.stop().catch(() => {}))) + while (this._backgroundTasks.size) await Promise.allSettled([...this._backgroundTasks]) this.links.clear() this.bridges.clear() this.slotRelays.clear() @@ -203,21 +210,44 @@ export class ClusterManager { return path.join(this.socketDir, id, 'docker.sock') } + _background(kind, callback) { + if (this._stopping) return Promise.reject(new ClusterError(503, 'cluster_stopping', '集群正在停止')) + const coordinator = getActivityCoordinator(this.repo.db) + const pending = coordinator + ? coordinator.runActivity({ kind }, async () => ({ ended: true, value: await callback() })) + : Promise.resolve().then(callback) + this._backgroundTasks.add(pending) + pending.then( + () => this._backgroundTasks.delete(pending), + (error) => { + this._backgroundTasks.delete(pending) + this.logger.warn(`[cluster] ${kind} failed: ${error.code || error.name}`) + }, + ) + return pending + } + _attach(node) { - const link = new SshLink({ id: node.id, resolveTarget: () => this._targetFor(node.id) }) + const link = new SshLink({ + id: node.id, + resolveTarget: () => this._background('cluster_ssh_target', () => this._targetFor(node.id)), + }) link.on('state', (state) => { this.logger.log(JSON.stringify({ event: 'cluster-link', node: node.id, state, error: link.error?.code || null })) if (state === 'ready') { // Host facts first: they carry the platform the health check branches on. - Promise.resolve(this.remoteHost(node.id)) - .catch(() => {}) - .then(() => this._checkHealth(node.id)) - .catch(() => {}) - .finally(() => this._linkState(node.id, state)) - this._ensureGate(node.id).catch(() => {}) + void this._background('cluster_link_ready', async () => { + try { + await this.remoteHost(node.id) + await this._checkHealth(node.id) + } finally { + await this._linkState(node.id, state) + } + }).catch(() => {}) + void this._background('cluster_gate', () => this._ensureGate(node.id)).catch(() => {}) } else { this.health.delete(node.id) - this._linkState(node.id, state) + void this._background('cluster_link_state', () => this._linkState(node.id, state)).catch(() => {}) } }) this.links.set(node.id, link) @@ -415,7 +445,7 @@ export class ClusterManager { } catch (err) { this._directPatch(id, { gate_error: err.message }) this.logger.warn(`[cluster] slot gate on ${id}: ${err.message}`) - if (strict) throw err + if (strict || getActivityCoordinator(this.repo.db)) throw err } } @@ -505,7 +535,7 @@ export class ClusterManager { this.nodes.set(node.id, node) this.platforms.set(node.id, 'linux') this._attach(node) - this._nodesChanged() + await this._nodesChanged() return this._view(node) } @@ -710,20 +740,17 @@ export class ClusterManager { } } - _linkState(id, state) { + async _linkState(id, state) { try { - this.onLinkState?.(id, state) + await this.onLinkState?.(id, state) } catch (err) { this.logger.warn(`[cluster] link-state hook failed: ${err.message}`) + if (getActivityCoordinator(this.repo.db)) throw err } } _nodesChanged() { - try { - this.onNodesChanged?.() - } catch (err) { - this.logger.warn(`[cluster] nodes-changed hook failed: ${err.message}`) - } + return this._background('cluster_nodes_changed', () => this.onNodesChanged?.()).catch(() => {}) } async remove(id) { @@ -748,7 +775,7 @@ export class ClusterManager { this.nodes.delete(id) this.platforms.delete(id) this.directLinksState.delete(id) - this._nodesChanged() + await this._nodesChanged() } reconnect(id) { @@ -811,6 +838,7 @@ export class ClusterManager { } } catch (err) { this.logger.warn(`[cluster] record platform of ${id}: ${err.message}`) + if (getActivityCoordinator(this.repo.db)) throw err } } diff --git a/test/unit/distill-detect.test.mjs b/test/unit/distill-detect.test.mjs index 7f5d23066..026999e6a 100644 --- a/test/unit/distill-detect.test.mjs +++ b/test/unit/distill-detect.test.mjs @@ -608,7 +608,7 @@ test('normalize keeps default fingerprints when omitted', () => { test('handleProtocol intercepts before credential hop', () => { const src = fs.readFileSync(path.join(root, 'src/lib/protocol/handle-protocol.mjs'), 'utf8') const early = src.indexOf('await applyProtocolIntercept(') - const codex = src.indexOf('await handleCodexProtocol(options)') + const codex = src.indexOf('await handleCodexProtocol(') assert.ok(early > 0 && early < codex) const before = src.indexOf("applyIntercept(cfg.intercept.rules, 'before_upstream'") const late = src.indexOf('await applyProtocolIntercept(', before) From 2923a65ff170bb289356b2f5c5ae74afbcf84ac0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:43:51 +0800 Subject: [PATCH 206/473] test: model execution and maintenance lifetimes in protocol fixtures --- test/fixtures/protocol-settlement-failure.mjs | 5 ++- .../unit/shared-background-producers.test.mjs | 31 ++++++++++--------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/test/fixtures/protocol-settlement-failure.mjs b/test/fixtures/protocol-settlement-failure.mjs index b33ee4386..f43771955 100644 --- a/test/fixtures/protocol-settlement-failure.mjs +++ b/test/fixtures/protocol-settlement-failure.mjs @@ -8,11 +8,14 @@ import { getDb } from '../../src/lib/db/database.mjs' import { readBody } from '../../src/lib/http/respond.mjs' const [route, failure] = process.env.VM2API_SETTLEMENT_FAILURE_CASE.split(':') -async function completedUpstream({ logBag, json, res }) { +async function completedUpstream({ req, logBag, json, res, onExecutionState }) { + const execution = { requestId: 'synthetic-settlement-job', started: true, ended: false } + await onExecutionState?.(req, execution) logBag.final_state = 'verified' logBag.usage = { input_tokens: 1, output_tokens: 1 } const result = json(res, 200, { upstream_completed: true }) if (failure === 'backend') throw new Error('simulated execution-stage backend failure') + await onExecutionState?.(req, { ...execution, ended: true, reason: 'http_body_complete' }) return result } mock.module(new URL('../../src/lib/protocol/handle-codex.mjs', import.meta.url), { diff --git a/test/unit/shared-background-producers.test.mjs b/test/unit/shared-background-producers.test.mjs index 44736e936..9b23a1419 100644 --- a/test/unit/shared-background-producers.test.mjs +++ b/test/unit/shared-background-producers.test.mjs @@ -4,6 +4,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { getDb, createDatabase } from '../../src/lib/db/database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' import { RuntimeSettings, loadSettingsState } from '../../src/lib/db/settings-state.mjs' import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' import { createHealthProbeMonitor, cachedClaudeMessage } from '../../src/lib/admin/health-probe.mjs' @@ -133,21 +134,23 @@ test('Kubernetes health uses the default shared service and refreshes routing be merge: (current, patch) => ({ ...current, ...patch }), apply: (config) => monitor?.setConfig(config.health_probe, { restart: false }), }) - await routing.init() - monitor = createHealthProbeMonitor({ - config: { enabled: true, real: { prompt: 'old prompt' } }, - listTargets: () => [live], - runChat: async (_vm, config) => { - assert.equal(config.prompt, 'new prompt') - return { ok: true, text: 'synthetic', body: { id: 'synthetic-message' } } - }, + await withDatabaseActivities([f.db], async () => { + await routing.init() + monitor = createHealthProbeMonitor({ + config: { enabled: true, real: { prompt: 'old prompt' } }, + listTargets: () => [live], + runChat: async (_vm, config) => { + assert.equal(config.prompt, 'new prompt') + return { ok: true, text: 'synthetic', body: { id: 'synthetic-message' } } + }, + }) + await new RoutingConfigRepo(f.other).patch((current) => ({ + ...current, + health_probe: { enabled: true, real: { prompt: 'new prompt' } }, + })) + assert.equal((await monitor.runRealProbe()).ok, true) + assert.equal((await f.other.prepare('SELECT count(*) AS n FROM background_task_runs').get()).n, 1) }) - await new RoutingConfigRepo(f.other).patch((current) => ({ - ...current, - health_probe: { enabled: true, real: { prompt: 'new prompt' } }, - })) - assert.equal((await monitor.runRealProbe()).ok, true) - assert.equal((await f.other.prepare('SELECT count(*) AS n FROM background_task_runs').get()).n, 1) }) test('usage timer and event probes share one persisted VM task plan across monitors', async (t) => { From 818d58791a2edbd2a2e6365fc89411a5a6907213 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:46:09 +0800 Subject: [PATCH 207/473] feat: transfer trusted maintenance activities to durable operation history --- src/lib/db/database.mjs | 5 +- src/lib/db/global-maintenance.mjs | 284 +++++++++++++++++++++++--- test/unit/global-maintenance.test.mjs | 2 +- 3 files changed, 265 insertions(+), 26 deletions(-) diff --git a/src/lib/db/database.mjs b/src/lib/db/database.mjs index ed1e0b4c5..a4b9793fd 100644 --- a/src/lib/db/database.mjs +++ b/src/lib/db/database.mjs @@ -86,17 +86,18 @@ export class PostgresDatabase { async query(sql, params = []) { if (this.closed) throw new Error('database is closed') - getActivityCoordinator(this)?.assertActivity() const tx = transactions.getStore() if (this.maintaining && maintenance.getStore() !== this && tx?.db !== this) { throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) } + const finishPersistence = getActivityCoordinator(this)?.trackPersistence() this.active += 1 try { return await (tx?.db === this ? tx.client : this.pool).query(sql, params) } catch (error) { throw databaseError(error) } finally { + finishPersistence?.() this.active -= 1 if (!this.active) for (const resolve of this.waiters.splice(0)) resolve() } @@ -174,6 +175,7 @@ export async function withTransaction(db, fn) { throw Object.assign(new Error('database maintenance in progress'), { code: 'database_maintenance' }) } // Count the entire transaction, including time between individual queries. + const finishPersistence = getActivityCoordinator(db)?.trackPersistence() db.active += 1 let client try { @@ -188,6 +190,7 @@ export async function withTransaction(db, fn) { if (client) await client.query('ROLLBACK').catch(() => {}) throw error } finally { + finishPersistence?.() client?.release() db.active -= 1 if (!db.active) for (const resolve of db.waiters.splice(0)) resolve() diff --git a/src/lib/db/global-maintenance.mjs b/src/lib/db/global-maintenance.mjs index a4afdcb71..bfe7906af 100644 --- a/src/lib/db/global-maintenance.mjs +++ b/src/lib/db/global-maintenance.mjs @@ -57,6 +57,7 @@ export class GlobalMaintenanceCoordinator { this._held = false this._refs = 0 this._bootstrapUsed = false + this._transfers = new Map() this._serial = Promise.resolve() this._onError = (error) => this._lose(error) this._onEnd = () => this._lose(failure('maintenance_connection_lost', 'Activity connection ended')) @@ -177,6 +178,18 @@ export class GlobalMaintenanceCoordinator { reason text ); CREATE INDEX IF NOT EXISTS activities_pending ON ${q}.activities (owner_id) WHERE completed_at IS NULL; + CREATE TABLE IF NOT EXISTS ${q}.operations ( + id text PRIMARY KEY, + owner_id text NOT NULL, + kind text NOT NULL, + source_activity_id text, + task_key text, + execution_id text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ended_at timestamptz, + shared_result jsonb, + UNIQUE (task_key, execution_id) + ); REVOKE ALL ON ALL TABLES IN SCHEMA ${q} FROM PUBLIC`) await client.query( `INSERT INTO ${q}.control(id,application_schema,phase,epoch) @@ -322,10 +335,12 @@ export class GlobalMaintenanceCoordinator { } _gate(id, epoch, kind) { - const scope = { coordinator: this, id, epoch, kind, active: true, remoteStarted: false } + const scope = { coordinator: this, id, epoch, kind, active: true, remoteStarted: false, pending: new Set() } const assertActive = () => { this._assertHealthy() - if (!scope.active) throw failure('maintenance_activity_closed', 'The activity scope has completed') + if ((!scope.active && !scope.transferredTo?.healthy) || scope.handoffPending) { + throw failure('maintenance_activity_closed', 'The activity scope has completed or is transferring') + } } const gate = { activityId: id, @@ -342,6 +357,9 @@ export class GlobalMaintenanceCoordinator { }, noteExecution: (execution) => { assertActive() + if (scope.transferredTo && execution?.started === true) { + throw failure('maintenance_handoff_started', 'Maintenance ownership cannot submit inference') + } if (execution?.started === true) scope.remoteStarted = true }, complete: (execution) => this._complete(scope, execution), @@ -354,12 +372,22 @@ export class GlobalMaintenanceCoordinator { this._assertHealthy() for (let context = activities.getStore(); context; context = context.parent) { const { scope } = context - if (!scope.active) throw failure('maintenance_activity_closed', 'An inherited activity scope has completed') + if ((!scope.active && !scope.transferredTo?.healthy) || scope.handoffPending) { + throw failure('maintenance_activity_closed', 'An inherited activity scope has completed') + } + scope.operation?._assertHealthy() if (scope.coordinator === this) return scope } throw failure('maintenance_activity_closed', 'SQL requires a live admitted activity scope') } + trackPersistence() { + const scope = this.assertActivity() + const token = Symbol('persistence') + scope.pending.add(token) + return () => scope.pending.delete(token) + } + async _complete(scope, execution) { return this._exclusiveLocal(async () => { this._assertHealthy() @@ -399,7 +427,7 @@ export class GlobalMaintenanceCoordinator { this._held = false } - async _checkClaims(client) { + async _checkClaims(client, operationId = '') { const claims = { activities: Number( ( @@ -408,6 +436,15 @@ export class GlobalMaintenanceCoordinator { ).rows[0].count, ), } + claims.operations = Number( + ( + await client.query( + `SELECT count(*) FROM ${this.qualified}.operations + WHERE ended_at IS NULL AND id <> $1`, + [operationId], + ) + ).rows[0].count, + ) for (const [table, condition] of BUSINESS_CLAIMS) { const name = `${quoteIdentifier(this.db.schema)}.${quoteIdentifier(table)}` if (!(await client.query('SELECT to_regclass($1) AS name', [name])).rows[0].name) { @@ -421,7 +458,91 @@ export class GlobalMaintenanceCoordinator { return claims } - async beginMaintenance({ kind, signal = null, timeoutMs = 60000 } = {}) { + beginMaintenance(options) { + return this._beginMaintenance(options) + } + + async beginMaintenanceFromCurrentActivity(options) { + const scope = this.assertActivity() + if (!scope.active || scope.operation) + throw failure('maintenance_activity_closed', 'An ordinary activity is required') + if (scope.remoteStarted) + throw failure('maintenance_handoff_started', 'Submitted remote work cannot transfer ownership') + if (scope.pending.size || this.db.isTransaction) { + throw failure('maintenance_persistence_pending', 'Finish activity persistence before transferring ownership') + } + scope.handoffPending = true + try { + return await this._beginMaintenance(options, scope) + } catch (error) { + scope.handoffPending = false + throw error + } + } + + async runScheduledMaintenance(plan, callback) { + let operation + try { + operation = await this._beginMaintenance({ kind: 'backup' }, null, plan) + } catch (error) { + if (error.code === 'maintenance_already_completed') { + return { ran: false, code: error.code, sharedResult: error.sharedResult } + } + throw error + } + try { + const outcome = await operation.runAdmin(() => callback(operation)) + if (outcome?.ended !== true) throw failure('maintenance_execution_unknown', 'Maintenance completion is unknown') + if (!operation._resumed) await operation.resume() + await operation.finish(outcome.value) + return { ran: true, value: outcome.value } + } catch (error) { + operation._completionUnknown = true + throw error + } finally { + await operation.close() + } + } + + async _resumeTransfer(operationId) { + const scope = this._transfers.get(operationId) + if (!scope) return + await this._exclusiveLocal(async () => { + this._assertHealthy() + try { + if (!this._held) { + if (!(await this._client.query(TRY_SHARED, [this.activityResource])).rows[0].acquired) { + throw failure('database_maintenance', 'Cannot resume the transferred request during maintenance') + } + this._held = true + } + const epoch = await this._transaction(async (client) => { + if (!(await client.query(TRY_SHARED_TX, [this.activityResource])).rows[0].acquired) + throw failure('database_maintenance', 'Maintenance is queued') + const row = (await client.query(`SELECT phase, epoch FROM ${this.qualified}.control WHERE id=1 FOR SHARE`)) + .rows[0] + if (row.phase !== 'open') throw failure('database_maintenance', 'Maintenance has not resumed') + const result = await client.query( + `UPDATE ${this.qualified}.activities SET completed_at=NULL, reason=NULL, epoch=$3 + WHERE id=$1 AND owner_id=$2 AND completed_at IS NOT NULL AND reason='maintenance_handoff'`, + [scope.id, this.ownerId, row.epoch], + ) + if (result.rowCount !== 1) throw failure('maintenance_activity_closed', 'The transferred request changed') + return Number(row.epoch) + }) + scope.epoch = epoch + scope.active = true + scope.transferredTo = null + this._refs += 1 + this._transfers.delete(operationId) + } catch (error) { + if (!this._refs) await this._releaseHolder() + throw error + } + }) + } + + async _beginMaintenance({ kind, signal = null, timeoutMs = 60000 } = {}, sourceScope = null, schedule = null) { this._assertHealthy() if (!['backup', 'restore'].includes(kind)) throw new Error('invalid maintenance kind') if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) throw new Error('invalid maintenance timeout') @@ -434,20 +555,87 @@ export class GlobalMaintenanceCoordinator { if (!(await client.query(TRY_EXCLUSIVE, [this.adminResource])).rows[0].acquired) { throw failure('maintenance_busy', 'Another maintenance or schema operation is active') } - await client.query('BEGIN') - try { - const row = (await client.query(`SELECT * FROM ${this.qualified}.control WHERE id = 1 FOR UPDATE`)).rows[0] - if (row.phase !== 'open') throw failure('database_maintenance', 'A durable maintenance operation is pending') - await client.query( - `UPDATE ${this.qualified}.control SET phase = 'draining', operation_id = $1, + await this._exclusiveLocal(async () => { + await client.query('BEGIN') + try { + const row = (await client.query(`SELECT * FROM ${this.qualified}.control WHERE id = 1 FOR UPDATE`)).rows[0] + if (row.phase !== 'open') throw failure('database_maintenance', 'A durable maintenance operation is pending') + let executionId = null + if (schedule) { + if (typeof schedule.taskKey !== 'string' || !schedule.taskKey || schedule.taskKey.length > 512) + throw new Error('invalid maintenance task key') + if (schedule.executionId != null) { + if ( + typeof schedule.executionId !== 'string' || + !schedule.executionId || + schedule.executionId.length > 1024 + ) + throw new Error('invalid maintenance execution ID') + executionId = schedule.executionId + } else { + if (!Number.isSafeInteger(schedule.intervalMs) || schedule.intervalMs <= 0) + throw new Error('invalid maintenance interval') + const now = Number( + (await client.query('SELECT extract(epoch FROM clock_timestamp()) * 1000 AS now')).rows[0].now, + ) + executionId = `interval:${schedule.intervalMs}:${Math.floor(now / schedule.intervalMs)}` + } + const previous = ( + await client.query( + `SELECT ended_at, shared_result FROM ${this.qualified}.operations + WHERE task_key = $1 AND execution_id = $2`, + [schedule.taskKey, executionId], + ) + ).rows[0] + if (previous) + throw failure( + previous.ended_at ? 'maintenance_already_completed' : 'maintenance_claims_active', + 'Maintenance schedule has already been claimed', + { sharedResult: previous.shared_result }, + ) + } + await client.query( + `INSERT INTO ${this.qualified}.operations(id,owner_id,kind,source_activity_id,task_key,execution_id) + VALUES ($1,$2,$3,$4,$5,$6)`, + [ + operation.operationId, + this.ownerId, + kind, + sourceScope?.id || null, + schedule?.taskKey || null, + executionId, + ], + ) + await client.query( + `UPDATE ${this.qualified}.control SET phase = 'draining', operation_id = $1, owner_id = $2, kind = $3, reason = NULL, changed_at = clock_timestamp() WHERE id = 1`, - [operation.operationId, this.ownerId, kind], - ) - await client.query('COMMIT') - } catch (error) { - await client.query('ROLLBACK').catch(() => {}) - throw error - } + [operation.operationId, this.ownerId, kind], + ) + if (sourceScope) { + const transferred = await client.query( + `UPDATE ${this.qualified}.activities + SET completed_at = clock_timestamp(), reason = 'maintenance_handoff' + WHERE id = $1 AND owner_id = $2 AND epoch = $3 AND completed_at IS NULL AND uncertain_at IS NULL`, + [sourceScope.id, this.ownerId, sourceScope.epoch], + ) + if (transferred.rowCount !== 1) + throw failure('maintenance_activity_closed', 'Activity ownership changed before handoff') + } + await client.query('COMMIT') + operation._registered = true + if (sourceScope) { + sourceScope.active = false + sourceScope.handoffPending = false + sourceScope.transferredTo = operation + this._transfers.set(operation.operationId, sourceScope) + this._refs -= 1 + if (!this._refs) await this._releaseHolder() + } + } catch (error) { + await client.query('ROLLBACK').catch(() => {}) + throw error + } + }) // DRAINING is committed before waiting. Existing scopes never reacquire this lock for their SQL. await client.query("SELECT set_config('statement_timeout', $1, false)", [String(timeoutMs)]) const abort = () => { @@ -462,7 +650,7 @@ export class GlobalMaintenanceCoordinator { } operation._assertHealthy() await client.query("SELECT set_config('statement_timeout', '60000', false)") - await this._checkClaims(client) + await this._checkClaims(client, operation.operationId) return operation } catch (error) { await operation.close() @@ -480,6 +668,7 @@ export class GlobalMaintenanceCoordinator { if (!(await client.query(TRY_EXCLUSIVE, [this.adminResource])).rows[0].acquired) { throw failure('maintenance_busy', 'The maintenance owner is still active') } + await client.query('BEGIN') const result = await client.query( `UPDATE ${this.qualified}.control SET phase = 'open', operation_id = NULL, owner_id = NULL, kind = NULL, reason = NULL, changed_at = clock_timestamp() @@ -488,8 +677,16 @@ export class GlobalMaintenanceCoordinator { ) if (result.rowCount !== 1) throw failure('maintenance_operation_changed', 'The maintenance operation cannot be cancelled') + await client.query( + `UPDATE ${this.qualified}.operations SET ended_at=clock_timestamp() + WHERE id=$1 AND owner_id=$2 AND ended_at IS NULL`, + [operationId, this.ownerId], + ) + await client.query('COMMIT') + await this._resumeTransfer(operationId) return this.getState() } finally { + await client.query('ROLLBACK').catch(() => {}) await client.end().catch(() => {}) } } @@ -531,6 +728,8 @@ class MaintenanceOperation { this.backendPid = null this._closed = false this._lost = false + this._registered = false + this._resumed = false client.on('error', () => { this._lost = true }) @@ -539,6 +738,37 @@ class MaintenanceOperation { }) } + get healthy() { + return !this._closed && !this._lost && this.coordinator.healthy + } + + async runAdmin(callback) { + this._assertHealthy() + const scope = { + coordinator: this.coordinator, + kind: 'maintenance', + operation: this, + active: true, + pending: new Set(), + } + try { + return await activities.run({ scope, parent: activities.getStore() }, callback) + } finally { + scope.active = false + } + } + + async finish(sharedResult) { + this._assertHealthy() + if (!this._resumed) throw failure('maintenance_operation_changed', 'Resume before completing maintenance history') + await this.client.query( + `UPDATE ${this.coordinator.qualified}.operations SET ended_at=COALESCE(ended_at,clock_timestamp()), + shared_result=CASE WHEN $3::jsonb IS NULL THEN shared_result ELSE $3::jsonb END + WHERE id=$1 AND owner_id=$2`, + [this.operationId, this.coordinator.ownerId, sharedResult === undefined ? null : JSON.stringify(sharedResult)], + ) + } + _assertHealthy() { this.coordinator._assertHealthy() if (this._closed || this._lost) @@ -568,7 +798,7 @@ class MaintenanceOperation { return this._transaction(async (row) => { if (this.kind !== 'restore' || row.phase !== 'draining') throw failure('maintenance_operation_changed', 'Restore is not in its pre-restore stage') - await this.coordinator._checkClaims(this.client) + await this.coordinator._checkClaims(this.client, this.operationId) if (!Number.isSafeInteger(Number(row.epoch) + 1)) throw new Error('application epoch exhausted') const result = await this.client.query(`UPDATE ${this.coordinator.qualified}.control SET phase = 'restoring', epoch = epoch + 1, changed_at = clock_timestamp() WHERE id = 1 RETURNING *`) @@ -594,7 +824,7 @@ class MaintenanceOperation { ) { throw failure('maintenance_operation_changed', 'The maintenance phase does not permit resume') } - await this.coordinator._checkClaims(this.client) + await this.coordinator._checkClaims(this.client, this.operationId) const pending = ( await this.client.query( `SELECT owner_id FROM ${this.coordinator.qualified}.members @@ -612,6 +842,8 @@ class MaintenanceOperation { return stateOf(result.rows[0]) }) await this.client.query(UNLOCK_EXCLUSIVE, [this.coordinator.activityResource]) + await this.coordinator._resumeTransfer(this.operationId) + this._resumed = true return state } @@ -628,8 +860,12 @@ class MaintenanceOperation { async close() { if (this._closed) return - this._closed = true - // Closing releases session locks, never the durable latch or unfinished claims. - await this.client.end().catch(() => {}) + try { + if (this._registered && this._resumed && this.healthy && !this._completionUnknown) await this.finish() + } finally { + this._closed = true + // A pre-restore or failed owner retains its latch and unfinished operation history. + await this.client.end().catch(() => {}) + } } } diff --git a/test/unit/global-maintenance.test.mjs b/test/unit/global-maintenance.test.mjs index a8c31476f..c38455c8e 100644 --- a/test/unit/global-maintenance.test.mjs +++ b/test/unit/global-maintenance.test.mjs @@ -67,7 +67,7 @@ test('coordination DDL belongs only to a private schema derived from this applic [schema], ) ).rows.map((row) => row.table_name) - assert.deepEqual(tables, ['activities', 'control', 'members']) + assert.deepEqual(tables, ['activities', 'control', 'members', 'operations']) assert.equal((await db.pool.query('SELECT to_regclass($1) AS name', [`${db.schema}.activities`])).rows[0].name, null) }) From f0d2172f033003e87575d8b490ec72149a3caad8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:48:12 +0800 Subject: [PATCH 208/473] test: reproduce panel VM mutations bypassing shared operation claims --- test/unit/panel-vm-shared-mutations.test.mjs | 168 +++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 test/unit/panel-vm-shared-mutations.test.mjs diff --git a/test/unit/panel-vm-shared-mutations.test.mjs b/test/unit/panel-vm-shared-mutations.test.mjs new file mode 100644 index 000000000..bd4944fc5 --- /dev/null +++ b/test/unit/panel-vm-shared-mutations.test.mjs @@ -0,0 +1,168 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' + +async function fixture(t) { + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-shared-vm-')) + fs.mkdirSync(path.join(project, 'vms')) + const bin = path.join(project, 'bin') + fs.mkdirSync(bin) + // Baseline lifecycle paths must never reach the real Docker daemon. + fs.writeFileSync(path.join(bin, 'docker'), '#!/bin/sh\necho "Error: No such object: synthetic" >&2\nexit 1\n', { + mode: 0o700, + }) + const saved = { mode: process.env.VM2API_CONTROL_PLANE_MODE, path: process.env.PATH } + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + process.env.PATH = bin + path.delimiter + process.env.PATH + t.after(() => { + if (saved.mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = saved.mode + process.env.PATH = saved.path + fs.rmSync(project, { recursive: true, force: true }) + }) + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => peer.close()) + const operations = new SharedVmOperations(db, { ownerId: 'pod-a' }) + const otherOperations = new SharedVmOperations(peer, { ownerId: 'pod-b' }) + function seed(id, extra = {}) { + fs.writeFileSync( + path.join(project, 'vms', id + '.json'), + JSON.stringify({ id, name: id, status: 'running', schedulable: true, policy: { weight: 1 }, ...extra }), + ) + } + function read(id) { + return JSON.parse(fs.readFileSync(path.join(project, 'vms', id + '.json'), 'utf8')) + } + function handler(body = {}, extra = {}) { + const response = {} + const handle = createPanelHandler({ + cfg: { paths: { project } }, + routingConfig: {}, + vmOperations: operations, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + json(_res, status, payload) { + response.status = status + response.body = payload + return true + }, + readBody: async () => body, + proxyPool: { + unbindVm: async () => {}, + ensureBoundToVm: async () => null, + allocateForVm: async () => null, + getProxyForVm: () => null, + }, + poolScheduler: { vmInflight: new Map() }, + accountQuota: { snapshot: () => ({ accounts: [] }) }, + vmTestSchedule: { remove: async () => {} }, + ...extra, + }) + return { response, request: (method, route) => handle({ method }, {}, new URL('http://localhost' + route)) } + } + return { db, peer, project, seed, read, operations, otherOperations, handler } +} + +for (const [method, suffix, body] of [ + ['PATCH', '', { weight: 8 }], + ['POST', '/reload', {}], + ['POST', '/start', {}], + ['POST', '/recover', {}], + ['POST', '/stop', {}], + ['DELETE', '', {}], + ['POST', '/schedulable', { schedulable: false }], +]) { + test(`panel ${method} VM${suffix} rejects a global execution admission before any mutation`, async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + const before = f.read('vm-synthetic') + await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + vmId: 'vm-synthetic', + accountId: 'synthetic', + accountLimit: 2, + vmLimit: 2, + rpmLimit: 0, + fableLimit: 0, + }) + const h = f.handler(body) + await h.request(method, '/api/panel/vms/vm-synthetic' + suffix) + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(h.response.body.error.code, 'vm_execution_active') + assert.deepEqual(f.read('vm-synthetic'), before) + }) +} + +test('create claims the new physical identity before writing a VM or fingerprint', async (t) => { + const f = await fixture(t) + await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' }) + const h = f.handler({ id: 'synthetic', start: false, auto_allocate_proxy: false }) + await h.request('POST', '/api/panel/vms/create') + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(fs.existsSync(path.join(f.project, 'vms', 'synthetic.json')), false) + assert.equal(fs.existsSync(path.join(f.project, 'vms', 'synthetic')), false) +}) + +test('create rejects a committed PG identity even when this Pod has no corresponding file', async (t) => { + const f = await fixture(t) + await new VmsRepo(f.peer).upsertFromVmJson({ id: 'vm-synthetic', status: 'stopped' }) + const h = f.handler({ id: 'synthetic', start: false, auto_allocate_proxy: false }) + await h.request('POST', '/api/panel/vms/create') + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(fs.existsSync(path.join(f.project, 'vms', 'synthetic.json')), false) +}) + +test('hold cannot report global drain from an empty local inflight map', async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + vmId: 'vm-synthetic', + accountId: 'synthetic', + accountLimit: 2, + vmLimit: 2, + rpmLimit: 0, + fableLimit: 0, + }) + const h = f.handler({}) + await h.request('POST', '/api/panel/vms/schedule/hold') + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(f.read('vm-synthetic').schedulable, true) +}) + +test('batch policy acquires every claim before writing and releases earlier unstarted claims on busy', async (t) => { + const f = await fixture(t) + f.seed('vm-a') + f.seed('vm-z') + await f.otherOperations.tryAcquire({ vmId: 'vm-z' }) + const h = f.handler({ ids: ['vm-z', 'vm-a'], weight: 8 }) + await h.request('POST', '/api/panel/vms/pool-policy') + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(f.read('vm-a').policy.weight, 1) + assert.equal(f.read('vm-z').policy.weight, 1) + const first = await f.otherOperations.tryAcquire({ vmId: 'vm-a' }) + assert.equal(first.ok, true) + await first.release() +}) + +test('a normal non-ok stop result preserves its durable claim', async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + const h = f.handler( + {}, + { vmMutationOps: { stopSlot: async () => ({ ok: false, code: 'synthetic_unknown', vmOperationUncertain: true }) } }, + ) + await h.request('POST', '/api/panel/vms/vm-synthetic/stop') + assert.equal(h.response.status, 500) + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' })).ok, false) +}) From caef01423ab87c50bdb5751014bfe92c0488f622 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:51:30 +0800 Subject: [PATCH 209/473] test(slotgate): specify durable executor generation authority --- worker/internal/slotgate/authority_test.go | 356 +++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 worker/internal/slotgate/authority_test.go diff --git a/worker/internal/slotgate/authority_test.go b/worker/internal/slotgate/authority_test.go new file mode 100644 index 000000000..f60523e34 --- /dev/null +++ b/worker/internal/slotgate/authority_test.go @@ -0,0 +1,356 @@ +package slotgate + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func testExecutor(n string) ExecutorIdentity { + return ExecutorIdentity{Kind: "container", ContainerID: strings.Repeat(n, 64), StartedAt: "2026-10-21T00:00:00Z", NodeBootID: "test-node-boot"} +} + +func terminatedExecutor(_ context.Context, executor ExecutorIdentity) (TerminationProof, error) { + return TerminationProof{ID: "proof-1", Executor: executor, Scope: "container_removed", ObservedAt: "2026-10-21T00:01:00Z"}, nil +} + +func authorityForTest(t *testing.T, root string, verify TerminationVerifier) *Authority { + t.Helper() + a, err := OpenAuthority(root, verify) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + return a +} + +func launchForTest(t *testing.T, a *Authority, vm string, executor ExecutorIdentity) AuthorityState { + t.Helper() + s, err := a.CreateSlot(vm) + if err != nil { + t.Fatal(err) + } + launch, err := a.BeginLaunch(vm, s.Fence, "launch-"+vm) + if err != nil { + t.Fatal(err) + } + s, err = a.BindExecutor(launch, executor) + if err != nil { + t.Fatal(err) + } + return s +} + +func requireAuthorityError(t *testing.T, err, want error) { + t.Helper() + if !errors.Is(err, want) { + t.Fatalf("error = %v, want %v", err, want) + } +} + +func TestAuthorityAllowsConcurrentControllersWithinOneIncarnation(t *testing.T) { + a := authorityForTest(t, t.TempDir(), terminatedExecutor) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + first, err := a.BeginCommand("vm-1", s.Fence, "controller-a-request", CommandForward) + if err != nil { + t.Fatal(err) + } + second, err := a.BeginCommand("vm-1", s.Fence, "controller-b-request", CommandForward) + if err != nil { + t.Fatalf("another controller must share the active incarnation: %v", err) + } + if err = a.CompleteCommand(first); err != nil { + t.Fatal(err) + } + if err = a.CompleteCommand(second); err != nil { + t.Fatal(err) + } + if _, err = a.BeginCommand("vm-1", s.Fence, first.ID, CommandForward); !errors.Is(err, ErrCommandExists) { + t.Fatalf("completed command was replayed: %v", err) + } +} + +func TestAuthorityRevocationRequiresProofAndRejectsCapturedOldCommands(t *testing.T) { + a := authorityForTest(t, t.TempDir(), terminatedExecutor) + old := launchForTest(t, a, "vm-1", testExecutor("a")) + pending, err := a.BeginCommand("vm-1", old.Fence, "old-request", CommandForward) + if err != nil { + t.Fatal(err) + } + if _, err = a.Revoke("vm-1", old.Fence); err != nil { + t.Fatal(err) + } + _, err = a.BeginCommand("vm-1", old.Fence, "late-old-command", CommandMutation) + requireAuthorityError(t, err, ErrAuthorityPhase) + next, err := a.Advance(context.Background(), "vm-1", old.Fence) + if err != nil { + t.Fatal(err) + } + if next.Phase != PhaseVacant || next.Fence.Generation != old.Fence.Generation+1 || next.Fence.Incarnation == old.Fence.Incarnation || next.Fence.Executor.Kind != "" { + t.Fatalf("new incarnation must be vacant: %+v", next) + } + if next.Commands[pending.ID].Status != CommandTerminated || next.LastTermination == nil { + t.Fatal("verified termination must retain the old command and proof") + } + _, err = a.BeginCommand("vm-1", old.Fence, "delayed-after-takeover", CommandMutation) + requireAuthorityError(t, err, ErrFenceMismatch) + requireAuthorityError(t, a.CompleteCommand(pending), ErrFenceMismatch) + launch, err := a.BeginLaunch("vm-1", next.Fence, "launch-new") + if err != nil { + t.Fatal(err) + } + active, err := a.BindExecutor(launch, testExecutor("b")) + if err != nil { + t.Fatal(err) + } + _, err = a.BeginCommand("vm-1", active.Fence, pending.ID, CommandForward) + requireAuthorityError(t, err, ErrCommandExists) +} + +func TestAuthorityRejectsUnverifiedOrWrongExecutorProof(t *testing.T) { + for _, tc := range []struct { + name string + verify TerminationVerifier + }{ + {"missing", nil}, + {"failure", func(context.Context, ExecutorIdentity) (TerminationProof, error) { + return TerminationProof{}, errors.New("termination unconfirmed") + }}, + {"wrong identity", func(ctx context.Context, _ ExecutorIdentity) (TerminationProof, error) { + return terminatedExecutor(ctx, testExecutor("b")) + }}, + {"cancel ack", func(_ context.Context, e ExecutorIdentity) (TerminationProof, error) { + return TerminationProof{ID: "ack", Executor: e, Scope: "cancel_ack", ObservedAt: "2026-10-21T00:01:00Z"}, nil + }}, + } { + t.Run(tc.name, func(t *testing.T) { + a := authorityForTest(t, t.TempDir(), tc.verify) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + if _, err := a.Revoke("vm-1", s.Fence); err != nil { + t.Fatal(err) + } + if _, err := a.Advance(context.Background(), "vm-1", s.Fence); err == nil { + t.Fatal("unverified termination advanced generation") + } + got, err := a.State("vm-1") + if err != nil || got.Fence != s.Fence || got.Phase != PhaseRevoking { + t.Fatalf("state changed without proof: %+v, %v", got, err) + } + }) + } +} + +func TestAuthorityPendingMutationCannotBeClearedByExecutorTermination(t *testing.T) { + verified := 0 + a := authorityForTest(t, t.TempDir(), func(ctx context.Context, e ExecutorIdentity) (TerminationProof, error) { + verified++ + return terminatedExecutor(ctx, e) + }) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + command, err := a.BeginCommand("vm-1", s.Fence, "docker-restart", CommandMutation) + if err != nil { + t.Fatal(err) + } + if _, err = a.Revoke("vm-1", s.Fence); err != nil { + t.Fatal(err) + } + _, err = a.Advance(context.Background(), "vm-1", s.Fence) + requireAuthorityError(t, err, ErrPendingMutation) + if verified != 0 { + t.Fatal("executor termination does not settle a delayed Docker operation") + } + if err = a.CompleteCommand(command); err != nil { + t.Fatal(err) + } + if _, err = a.Advance(context.Background(), "vm-1", s.Fence); err != nil { + t.Fatal(err) + } +} + +func TestAuthorityRestartRetainsPendingWorkAndRejectsOldDaemonCompletion(t *testing.T) { + for _, kind := range []CommandKind{CommandForward, CommandMutation} { + t.Run(string(kind), func(t *testing.T) { + root := t.TempDir() + a := authorityForTest(t, root, terminatedExecutor) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + command, err := a.BeginCommand("vm-1", s.Fence, "pending", kind) + if err != nil { + t.Fatal(err) + } + if err = a.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, root, terminatedExecutor) + got, err := reopened.State("vm-1") + if err != nil || got.Phase != PhaseRevoking || got.Commands[command.ID].Status != CommandUnknown { + t.Fatalf("restart admitted unknown work: %+v, %v", got, err) + } + _, err = reopened.BeginCommand("vm-1", s.Fence, "after-restart", CommandForward) + requireAuthorityError(t, err, ErrAuthorityPhase) + requireAuthorityError(t, reopened.CompleteCommand(command), ErrCommandOwner) + _, err = reopened.Advance(context.Background(), "vm-1", s.Fence) + if kind == CommandMutation { + requireAuthorityError(t, err, ErrPendingMutation) + } else if err != nil { + t.Fatal(err) + } + }) + } +} + +func TestAuthorityRestartNeverReplaysLaunchWithUnknownImmutableIdentity(t *testing.T) { + root := t.TempDir() + a := authorityForTest(t, root, terminatedExecutor) + s, err := a.CreateSlot("vm-1") + if err != nil { + t.Fatal(err) + } + launch, err := a.BeginLaunch("vm-1", s.Fence, "launch-may-have-started") + if err != nil { + t.Fatal(err) + } + if err = a.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, root, terminatedExecutor) + _, err = reopened.BindExecutor(launch, testExecutor("a")) + requireAuthorityError(t, err, ErrCommandOwner) + _, err = reopened.Advance(context.Background(), "vm-1", s.Fence) + requireAuthorityError(t, err, ErrPendingMutation) +} + +func TestAuthorityCompletedStatePersistsAndSnapshotsCannotModifyAuthority(t *testing.T) { + root := t.TempDir() + a := authorityForTest(t, root, terminatedExecutor) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + command, err := a.BeginCommand("vm-1", s.Fence, "completed", CommandForward) + if err != nil { + t.Fatal(err) + } + if err = a.CompleteCommand(command); err != nil { + t.Fatal(err) + } + snapshot, err := a.State("vm-1") + if err != nil { + t.Fatal(err) + } + delete(snapshot.Commands, command.ID) + if err = a.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, root, terminatedExecutor) + got, err := reopened.State("vm-1") + if err != nil || got.Phase != PhaseActive || got.Fence != s.Fence || got.Commands[command.ID].Status != CommandCompleted { + t.Fatalf("completed state lost: %+v, %v", got, err) + } + _, err = reopened.BeginCommand("vm-1", s.Fence, command.ID, CommandForward) + requireAuthorityError(t, err, ErrCommandExists) +} + +func TestAuthorityEveryMutationChecksCapturedGenerationIncarnationAndIdentity(t *testing.T) { + a := authorityForTest(t, t.TempDir(), terminatedExecutor) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + for _, alter := range []func(*Fence){ + func(f *Fence) { f.Generation++ }, + func(f *Fence) { f.Incarnation = "another-incarnation" }, + func(f *Fence) { f.Executor = testExecutor("b") }, + } { + wrong := s.Fence + alter(&wrong) + _, err := a.BeginCommand("vm-1", wrong, "stale", CommandMutation) + requireAuthorityError(t, err, ErrFenceMismatch) + _, err = a.Revoke("vm-1", wrong) + requireAuthorityError(t, err, ErrFenceMismatch) + _, err = a.Advance(context.Background(), "vm-1", wrong) + requireAuthorityError(t, err, ErrFenceMismatch) + } +} + +func TestAuthoritySerializesSameNodeJournalAcrossProcesses(t *testing.T) { + root := t.TempDir() + a := authorityForTest(t, root, terminatedExecutor) + _, err := OpenAuthority(root, terminatedExecutor) + requireAuthorityError(t, err, ErrAuthorityLocked) + if err = a.Close(); err != nil { + t.Fatal(err) + } + authorityForTest(t, root, terminatedExecutor) +} + +func TestAuthorityInvalidJournalIsNotReinitialized(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "authority.json"), []byte(`{"version":1,"slots":`), 0o600); err != nil { + t.Fatal(err) + } + _, err := OpenAuthority(root, terminatedExecutor) + if err == nil { + t.Fatal("corrupt authority was treated as a new node") + } + data, readErr := os.ReadFile(filepath.Join(root, "authority.json")) + if readErr != nil || string(data) != `{"version":1,"slots":` { + t.Fatalf("invalid journal overwritten: %q, %v", data, readErr) + } +} + +func TestAuthorityDurabilityFailurePreventsAuthorizationAndPoisonsInstance(t *testing.T) { + root := t.TempDir() + a := authorityForTest(t, root, terminatedExecutor) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + file := filepath.Join(root, "authority.json") + if err := os.Rename(file, file+".saved"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(file, 0o700); err != nil { + t.Fatal(err) + } + if _, err := a.BeginCommand("vm-1", s.Fence, "must-not-send", CommandForward); err == nil { + t.Fatal("command authorized before durable commit") + } + if err := os.Remove(file); err != nil { + t.Fatal(err) + } + if err := os.Rename(file+".saved", file); err != nil { + t.Fatal(err) + } + _, err := a.BeginCommand("vm-1", s.Fence, "still-must-not-send", CommandForward) + requireAuthorityError(t, err, ErrAuthorityUnavailable) +} + +func TestAuthoritySlowTerminationVerificationDoesNotSerializeOtherVMs(t *testing.T) { + entered, proceed := make(chan struct{}), make(chan struct{}) + a := authorityForTest(t, t.TempDir(), func(ctx context.Context, e ExecutorIdentity) (TerminationProof, error) { + close(entered) + <-proceed + return terminatedExecutor(ctx, e) + }) + first := launchForTest(t, a, "vm-1", testExecutor("a")) + second := launchForTest(t, a, "vm-2", testExecutor("b")) + if _, err := a.Revoke("vm-1", first.Fence); err != nil { + t.Fatal(err) + } + advanced := make(chan error, 1) + go func() { _, err := a.Advance(context.Background(), "vm-1", first.Fence); advanced <- err }() + <-entered + commandDone := make(chan error, 1) + go func() { + _, err := a.BeginCommand("vm-2", second.Fence, "parallel-vm", CommandForward) + commandDone <- err + }() + select { + case err := <-commandDone: + close(proceed) + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + close(proceed) + t.Fatal("another VM waited behind termination verification") + } + if err := <-advanced; err != nil { + t.Fatal(err) + } +} From a12219f9afebc0b3c60ba418c7597927d94fe23e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:51:40 +0800 Subject: [PATCH 210/473] test: require backup service ownership transfer and private scheduling --- test/unit/maintenance-backup.test.mjs | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 test/unit/maintenance-backup.test.mjs diff --git a/test/unit/maintenance-backup.test.mjs b/test/unit/maintenance-backup.test.mjs new file mode 100644 index 000000000..f842712a2 --- /dev/null +++ b/test/unit/maintenance-backup.test.mjs @@ -0,0 +1,74 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import pg from 'pg' +import { getDb, registerActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { BackupService } from '../../src/lib/admin/backup-service.mjs' + +async function fixture(t) { + const db = getDb() + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + registerActivityCoordinator(db, coordinator) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'maintenance-backup-')) + const service = new BackupService({ dataDir: path.join(root, 'data'), projectRoot: root, sharedTasks: null }) + service._runBackupWorker = async (job, _database, copied) => { + await copied() + return { id: job.id, created_at: job.createdAt, kind: job.kind, status: 'ok' } + } + t.after(async () => { + await coordinator.stop().catch(() => {}) + fs.rmSync(root, { recursive: true, force: true }) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + return { db, coordinator, service, q: quoteIdentifier(coordinator.schema) } +} + +test('BackupService forwards exact maintenance ownership to local drain and retains the HTTP scope for response writes', async (t) => { + const { db, coordinator, service, q } = await fixture(t) + const request = await coordinator.admit({ kind: 'http' }) + const observed = [] + service.onMaintenance(async (enabled, _reason, context) => { + if (enabled) { + assert.equal(context?.operation?.sourceActivityId, request.activityId) + observed.push(context.operation.operationId) + await db.query('SELECT 1') + } + }) + const record = await request.run(() => service.createBackup()) + assert.equal(record.status, 'ok') + assert.equal(observed.length, 1) + const operation = (await db.pool.query(`SELECT ended_at FROM ${q}.operations WHERE id=$1`, [observed[0]])).rows[0] + assert.ok(operation.ended_at) + await request.run(() => db.query('SELECT 1')) + await request.complete({ ended: true }) +}) + +test('BackupService scheduled checks finish their read activity before claiming a private maintenance operation', async (t) => { + const { db, coordinator, service, q } = await fixture(t) + const result = await service.runScheduled() + assert.equal(result.status, 'ok') + assert.equal( + (await db.pool.query("SELECT count(*) AS n FROM background_task_runs WHERE task_key='scheduled-backup'")).rows[0].n, + 0, + ) + assert.equal( + ( + await db.pool.query( + `SELECT count(*) AS n FROM ${q}.operations WHERE task_key='scheduled-backup' AND ended_at IS NOT NULL`, + ) + ).rows[0].n, + 1, + ) + assert.equal(await coordinator.isReady(), true) +}) From bb05477ba4a9fff6277e374f62ca27a83a091eb5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:55:14 +0800 Subject: [PATCH 211/473] test: reproduce stale execution candidates bypassing committed VM holds --- test/unit/execution-vm-schedulable.test.mjs | 62 +++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 test/unit/execution-vm-schedulable.test.mjs diff --git a/test/unit/execution-vm-schedulable.test.mjs b/test/unit/execution-vm-schedulable.test.mjs new file mode 100644 index 000000000..247aca85c --- /dev/null +++ b/test/unit/execution-vm-schedulable.test.mjs @@ -0,0 +1,62 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' + +const input = (vmId) => ({ vmId, accountId: 'synthetic', accountLimit: 2, vmLimit: 2, rpmLimit: 0, fableLimit: 0 }) + +async function fixture(t) { + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => peer.close()) + return { + db, + peer, + vms: new VmsRepo(peer), + admissions: new ExecutionAdmissionsRepo(db), + operations: new SharedVmOperations(peer), + } +} + +test('a candidate read before a completed hold cannot reserve after its claim is released', async (t) => { + const f = await fixture(t) + await f.vms.upsertFromVmJson({ id: 'vm-synthetic', status: 'running', schedulable: true }) + const candidate = input('vm-synthetic') + const hold = await f.operations.tryAcquire({ vmId: candidate.vmId }) + await f.vms.upsertFromVmJson({ id: candidate.vmId, status: 'running', schedulable: false }) + await hold.release() + const result = await f.admissions.tryReserve(candidate) + assert.equal(result.ok, false) + assert.equal(result.code, 'vm_not_schedulable') +}) + +test('every physical alias record must be schedulable regardless of node placement', async (t) => { + const f = await fixture(t) + await f.vms.upsertFromVmJson({ id: 'vm-synthetic', node_id: 'node-a', status: 'running', schedulable: true }) + await f.vms.upsertFromVmJson({ id: 'synthetic', node_id: 'node-b', status: 'running', schedulable: false }) + for (const id of ['synthetic', 'vm-synthetic']) { + const result = await f.admissions.tryReserve(input(id)) + assert.equal(result.ok, false) + assert.equal(result.code, 'vm_not_schedulable') + } +}) + +test('re-enabling the committed record admits the next request', async (t) => { + const f = await fixture(t) + await f.vms.upsertFromVmJson({ id: 'vm-synthetic', status: 'running', schedulable: false }) + assert.equal((await f.admissions.tryReserve(input('vm-synthetic'))).ok, false) + await f.vms.upsertFromVmJson({ id: 'vm-synthetic', status: 'running', schedulable: true }) + const result = await f.admissions.tryReserve(input('vm-synthetic')) + assert.equal(result.ok, true) + await result.release() +}) + +test('repo-only callers without mirrored VM metadata keep their existing admission behavior', async (t) => { + const f = await fixture(t) + const result = await f.admissions.tryReserve(input('vm-synthetic')) + assert.equal(result.ok, true) + await result.release() +}) From 8af06ed63569e8539c710329fec6a6bcc37d4af6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:56:22 +0800 Subject: [PATCH 212/473] test: reproduce CLI guard races across controllers --- test/unit/cli-node-guard.test.mjs | 8 +++ test/unit/shared-cli-node-guard.test.mjs | 86 ++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 test/unit/shared-cli-node-guard.test.mjs diff --git a/test/unit/cli-node-guard.test.mjs b/test/unit/cli-node-guard.test.mjs index eaf07c6a9..e755cc9f3 100644 --- a/test/unit/cli-node-guard.test.mjs +++ b/test/unit/cli-node-guard.test.mjs @@ -78,6 +78,14 @@ test('kills leaked CLIs from closed panel shells and unmarked -p runs', async (t assert.equal(alive(stray), false) }) +test('shared guard retains a panel CLI whose remote owner cannot be disproved', async (t) => { + const slot = slotFixture(t) + const panel = slot.add(['-p', 'hello'], ['KIN_PANEL_SHELL=other-controller']) + slot.run(['--preserve-panel']) + await settle() + assert.equal(alive(panel), true) +}) + test('keeps a not-yet-synced kernel CLI that still has the prefixed slots env', async (t) => { const slot = slotFixture(t) const kernel = slot.add(['-p'], ['CLAUDE_CODE_KIN_NATIVE_SLOTS=20']) diff --git a/test/unit/shared-cli-node-guard.test.mjs b/test/unit/shared-cli-node-guard.test.mjs new file mode 100644 index 000000000..370555699 --- /dev/null +++ b/test/unit/shared-cli-node-guard.test.mjs @@ -0,0 +1,86 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { createCliNodeGuard } from '../../src/lib/vm/cli-node-guard.mjs' + +function guard(exec, vmId = 'vm-shared-guard') { + const db = getDb() + return createCliNodeGuard({ + listTargets: () => [{ id: vmId, status: 'running' }], + sharedTasks: new SharedBackgroundTasks(db), + operations: new SharedVmOperations(db), + exec, + logger: { warn() {} }, + }) +} + +function reserve(vmId = 'vm-shared-guard') { + return new ExecutionAdmissionsRepo(getDb()).tryReserve({ + accountId: 'shared-guard-account', + vmId, + accountLimit: 2, + vmLimit: 2, + fableLimit: 0, + rpmLimit: 0, + }) +} + +test('a shared CLI guard cannot modify a VM with an admitted inference', async () => { + const active = await reserve() + assert.equal(active.ok, true) + let sent = 0 + await guard(async () => { + sent++ + return { ended: true, ok: true } + }).tick() + assert.equal(sent, 0) + await active.release() +}) + +test('shared CLI guards claim one plan across physical VM aliases', async () => { + let sent = 0 + const exec = async (_connect, _container, cmd) => { + assert.ok(cmd.includes('--preserve-panel')) + sent++ + return { ended: true, ok: true } + } + await Promise.all([guard(exec).tick(), guard(exec, 'shared-guard').tick()]) + assert.equal(sent, 1) +}) + +test('unconfirmed CLI guard completion retains mutation and background occupancy', async () => { + await guard(async () => undefined).tick() + assert.equal((await reserve()).code, 'vm_operation_active') + const row = await getDb().prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.ok(row.unknown_at) + assert.equal(row.finished_at, null) +}) + +test('CLI guard keeps the VM unavailable until its actual command has ended', async () => { + let entered + let finish + const started = new Promise((resolve) => { + entered = resolve + }) + const stopped = new Promise((resolve) => { + finish = resolve + }) + const running = guard(async () => { + entered() + await stopped + return { ended: true, ok: true } + }).tick() + await started + try { + assert.equal((await reserve()).code, 'vm_operation_active') + } finally { + finish() + await running + } + const next = await reserve() + assert.equal(next.ok, true) + await next.release() +}) From bffb8eca90b54d5b35422bdc7b6ee72aa8304543 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:58:00 +0800 Subject: [PATCH 213/473] fix: coordinate CLI cleanup and retain unconfirmed execution --- src/lib/vm/cli-node-guard.mjs | 76 +++++++++++++++++++++--- test/unit/shared-cli-node-guard.test.mjs | 58 +++++++++++++++++- 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/src/lib/vm/cli-node-guard.mjs b/src/lib/vm/cli-node-guard.mjs index d747773fb..9a3cf89b5 100644 --- a/src/lib/vm/cli-node-guard.mjs +++ b/src/lib/vm/cli-node-guard.mjs @@ -10,7 +10,10 @@ * from a closed terminal (cli-node ignores SIGHUP) and is killed. A process * whose environ cannot be read is left alone. */ -import { execDetached } from '../cluster/docker-remote.mjs' +import { execDetached, dockerJson, inspectExec } from '../cluster/docker-remote.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { SharedVmOperations } from './shared-vm-operations.mjs' +import { physicalContainerName } from './vm-physical-target.mjs' import { slotContainerName } from '../transport/rust-kernel-supervisor.mjs' import { isCodexVm } from './vm-kind.mjs' import { isKvmRuntime } from './runtime-kind.mjs' @@ -30,6 +33,8 @@ export function isCliNodeGuardTarget(vm) { // $0 is "guard"; "$@" are live KIN_PANEL_SHELL tokens. KIN_GUARD_PROC is a test seam. export const GUARD_SCRIPT = ` +preserve_panels=0 +if [ "\${1:-}" = --preserve-panel ]; then preserve_panels=1; shift; fi proc=\${KIN_GUARD_PROC:-/proc} keep= prefer= @@ -59,6 +64,7 @@ for d in "$proc"/[0-9]*; do printf '%s\\n' "$env" | grep -qx -e 'KIN_OFFICIAL_CC=1' -e 'KIN_SETUP_TOKEN=1' -e 'KIN_AUTH_LOGIN=1' && continue panel=$(printf '%s\\n' "$env" | sed -n 's/^KIN_PANEL_SHELL=//p' | head -n 1) if [ -n "$panel" ]; then + [ "$preserve_panels" = 1 ] && continue for token in "$@"; do [ "$token" = "$panel" ] && continue 2 done @@ -68,15 +74,49 @@ done exit 0 ` +/** A start response does not prove that the cleanup command has exited. */ +export async function runCliNodeGuardCommand(connect, container, cmd, { user, timeoutMs = 15000, pollMs = 100 } = {}) { + const created = await dockerJson( + connect, + { + method: 'POST', + path: `/containers/${encodeURIComponent(container)}/exec`, + body: { Cmd: cmd, AttachStdout: false, AttachStderr: false, ...(user ? { User: user } : {}) }, + }, + [201], + ) + const execId = created.Id + await dockerJson( + connect, + { method: 'POST', path: `/exec/${encodeURIComponent(execId)}/start`, body: { Detach: true } }, + [200], + ) + const deadline = Date.now() + timeoutMs + do { + const state = await inspectExec(connect, execId) + if (state.Running === false && Number.isInteger(state.ExitCode)) { + return { ended: true, ok: state.ExitCode === 0, execId, exitCode: state.ExitCode } + } + if (Date.now() >= deadline) break + await new Promise((resolve) => setTimeout(resolve, pollMs)) + } while (true) + return { ended: false, ok: false, execId } +} + export function createCliNodeGuard({ listTargets, liveTokens = () => [], - exec = execDetached, + exec = null, + sharedTasks, + operations = null, intervalMs = CLI_NODE_GUARD_INTERVAL_MS, logger = console, } = {}) { let timer = null let inTick = false + const tasks = backgroundTasksFor(sharedTasks) + const mutations = operations || (tasks ? new SharedVmOperations(tasks.db) : null) + const execute = exec || (tasks ? runCliNodeGuardCommand : execDetached) async function tick() { if (inTick) return @@ -91,11 +131,33 @@ export function createCliNodeGuard({ if (host.native) continue const connect = host.dockerApi() const container = slotContainerName({ vm }) - const { Cmd, User } = slotExecCmd(vm, ['/bin/sh', '-c', GUARD_SCRIPT, 'guard', ...tokens], { - detach: true, - user: isKvmRuntime(vm) ? '0' : undefined, - }) - await exec(connect, container, Cmd, { user: User }) + const { Cmd, User } = slotExecCmd( + vm, + ['/bin/sh', '-c', GUARD_SCRIPT, 'guard', ...(tasks ? ['--preserve-panel'] : []), ...tokens], + { + detach: true, + user: isKvmRuntime(vm) ? '0' : undefined, + }, + ) + if (!tasks) await execute(connect, container, Cmd, { user: User }) + else + await tasks.run({ taskKey: `cli_node_guard:${physicalContainerName(vm.id)}`, intervalMs }, async () => { + let result + try { + result = await mutations.run( + vm.id, + async () => { + const outcome = await execute(connect, container, Cmd, { user: User }) + return { ...outcome, vmOperationUncertain: outcome?.ended !== true } + }, + { kind: 'cli_node_guard' }, + ) + } catch (error) { + if (['vm_execution_active', 'vm_operation_active'].includes(error.code)) error.taskNotStarted = true + throw error + } + return { terminal: result.ended === true, value: result } + }) } catch (err) { logger.warn?.(`[cli-node-guard] ${vm.id} ${err?.message || err}`) } diff --git a/test/unit/shared-cli-node-guard.test.mjs b/test/unit/shared-cli-node-guard.test.mjs index 370555699..55339c3a9 100644 --- a/test/unit/shared-cli-node-guard.test.mjs +++ b/test/unit/shared-cli-node-guard.test.mjs @@ -1,10 +1,12 @@ import test from 'node:test' import assert from 'node:assert/strict' +import http from 'node:http' +import net from 'node:net' import { getDb } from '../../src/lib/db/database.mjs' import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' -import { createCliNodeGuard } from '../../src/lib/vm/cli-node-guard.mjs' +import { createCliNodeGuard, runCliNodeGuardCommand } from '../../src/lib/vm/cli-node-guard.mjs' function guard(exec, vmId = 'vm-shared-guard') { const db = getDb() @@ -84,3 +86,57 @@ test('CLI guard keeps the VM unavailable until its actual command has ended', as assert.equal(next.ok, true) await next.release() }) + +async function engine(t, states) { + const execId = 'a'.repeat(64) + const calls = [] + const server = http.createServer(async (req, res) => { + for await (const _chunk of req) { + /* consume the Docker request */ + } + calls.push(req.url) + let body + if (req.url.endsWith('/exec')) { + res.statusCode = 201 + body = { Id: execId } + } else if (req.url.endsWith('/start')) body = {} + else body = states.length > 1 ? states.shift() : states[0] + res.setHeader('content-type', 'application/json') + res.end(JSON.stringify(body)) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + t.after(async () => { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + }) + const connect = () => + new Promise((resolve, reject) => { + const socket = net.connect(server.address().port, '127.0.0.1') + socket.once('connect', () => resolve(socket)) + socket.once('error', reject) + }) + return { calls, connect, execId } +} + +test('the actual guard executor waits for immutable exec inspection after a start acknowledgement', async (t) => { + const fx = await engine(t, [ + { Running: true, ExitCode: 0 }, + { Running: false, ExitCode: 0 }, + ]) + const result = await runCliNodeGuardCommand(fx.connect, 'kin-synthetic', ['synthetic'], { pollMs: 1 }) + assert.equal(result.ended, true) + assert.equal(result.execId, fx.execId) + assert.deepEqual(fx.calls, [ + '/containers/kin-synthetic/exec', + `/exec/${fx.execId}/start`, + `/exec/${fx.execId}/json`, + `/exec/${fx.execId}/json`, + ]) +}) + +test('guard executor timeout leaves a running Docker exec unconfirmed', async (t) => { + const fx = await engine(t, [{ Running: true, ExitCode: 0 }]) + const result = await runCliNodeGuardCommand(fx.connect, 'kin-synthetic', ['synthetic'], { timeoutMs: 0 }) + assert.equal(result.ended, false) + assert.equal(result.ok, false) +}) From b942feb2a05a9a73ad5b8e5274bbfa9df971a5db Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:59:29 +0800 Subject: [PATCH 214/473] test: retain SQL work and cancel refused pre-restore drains --- test/unit/maintenance-backup.test.mjs | 27 +++++++++++++++++++++++++++ test/unit/maintenance-scopes.test.mjs | 11 ++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/test/unit/maintenance-backup.test.mjs b/test/unit/maintenance-backup.test.mjs index f842712a2..5f1bb063e 100644 --- a/test/unit/maintenance-backup.test.mjs +++ b/test/unit/maintenance-backup.test.mjs @@ -72,3 +72,30 @@ test('BackupService scheduled checks finish their read activity before claiming ) assert.equal(await coordinator.isReady(), true) }) + +test('a pre-restore refusal cancels only its own drain and leaves ordinary requests available', async (t) => { + const { db, coordinator, service } = await fixture(t) + fs.mkdirSync(service.dataDir, { recursive: true }) + const file = path.join(service.dataDir, 'synthetic.tar.gz') + fs.writeFileSync(file, 'synthetic validation fixture') + await db.pool.query( + "INSERT INTO backup_records(id,created_at,kind,status,file_path,file_name) VALUES ('restore-refused','now','manual','ok',$1,'synthetic.tar.gz')", + [file], + ) + await db.pool.query( + "INSERT INTO background_task_runs(task_key,execution_id,owner_id,unknown_at) VALUES ('unknown-remote','one','old-pod',clock_timestamp())", + ) + service._extractArchive = async () => ({ + tmp: fs.mkdtempSync(path.join(service.dataDir, 'extract-')), + tables: new Set(), + }) + const request = await coordinator.admit({ kind: 'http' }) + await request.run(() => + assert.rejects(service.restoreBackup('restore-refused'), { code: 'maintenance_claims_active' }), + ) + assert.equal((await coordinator.getState()).phase, 'open') + assert.equal(service.isMaintaining, false) + assert.equal(service.isRestoring, false) + assert.equal(fs.existsSync(service.maintenanceFile), false) + await request.complete({ ended: true }) +}) diff --git a/test/unit/maintenance-scopes.test.mjs b/test/unit/maintenance-scopes.test.mjs index 4c149a108..f0888e318 100644 --- a/test/unit/maintenance-scopes.test.mjs +++ b/test/unit/maintenance-scopes.test.mjs @@ -68,9 +68,14 @@ test('a transaction cannot reuse an activity that closed while its JavaScript ca ) transaction.catch(() => {}) await started - await gate.complete({ ended: true }) - release() - await assert.rejects(transaction, { code: 'maintenance_activity_closed' }) + try { + await assert.rejects(gate.complete({ ended: true }), { code: 'maintenance_persistence_pending' }) + await coordinator.stop() + } finally { + release() + await transaction.catch(() => {}) + } + await assert.rejects(transaction, { code: 'maintenance_connection_lost' }) assert.equal((await db.pool.query("SELECT count(*) AS n FROM settings WHERE key='closed-scope-write'")).rows[0].n, 0) }) From ab4f44cb469b762b1be7229033d27fef346f1bbe Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:59:43 +0800 Subject: [PATCH 215/473] docs: record maintenance regression checkpoints and remaining gaps --- ...-21-kubernetes-shared-snapshots-results.md | 14 +- .../maintenance-progress/verification.json | 258 ++++++++++++++++++ 2 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 docs/plans/evidence/kubernetes/maintenance-progress/verification.json diff --git a/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md b/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md index 6202f4580..6e582130e 100644 --- a/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md +++ b/docs/plans/2026-10-21-kubernetes-shared-snapshots-results.md @@ -55,7 +55,7 @@ Pod imageID 对应 manifest 中的 config digest，不能与 OCI manifest digest | `08998f406eb7a2bf7ac3512d4b20895879cc2829` | API endpoint 执行 hook、终态与 PG job：72/72 通过 | | `bdda2026dca0ed5905c70514bc3fad94e727a53e` | 固定源码归档上的请求 activity hook 接线：75/75 通过 | -前两项提交位于全量检查点之前。后两项在其后。Search 直接 HTTP 请求只有在响应体读取完成后才结束作业；401/403 重发前完成旧作业并使用新 ID。响应体截断、取消、超时及 PG 故障保留容量。Search 不增加原协议没有定义的同会话队列。 +前两项提交位于全量检查点之前，其余位于其后。Search 直接 HTTP 请求只有在响应体读取完成后才结束作业；401/403 重发前完成旧作业并使用新 ID。响应体截断、取消、超时及 PG 故障保留容量。Search 不增加原协议没有定义的同会话队列。 面板路由更新显式返回已提交 revision 和 application 状态。VM/kernel 投影未执行时返回 pending。PG 提交失败不回退到可能过期的本地配置，也不在提交前切换执行端引擎。 @@ -63,6 +63,18 @@ Pod imageID 对应 manifest 中的 config digest，不能与 OCI manifest digest `e854273a5d7016c160a24237ef9aeac3df0cf988` 修正 Go API proxy 在 `io.Copy` 失败时正常结束响应、隐瞒上游 body 截断的问题。`08998f40` 和 `bdda2026` 接入 API endpoint 及请求 activity 的发送前登记和读取结束确认。取消、超时、截断、`upstream_transport` 或 SSE 缺终态不释放未知执行。源码验证与现有 payload 反例见 [后续生命周期证据](evidence/kubernetes/singleton-6732a864/later-lifecycle.json)。这些修正及新的认证、后台任务/全局维护代码没有部署到 `6732a864` 候选镜像。 +## 全局维护和后台活动的后续回归 + +`4e5cc46870b6b4bbb69583eb123e2db2ab823c64` 的独立源码归档完成第二次全量检查。unit 为 333 文件、3860 项：3835 通过、15 失败、10 跳过。E2E 为 29 文件、124 项：106 通过、17 失败、1 跳过。E2E 的 17 个失败及 unit 中原有 7 个失败的断言文本与 `6732a864` 记录一致。 + +新增的 8 个 unit 失败没有被当作既有失败：4 个来自 API 准备阶段异常被转换为响应，1 个来自拦截顺序的静态断言，2 个计费失败 mock 缺少新的发送事件，1 个默认后台服务测试没有创建生产维护活动。修正后的固定源码 `2923a65ff170bb289356b2f5c5ae74afbcf84ac0` 在独立归档上的 230 项相关回归全部通过。该结果覆盖这 8 个失败及后台、集群调用链；没有替代修正后整套 unit 的重新执行。 + +这些增量为每次 HTTP、终端和后台数据库工作登记独立活动。数据库查询和事务必须处于对应连接的有效活动范围。配置轮询、日志清理、代理探测、文件监听、SSH 重连及集群事件等待各自的异步持久化。终端连接关闭不证明执行结束；SSH exit 或 Docker exec 明确停止才可确认结束。 + +`bffb8eca90b54d5b35422bdc7b6ee72aa8304543` 的 CLI 清理器增量在独立归档上 46/46 通过。Kubernetes 模式使用 PG 共享计划及 VM 物理目标互斥，活动推理期间不执行清理；启动 Docker exec 后检查不可变 exec ID 的 Running/ExitCode。未确认退出时保留后台与 VM 占用。其他控制面的面板进程无法仅靠本副本列表确认已结束，因此保守保留。该行为尚未提供孤立面板进程的自动回收。 + +上述验证只使用专用 PostgreSQL、合成 HTTP/Unix 上游、模拟节点以及测试自己创建的子进程。检查统计、失败摘要和日志 SHA-256 见 [维护增量证据](evidence/kubernetes/maintenance-progress/verification.json)。后续备份恢复接线、其他 VM 管理入口及执行端 generation 权威仍在实现。 + ## 未完成和未验证 - 后台任务共享执行、跨副本一次性 OAuth/终端 ticket、全局维护和请求 activity 的后续源码增量，尚未完成新镜像的部署验收。 diff --git a/docs/plans/evidence/kubernetes/maintenance-progress/verification.json b/docs/plans/evidence/kubernetes/maintenance-progress/verification.json new file mode 100644 index 000000000..540223b80 --- /dev/null +++ b/docs/plans/evidence/kubernetes/maintenance-progress/verification.json @@ -0,0 +1,258 @@ +{ + "checkpoint": "4e5cc46870b6b4bbb69583eb123e2db2ab823c64", + "full": { + "unit": { + "files": 333, + "totals": { + "tests": 3860, + "pass": 3835, + "fail": 15, + "cancelled": 0, + "skipped": 10 + }, + "failures": [ + { + "file": "test/unit/api-endpoints-shared.test.mjs", + "test": "API inference rejects a shared endpoint read failure before selecting a cached endpoint", + "error_sha256": "92efdc7c9dd07289907f459d36f835ef3a32af17a791977ae81c224d70181d70", + "matches_prior_error": false + }, + { + "file": "test/unit/claude-code-only.test.mjs", + "test": "switch on: official Claude Code and its own quota probe reach the scheduler", + "error_sha256": "7b37bbe628b2b4bd13d3914908e2b78d68b45d15a6c436810675f424167ed80b", + "matches_prior_error": false + }, + { + "file": "test/unit/claude-code-only.test.mjs", + "test": "switch on: local-agent desktop reaches the scheduler without relay detection", + "error_sha256": "7b37bbe628b2b4bd13d3914908e2b78d68b45d15a6c436810675f424167ed80b", + "matches_prior_error": false + }, + { + "file": "test/unit/claude-code-only.test.mjs", + "test": "switch off: third-party Claude requests are not gated", + "error_sha256": "7b37bbe628b2b4bd13d3914908e2b78d68b45d15a6c436810675f424167ed80b", + "matches_prior_error": false + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "error_sha256": "1afab608e2476a88783807d547c9b239ad4ac831679731a3ed4d44812b0c31d2", + "matches_prior_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_prior_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_prior_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "error_sha256": "7862d1144e6748d63aa6e98dabf5a9fa7d8d1582f7fbfe851d978eb15f6d5c92", + "matches_prior_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "error_sha256": "81b2ee3a4bb8b1ab4f351bede4398032dc9637c683b82b146630fd6857e94c2e", + "matches_prior_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "error_sha256": "01732f4307f58f479e9b3d8329d92a93d9c748c58028276022257d3645e77e87", + "matches_prior_error": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_prior_error": true + }, + { + "file": "test/unit/distill-detect.test.mjs", + "test": "handleProtocol intercepts before credential hop", + "error_sha256": "3bd760b3d1a2dd1343cbceabc0722582b8308ed3558fc834f718294610dc1306", + "matches_prior_error": false + }, + { + "file": "test/unit/protocol-settlement-failure.test.mjs", + "test": "responses preserves pending request quota after an execution-stage multiplier failure", + "error_sha256": "2720fa5a2a4c29a41cec7a61e642f1c062064d15d411fb9e0e62ab713559ec9e", + "matches_prior_error": false + }, + { + "file": "test/unit/protocol-settlement-failure.test.mjs", + "test": "responses preserves pending request quota after an execution-stage backend failure", + "error_sha256": "d1c2107abec3eca7836b548354ca51b0e27ed58838d2c6709b2d54d0817b1897", + "matches_prior_error": false + }, + { + "file": "test/unit/shared-background-producers.test.mjs", + "test": "Kubernetes health uses the default shared service and refreshes routing before executing", + "error_sha256": "0cb86e699bfa5eb5be05acb7875308b1824e201c12239636003b51a4102f433a", + "matches_prior_error": false + } + ], + "log_sha256": "28e5d205d04f29d3a23748559bc3554315e589ad51ad54626ddc072a154fcced" + }, + "e2e": { + "files": 29, + "totals": { + "tests": 124, + "pass": 106, + "fail": 17, + "cancelled": 0, + "skipped": 1 + }, + "failures": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "error_sha256": "a25cb59468e4decc45b1d8b3a66b0a4fbd2195af9f498da39aa83e92169825b5", + "matches_prior_error": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "error_sha256": "84e06146cf59485c3b46e05070f6a9a184ecb010a57ef8ab1b62a3ebae90c36a", + "matches_prior_error": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_prior_error": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "error_sha256": "dfd84d60931072a864eb94f48ce13d4223e963ae919898236f1859bedc8d917e", + "matches_prior_error": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_prior_error": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "error_sha256": "6ea819807eaee98e6a4c38817c9f011fb426e6a5ff499fa441d8f7bb07371461", + "matches_prior_error": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "error_sha256": "c23279127cad70a092d0cc7bb6ac400e05aff03e4614d1eaaa40252878c221c2", + "matches_prior_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "error_sha256": "97ae4156650fa0de2f613aa7753666781545cb2fe7818350a6df904ab696cdbc", + "matches_prior_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "error_sha256": "a2c6319267f1b752b63d96a41e15b1a5133952e88f0c145bc96dda53fd31cbe5", + "matches_prior_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_prior_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "error_sha256": "ba0ac5e90ea008839723c328e8b172228820837ab895e425d2190c1aa5caa708", + "matches_prior_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_prior_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "error_sha256": "8353154a58e84007d91c2d217c516dc771e359ddb43d527c7db6734ab5cff6ca", + "matches_prior_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "error_sha256": "085c13ab1e54fb324bce5520b12006fbf7f84b4a69a1c5cf9e09727b5b33da99", + "matches_prior_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_prior_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "error_sha256": "0be25feaedaa697c09835d15c8cfdae694de2710330cadfadcbdfa146625a099", + "matches_prior_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_prior_error": true + } + ], + "log_sha256": "f802515c0e465bee8cc7b2b2ad62ea81b60e49596b0cde9be94cfd3fbe746a55" + } + }, + "followup": [ + { + "commit": "2923a65ff170bb289356b2f5c5ae74afbcf84ac0", + "files": 1, + "totals": { + "tests": 230, + "pass": 230, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "15e835f63411a37acd3aae8af5ffdb28c82d30e835c18f66f1dd3b871a0c9479", + "log": "vm2api-648-2923a65f-regression.log" + }, + { + "commit": "bffb8eca90b54d5b35422bdc7b6ee72aa8304543", + "files": 1, + "totals": { + "tests": 46, + "pass": 46, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "151d6a0de01ce3cb9776238c2b80169eb31886311bd92c8a8fff2c44a2636a2c", + "log": "vm2api-648-bffb8eca-guard-postcommit.log" + } + ], + "limitations": [ + "Single active controller guard remains enabled", + "No multi-node scale or executor fencing acceptance", + "New source has not been deployed after the 6732a864 singleton smoke" + ] +} From 32cb2c2cb649ab2791eef7e1793aa966a06e27f3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 21:59:57 +0800 Subject: [PATCH 216/473] fix: guard panel VM lifecycle and policy changes with shared claims --- src/lib/admin/panel-routes.mjs | 1819 +++++++++-------- .../db/repos/execution-admissions-repo.mjs | 6 + src/lib/vm/vm-mutations.mjs | 39 + test/unit/panel-vm-delete.test.mjs | 1 + test/unit/panel-vm-shared-mutations.test.mjs | 138 ++ 5 files changed, 1165 insertions(+), 838 deletions(-) create mode 100644 src/lib/vm/vm-mutations.mjs diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 1da9b5a93..6a2b0cabb 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -173,6 +173,8 @@ import { makeError, ErrorType, ErrorCode } from '../core/errors.mjs' import * as panel from './panel-api.mjs' import { GATEWAY_CAPABILITIES } from '../vm/execution-context.mjs' import { withVmLock, atomicWriteJson, isValidVmId, invalidateVmSlotReadCache } from '../vm/vm-file.mjs' +import { withVmMutations } from '../vm/vm-mutations.mjs' +import { physicalVmAliases } from '../vm/vm-physical-target.mjs' import { snapshotDatabaseMetrics } from '../db/database-metrics.mjs' import { getUsageCache } from '../oauth/usage-cache.mjs' import { getDb } from '../db/database.mjs' @@ -614,6 +616,42 @@ export function createPanelHandler(ctx) { error: { code: 'database_persistence_failed', message: 'Database persistence failed; maintenance is required' }, }) } + async function mutateVms(ids, callback) { + if (!isKubernetesMode() && !ctx.vmOperations) return callback(new Map()) + return withVmMutations( + ctx.cfg.paths.project, + ids, + async (handles) => { + const result = await callback(handles) + try { + await flushPanelWrites() + } catch (error) { + error.panelPersistenceFailure = true + throw error + } + return result + }, + { operations: ctx.vmOperations }, + ) + } + async function vmEffect(operation, callback) { + let result + try { + result = await callback() + } catch (error) { + await operation?.markUncertain?.() + throw error + } + if ( + result?.vmOperationUncertain || + result?.rust_ok === false || + result?.report?.ok === false || + (result?.ok !== true && result?.notStarted !== true && result?.vmOperationNotStarted !== true) + ) { + await operation?.markUncertain?.() + } + return result + } const hostUnsupported = (res, vm, cap) => { if (slotHost(vm).supports(cap)) return false json(res, 409, { @@ -1049,7 +1087,7 @@ export function createPanelHandler(ctx) { async function applySlotPolicyWithRuntime(id, patch) { const current = getVm(cfg.paths.project, id) - if (!current) return { ok: false, id, code: 'vm_not_found', error: 'vm not found' } + if (!current) return { ok: false, id, code: 'vm_not_found', error: 'vm not found', notStarted: true } const desired = vmWithSlotPolicyPatch(current, patch) const previousEngine = resolveInferenceEngine(current, ctx.routingConfig) const targetEngine = resolveInferenceEngine(desired, ctx.routingConfig) @@ -2495,18 +2533,21 @@ export function createPanelHandler(ctx) { const targets = parseSlotPolicyTargets(body) if (!targets.ok) return json(res, 400, { ok: false, error: { message: targets.error } }) const ids = targets.all ? listVms(cfg.paths.project).map((vm) => vm.id) : targets.ids - const results = [] - for (const id of ids) results.push(await applySlotPolicyWithRuntime(id, parsed.patch)) - const failed = results.filter((item) => !item.ok) - const report = { - updated: results.length - failed.length, - missing: failed.filter((item) => item.code === 'vm_not_found').map((item) => item.id), - failed: failed.map(({ id, code, error }) => ({ id, code, error })), - items: results - .filter((item) => item.ok) - .map((item) => summarizeVm(item.vm, cfg.paths.project, ctx.routingConfig)), - } - return json(res, 200, panel.ok(report)) + return mutateVms(ids, async (handles) => { + const results = [] + for (const id of ids) + results.push(await vmEffect(handles.get(id), () => applySlotPolicyWithRuntime(id, parsed.patch))) + const failed = results.filter((item) => !item.ok) + const report = { + updated: results.length - failed.length, + missing: failed.filter((item) => item.code === 'vm_not_found').map((item) => item.id), + failed: failed.map(({ id, code, error }) => ({ id, code, error })), + items: results + .filter((item) => item.ok) + .map((item) => summarizeVm(item.vm, cfg.paths.project, ctx.routingConfig)), + } + return json(res, 200, panel.ok(report)) + }) } // POST /api/panel/vms/pool-policy — batch slot policy // ({ ids: [...] } or { all: true }, plus any of weight / schedule_level / @@ -2579,67 +2620,75 @@ export function createPanelHandler(ctx) { nextSessions = parsed.value } const ids = targets.all ? listVms(cfg.paths.project).map((vm) => vm.id) : targets.ids - const missing = [] - const failed = [] - const items = [] - for (const id of ids) { - try { - let record = null - if (hasWeight || hasLevel) { - record = patchVmRecord(cfg.paths.project, id, (current) => { - const policy = { ...(current.policy || {}) } - if (hasWeight) policy.weight = nextWeight - if (hasLevel) { - if (nextLevel == null) delete policy.priority - else policy.priority = nextLevel - } - return { policy } + return mutateVms(ids, async (handles) => { + const missing = [] + const failed = [] + const items = [] + for (const id of ids) { + try { + let record = null + if (hasWeight || hasLevel) { + record = patchVmRecord(cfg.paths.project, id, (current) => { + const policy = { ...(current.policy || {}) } + if (hasWeight) policy.weight = nextWeight + if (hasLevel) { + if (nextLevel == null) delete policy.priority + else policy.priority = nextLevel + } + return { policy } + }) + } else { + record = getVm(cfg.paths.project, id) + } + if (!record) { + missing.push(id) + continue + } + if (hasConc && nextConc !== null) applyVmConcurrency(id, nextConc, { override: true }) + if (hasRpm && nextRpm !== null) applyVmRpm(id, nextRpm, { override: true }) + if (hasSessions && nextSessions !== null) ctx.applyVmMaxSessions(id, nextSessions, { override: true }) + const inherit = {} + if (hasConc && nextConc === null) inherit.concurrency = true + if (hasRpm && nextRpm === null) inherit.rpm = true + if (hasSessions && nextSessions === null) inherit.maxSessions = true + if (Object.keys(inherit).length) inheritVmScheduling(id, inherit) + const fresh = getVm(cfg.paths.project, id) || record + const policy = fresh.policy || {} + items.push({ + id, + weight: Number(policy.weight ?? 1), + schedule_level: manualScheduleLevelOf(fresh) ?? 'auto', + max_concurrency: policy.maxConcurrency ?? null, + concurrency_override: !!policy.concurrencyOverride, + max_rpm: policy.maxRpm ?? null, + max_sessions: policy.maxSessions ?? null, }) - } else { - record = getVm(cfg.paths.project, id) - } - if (!record) { - missing.push(id) - continue + } catch (error) { + failed.push({ id, error: (error && error.message) || String(error) }) } - if (hasConc && nextConc !== null) applyVmConcurrency(id, nextConc, { override: true }) - if (hasRpm && nextRpm !== null) applyVmRpm(id, nextRpm, { override: true }) - if (hasSessions && nextSessions !== null) ctx.applyVmMaxSessions(id, nextSessions, { override: true }) - const inherit = {} - if (hasConc && nextConc === null) inherit.concurrency = true - if (hasRpm && nextRpm === null) inherit.rpm = true - if (hasSessions && nextSessions === null) inherit.maxSessions = true - if (Object.keys(inherit).length) inheritVmScheduling(id, inherit) - const fresh = getVm(cfg.paths.project, id) || record - const policy = fresh.policy || {} - items.push({ - id, - weight: Number(policy.weight ?? 1), - schedule_level: manualScheduleLevelOf(fresh) ?? 'auto', - max_concurrency: policy.maxConcurrency ?? null, - concurrency_override: !!policy.concurrencyOverride, - max_rpm: policy.maxRpm ?? null, - max_sessions: policy.maxSessions ?? null, - }) - } catch (error) { - failed.push({ id, error: (error && error.message) || String(error) }) } - } - return json(res, 200, panel.ok({ updated: items.length, missing, failed, items })) + return json(res, 200, panel.ok({ updated: items.length, missing, failed, items })) + }) } // POST /api/panel/vms/:id/reload if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/reload$/.test(p)) { const id = p.split('/')[4] - const vm = getVm(cfg.paths.project, id) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const reloaded = await reloadSlotReady(vm, cfg.paths.project, { routing: ctx.routingConfig }) - if (!reloaded.ok) { - return json(res, 500, { - ok: false, - error: { message: reloaded.error || 'reload failed', code: reloaded.code }, - }) - } - return json(res, 200, panel.ok({ id, reload: reloaded })) + return mutateVms([id], async (handles) => { + const vm = getVm(cfg.paths.project, id) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const reloaded = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.reloadSlotReady || reloadSlotReady)(vm, cfg.paths.project, { + routing: ctx.routingConfig, + }), + ) + if (!reloaded.ok) { + return json(res, 500, { + ok: false, + error: { message: reloaded.error || 'reload failed', code: reloaded.code }, + }) + } + return json(res, 200, panel.ok({ id, reload: reloaded })) + }) } // POST /api/panel/vms/:id/shell-ticket — single-use ticket for the slot terminal WebSocket if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/shell-ticket$/.test(p)) { @@ -2710,335 +2759,339 @@ export function createPanelHandler(ctx) { if (req.method === 'PATCH' && /^\/api\/panel\/vms\/[^/]+$/.test(p)) { const id = p.split('/').pop() const body = await readBody(req, 8192).catch(() => ({})) - const currentVm = getVm(cfg.paths.project, id) - if (!currentVm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const strictInteger = (value, min, max, label, { nullable = false } = {}) => { - if (nullable && value === null) return null - if (typeof value !== 'number' || !Number.isFinite(value) || !Number.isInteger(value)) { - throw new TypeError(`${label} must be an integer`) + return mutateVms([id], async (handles) => { + const currentVm = getVm(cfg.paths.project, id) + if (!currentVm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const strictInteger = (value, min, max, label, { nullable = false } = {}) => { + if (nullable && value === null) return null + if (typeof value !== 'number' || !Number.isFinite(value) || !Number.isInteger(value)) { + throw new TypeError(`${label} must be an integer`) + } + if (value < min || value > max) throw new RangeError(`${label} must be between ${min} and ${max}`) + return value } - if (value < min || value > max) throw new RangeError(`${label} must be between ${min} and ${max}`) - return value - } - const hasConcurrency = - Object.prototype.hasOwnProperty.call(body || {}, 'max_concurrency') || - Object.prototype.hasOwnProperty.call(body || {}, 'maxConcurrency') - const hasRpm = - Object.prototype.hasOwnProperty.call(body || {}, 'max_rpm') || - Object.prototype.hasOwnProperty.call(body || {}, 'maxRpm') - const next = Object.prototype.hasOwnProperty.call(body || {}, 'max_concurrency') - ? body.max_concurrency - : body.maxConcurrency - const nextRpm = Object.prototype.hasOwnProperty.call(body || {}, 'max_rpm') ? body.max_rpm : body.maxRpm - const hasMaxSessions = body && Object.prototype.hasOwnProperty.call(body, 'max_sessions') - // Explicit null = drop the slot's pin and follow its tier / global default again. - const inheritConc = body?.max_concurrency === null - const inheritRpm = body?.max_rpm === null - const hasSessionSlots = body && Object.prototype.hasOwnProperty.call(body, 'session_slots') - const hasQuotaOverride = body && Object.prototype.hasOwnProperty.call(body, 'quota_override') - const hasModels = body && Object.prototype.hasOwnProperty.call(body, 'allowed_models') - const hasAuthScheme = body && (body.auth_scheme != null || body.authScheme != null) - const hasScheduleLevel = body && Object.prototype.hasOwnProperty.call(body, 'schedule_level') - const hasActiveHours = body && Object.prototype.hasOwnProperty.call(body, 'active_hours') - const hasTimezone = body && Object.prototype.hasOwnProperty.call(body, 'timezone') - const hasWeight = body && Object.prototype.hasOwnProperty.call(body, 'weight') - const hasMemory = - body && - (Object.prototype.hasOwnProperty.call(body, 'memory') || - (body.machine && - typeof body.machine === 'object' && - !Array.isArray(body.machine) && - Object.prototype.hasOwnProperty.call(body.machine, 'memory'))) - // `timezone_follow_proxy: true` re-attaches the slot to its exit node's - // zone, undoing an earlier hand-pinned value. - const followProxyTz = body?.timezone_follow_proxy === true || body?.timezoneFollowProxy === true - const hasSlotPolicy = - body && - (Object.prototype.hasOwnProperty.call(body, 'inference_engine') || - Object.prototype.hasOwnProperty.call(body, 'persona_preset')) - if ( - !hasConcurrency && - !hasRpm && - !inheritConc && - !inheritRpm && - !hasSessionSlots && - !hasMaxSessions && - !hasQuotaOverride && - !hasModels && - !hasAuthScheme && - !hasSlotPolicy && - !hasScheduleLevel && - !hasActiveHours && - !hasTimezone && - !hasWeight && - !hasMemory && - !followProxyTz - ) { - return json(res, 400, { - ok: false, - error: { - message: - 'max_concurrency, max_rpm, max_sessions, session_slots, quota_override, allowed_models, auth_scheme, inference_engine, persona_preset, schedule_level, active_hours, timezone, weight, machine.memory or timezone_follow_proxy required', - }, - }) - } - try { - if (hasConcurrency && next !== null) strictInteger(next, 1, 256, 'max_concurrency') - if (hasRpm && nextRpm !== null) strictInteger(nextRpm, 0, 1_000_000, 'max_rpm') - if (hasSessionSlots && body.session_slots !== null) - strictInteger(body.session_slots, SESSION_SLOT_MIN, SESSION_SLOT_MAX, 'session_slots') - if (hasMaxSessions && body.max_sessions !== null) strictInteger(body.max_sessions, 0, 256, 'max_sessions') - // null puts the slot back on the default weight of 1 - if (hasWeight && body.weight !== null) strictInteger(body.weight, 1, 100, 'weight') - } catch (error) { - return json(res, 400, { - ok: false, - error: { code: 'invalid_scheduling_value', message: error.message }, - }) - } - const parsedScheduleLevel = hasScheduleLevel ? parseScheduleLevelInput(body.schedule_level) : null - if (parsedScheduleLevel && !parsedScheduleLevel.ok) { - return json(res, 400, { ok: false, error: { message: parsedScheduleLevel.error } }) - } - const parsedActiveHours = hasActiveHours ? parseActiveHoursInput(body.active_hours) : null - if (parsedActiveHours && !parsedActiveHours.ok) { - return json(res, 400, { ok: false, error: { message: parsedActiveHours.error } }) - } - const parsedQuotaOverride = hasQuotaOverride ? parseVmQuotaOverride(body.quota_override) : null - if (parsedQuotaOverride && !parsedQuotaOverride.ok) { - return json(res, 400, { ok: false, error: { message: parsedQuotaOverride.error } }) - } - if (parsedQuotaOverride) { - if (isCodexVm(currentVm)) { + const hasConcurrency = + Object.prototype.hasOwnProperty.call(body || {}, 'max_concurrency') || + Object.prototype.hasOwnProperty.call(body || {}, 'maxConcurrency') + const hasRpm = + Object.prototype.hasOwnProperty.call(body || {}, 'max_rpm') || + Object.prototype.hasOwnProperty.call(body || {}, 'maxRpm') + const next = Object.prototype.hasOwnProperty.call(body || {}, 'max_concurrency') + ? body.max_concurrency + : body.maxConcurrency + const nextRpm = Object.prototype.hasOwnProperty.call(body || {}, 'max_rpm') ? body.max_rpm : body.maxRpm + const hasMaxSessions = body && Object.prototype.hasOwnProperty.call(body, 'max_sessions') + // Explicit null = drop the slot's pin and follow its tier / global default again. + const inheritConc = body?.max_concurrency === null + const inheritRpm = body?.max_rpm === null + const hasSessionSlots = body && Object.prototype.hasOwnProperty.call(body, 'session_slots') + const hasQuotaOverride = body && Object.prototype.hasOwnProperty.call(body, 'quota_override') + const hasModels = body && Object.prototype.hasOwnProperty.call(body, 'allowed_models') + const hasAuthScheme = body && (body.auth_scheme != null || body.authScheme != null) + const hasScheduleLevel = body && Object.prototype.hasOwnProperty.call(body, 'schedule_level') + const hasActiveHours = body && Object.prototype.hasOwnProperty.call(body, 'active_hours') + const hasTimezone = body && Object.prototype.hasOwnProperty.call(body, 'timezone') + const hasWeight = body && Object.prototype.hasOwnProperty.call(body, 'weight') + const hasMemory = + body && + (Object.prototype.hasOwnProperty.call(body, 'memory') || + (body.machine && + typeof body.machine === 'object' && + !Array.isArray(body.machine) && + Object.prototype.hasOwnProperty.call(body.machine, 'memory'))) + // `timezone_follow_proxy: true` re-attaches the slot to its exit node's + // zone, undoing an earlier hand-pinned value. + const followProxyTz = body?.timezone_follow_proxy === true || body?.timezoneFollowProxy === true + const hasSlotPolicy = + body && + (Object.prototype.hasOwnProperty.call(body, 'inference_engine') || + Object.prototype.hasOwnProperty.call(body, 'persona_preset')) + if ( + !hasConcurrency && + !hasRpm && + !inheritConc && + !inheritRpm && + !hasSessionSlots && + !hasMaxSessions && + !hasQuotaOverride && + !hasModels && + !hasAuthScheme && + !hasSlotPolicy && + !hasScheduleLevel && + !hasActiveHours && + !hasTimezone && + !hasWeight && + !hasMemory && + !followProxyTz + ) { return json(res, 400, { ok: false, - error: { code: 'gpt_quota_override_forbidden', message: 'GPT slots do not use Claude quota tiers' }, + error: { + message: + 'max_concurrency, max_rpm, max_sessions, session_slots, quota_override, allowed_models, auth_scheme, inference_engine, persona_preset, schedule_level, active_hours, timezone, weight, machine.memory or timezone_follow_proxy required', + }, }) } - } - if (hasSessionSlots && isCodexVm(currentVm)) { - return json(res, 400, { - ok: false, - error: { code: 'gpt_session_slots_forbidden', message: 'GPT slots do not use native session slots' }, - }) - } - if (hasMaxSessions && !isCodexVm(currentVm)) { - return json(res, 400, { - ok: false, - error: { code: 'claude_max_sessions_forbidden', message: 'Claude 槽不使用 OpenAI 会话容量上限' }, - }) - } - if (hasAuthScheme && hostUnsupported(res, currentVm, 'auth_scheme')) return - if ( - hasSlotPolicy && - Object.prototype.hasOwnProperty.call(body, 'inference_engine') && - (isCodexVm(currentVm) || hostUnsupported(res, currentVm, 'engine_switch')) - ) { - if (isCodexVm(currentVm)) { + try { + if (hasConcurrency && next !== null) strictInteger(next, 1, 256, 'max_concurrency') + if (hasRpm && nextRpm !== null) strictInteger(nextRpm, 0, 1_000_000, 'max_rpm') + if (hasSessionSlots && body.session_slots !== null) + strictInteger(body.session_slots, SESSION_SLOT_MIN, SESSION_SLOT_MAX, 'session_slots') + if (hasMaxSessions && body.max_sessions !== null) strictInteger(body.max_sessions, 0, 256, 'max_sessions') + // null puts the slot back on the default weight of 1 + if (hasWeight && body.weight !== null) strictInteger(body.weight, 1, 100, 'weight') + } catch (error) { return json(res, 400, { ok: false, - error: { code: 'gpt_engine_forbidden', message: 'GPT slots do not use go or rust inference engines' }, + error: { code: 'invalid_scheduling_value', message: error.message }, }) } - return - } - if (hasModels) { - const parsed = parseAllowedModelsPatch(body.allowed_models, { vm: currentVm }) - if (!parsed.ok) return json(res, 400, { ok: false, error: { message: parsed.error } }) - } - if (hasSlotPolicy) { - const parsed = parseSlotEnginePolicyPatch(body) - if (!parsed.ok) return json(res, 400, { ok: false, error: { message: parsed.error } }) - } - let timezoneSync = null - if (hasTimezone) { - // Any IANA zone is allowed — the US presets are UI defaults, not a - // whitelist. Reject only what Intl (and therefore the container TZ) - // cannot resolve, so a typo never reaches `docker run -e TZ=`. - const zone = validTimezone(body.timezone) - if (!zone) { + const parsedScheduleLevel = hasScheduleLevel ? parseScheduleLevelInput(body.schedule_level) : null + if (parsedScheduleLevel && !parsedScheduleLevel.ok) { + return json(res, 400, { ok: false, error: { message: parsedScheduleLevel.error } }) + } + const parsedActiveHours = hasActiveHours ? parseActiveHoursInput(body.active_hours) : null + if (parsedActiveHours && !parsedActiveHours.ok) { + return json(res, 400, { ok: false, error: { message: parsedActiveHours.error } }) + } + const parsedQuotaOverride = hasQuotaOverride ? parseVmQuotaOverride(body.quota_override) : null + if (parsedQuotaOverride && !parsedQuotaOverride.ok) { + return json(res, 400, { ok: false, error: { message: parsedQuotaOverride.error } }) + } + if (parsedQuotaOverride) { + if (isCodexVm(currentVm)) { + return json(res, 400, { + ok: false, + error: { code: 'gpt_quota_override_forbidden', message: 'GPT slots do not use Claude quota tiers' }, + }) + } + } + if (hasSessionSlots && isCodexVm(currentVm)) { return json(res, 400, { ok: false, - error: { - type: 'invalid_request_error', - code: 'invalid_timezone', - message: '时区必须是有效的 IANA 名称，如 Asia/Tokyo', - param: 'timezone', - }, + error: { code: 'gpt_session_slots_forbidden', message: 'GPT slots do not use native session slots' }, }) } - timezoneSync = { timezone: zone, source: 'manual' } - } else if (followProxyTz) { - const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, id, { - force: true, - persist: false, - }) - if (!synced.ok) { + if (hasMaxSessions && !isCodexVm(currentVm)) { return json(res, 400, { ok: false, - error: { - type: 'invalid_request_error', - code: synced.reason || 'proxy_timezone_unknown', - message: '未能从代理节点取到时区，请先检测代理地理位置', - }, + error: { code: 'claude_max_sessions_forbidden', message: 'Claude 槽不使用 OpenAI 会话容量上限' }, }) } - timezoneSync = { applied: false, timezone: synced.timezone, source: 'proxy_geo' } - } - if (hasConcurrency && next !== null) { - const vm = applyVmConcurrency(id, next, { override: true }) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - if (hasRpm && nextRpm !== null) { - const vm = applyVmRpm(id, nextRpm, { override: true }) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - if ((hasConcurrency && next === null) || (hasRpm && nextRpm === null)) { - const vm = inheritVmScheduling(id, { - concurrency: hasConcurrency && next === null, - rpm: hasRpm && nextRpm === null, - }) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - if (hasSessionSlots) { - if (body.session_slots === null) { - inheritVmScheduling(id, { sessionSlots: true }) - } else { - const vm = applyVmSessionSlots(id, normalizeSessionSlots(body.session_slots), { override: true }) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + if (hasAuthScheme && hostUnsupported(res, currentVm, 'auth_scheme')) return + if ( + hasSlotPolicy && + Object.prototype.hasOwnProperty.call(body, 'inference_engine') && + (isCodexVm(currentVm) || hostUnsupported(res, currentVm, 'engine_switch')) + ) { + if (isCodexVm(currentVm)) { + return json(res, 400, { + ok: false, + error: { code: 'gpt_engine_forbidden', message: 'GPT slots do not use go or rust inference engines' }, + }) + } + return } - } - if (hasMaxSessions) { - if (body.max_sessions === null) { - const vm = inheritVmScheduling(id, { maxSessions: true }) + if (hasModels) { + const parsed = parseAllowedModelsPatch(body.allowed_models, { vm: currentVm }) + if (!parsed.ok) return json(res, 400, { ok: false, error: { message: parsed.error } }) + } + if (hasSlotPolicy) { + const parsed = parseSlotEnginePolicyPatch(body) + if (!parsed.ok) return json(res, 400, { ok: false, error: { message: parsed.error } }) + } + let timezoneSync = null + if (hasTimezone) { + // Any IANA zone is allowed — the US presets are UI defaults, not a + // whitelist. Reject only what Intl (and therefore the container TZ) + // cannot resolve, so a typo never reaches `docker run -e TZ=`. + const zone = validTimezone(body.timezone) + if (!zone) { + return json(res, 400, { + ok: false, + error: { + type: 'invalid_request_error', + code: 'invalid_timezone', + message: '时区必须是有效的 IANA 名称，如 Asia/Tokyo', + param: 'timezone', + }, + }) + } + timezoneSync = { timezone: zone, source: 'manual' } + } else if (followProxyTz) { + const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, id, { + force: true, + persist: false, + }) + if (!synced.ok) { + return json(res, 400, { + ok: false, + error: { + type: 'invalid_request_error', + code: synced.reason || 'proxy_timezone_unknown', + message: '未能从代理节点取到时区，请先检测代理地理位置', + }, + }) + } + timezoneSync = { applied: false, timezone: synced.timezone, source: 'proxy_geo' } + } + if (hasConcurrency && next !== null) { + const vm = applyVmConcurrency(id, next, { override: true }) if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } else { - const vm = ctx.applyVmMaxSessions(id, body.max_sessions, { override: true }) + } + if (hasRpm && nextRpm !== null) { + const vm = applyVmRpm(id, nextRpm, { override: true }) if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) } - } - if (parsedQuotaOverride) { - const vm = applyVmQuotaOverride(id, parsedQuotaOverride.value) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - if (hasModels) { - const parsed = parseAllowedModelsPatch(body.allowed_models, { vm: currentVm }) - const vm = persistAllowedModels(cfg.paths.project, id, parsed.value) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - if (hasAuthScheme) { - const vm = persistSlotAuthScheme(cfg.paths.project, id, body.auth_scheme || body.authScheme) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - let slotPolicyResult = null - if (hasSlotPolicy) { - const parsed = parseSlotEnginePolicyPatch(body) - slotPolicyResult = await applySlotPolicyWithRuntime(id, parsed.patch) - if (!slotPolicyResult.ok) { - const status = - slotPolicyResult.code === 'vm_not_found' - ? 404 - : slotPolicyResult.code === 'gpt_engine_forbidden' - ? 400 - : 503 - return json(res, status, { - ok: false, - error: { - code: slotPolicyResult.code, - message: slotPolicyResult.error || 'inference engine switch failed', - }, + if ((hasConcurrency && next === null) || (hasRpm && nextRpm === null)) { + const vm = inheritVmScheduling(id, { + concurrency: hasConcurrency && next === null, + rpm: hasRpm && nextRpm === null, }) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) } - } - if (parsedScheduleLevel) { - const vm = persistVmScheduleLevel(cfg.paths.project, id, parsedScheduleLevel.value) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - if (parsedActiveHours) { - const vm = persistVmActiveHours(cfg.paths.project, id, parsedActiveHours.value) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - if (hasTimezone) { - const vm = persistVmTimezone(cfg.paths.project, id, timezoneSync.timezone, { source: 'manual' }) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const kernel = projectSlotKernelConfig(vm) - timezoneSync.applied = true - timezoneSync.kernel_hot = kernel?.changed === true - } else if (followProxyTz) { - const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, id, { force: true }) - timezoneSync = { applied: synced.applied, timezone: synced.timezone, source: 'proxy_geo' } - if (synced.applied) { - const kernel = projectSlotKernelConfig({ id }) + if (hasSessionSlots) { + if (body.session_slots === null) { + inheritVmScheduling(id, { sessionSlots: true }) + } else { + const vm = applyVmSessionSlots(id, normalizeSessionSlots(body.session_slots), { override: true }) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + } + if (hasMaxSessions) { + if (body.max_sessions === null) { + const vm = inheritVmScheduling(id, { maxSessions: true }) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } else { + const vm = ctx.applyVmMaxSessions(id, body.max_sessions, { override: true }) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + } + if (parsedQuotaOverride) { + const vm = applyVmQuotaOverride(id, parsedQuotaOverride.value) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + if (hasModels) { + const parsed = parseAllowedModelsPatch(body.allowed_models, { vm: currentVm }) + const vm = persistAllowedModels(cfg.paths.project, id, parsed.value) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + if (hasAuthScheme) { + const vm = persistSlotAuthScheme(cfg.paths.project, id, body.auth_scheme || body.authScheme) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + let slotPolicyResult = null + if (hasSlotPolicy) { + const parsed = parseSlotEnginePolicyPatch(body) + slotPolicyResult = await vmEffect(handles.get(id), () => applySlotPolicyWithRuntime(id, parsed.patch)) + if (!slotPolicyResult.ok) { + const status = + slotPolicyResult.code === 'vm_not_found' + ? 404 + : slotPolicyResult.code === 'gpt_engine_forbidden' + ? 400 + : 503 + return json(res, status, { + ok: false, + error: { + code: slotPolicyResult.code, + message: slotPolicyResult.error || 'inference engine switch failed', + }, + }) + } + } + if (parsedScheduleLevel) { + const vm = persistVmScheduleLevel(cfg.paths.project, id, parsedScheduleLevel.value) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + if (parsedActiveHours) { + const vm = persistVmActiveHours(cfg.paths.project, id, parsedActiveHours.value) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + if (hasTimezone) { + const vm = persistVmTimezone(cfg.paths.project, id, timezoneSync.timezone, { source: 'manual' }) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const kernel = projectSlotKernelConfig(vm) + timezoneSync.applied = true timezoneSync.kernel_hot = kernel?.changed === true + } else if (followProxyTz) { + const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, id, { force: true }) + timezoneSync = { applied: synced.applied, timezone: synced.timezone, source: 'proxy_geo' } + if (synced.applied) { + const kernel = projectSlotKernelConfig({ id }) + timezoneSync.kernel_hot = kernel?.changed === true + } } - } - if (hasWeight) { - const nextWeight = body.weight === null ? 1 : body.weight - const vm = patchVmRecord(cfg.paths.project, id, (record) => ({ - policy: { ...(record.policy || {}), weight: nextWeight }, - })) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - let memoryRaiseResult = null - if (hasMemory) { - const requestedMemory = - body.machine && typeof body.machine === 'object' && !Array.isArray(body.machine) - ? body.machine.memory - : body.memory - const liveVm = getVm(cfg.paths.project, id) || currentVm - const raiseMemory = ctx.raiseDockerSlotMemory || raiseDockerSlotMemory - const raised = raiseMemory(cfg.paths.project, liveVm, requestedMemory, { - routing: ctx.routingConfig, - }) - if (!raised.ok) { - const status = - raised.code === 'vm_not_found' - ? 404 - : raised.code === 'invalid_memory' || - raised.code === 'memory_not_increased' || - raised.code === 'kvm_memory_raise_forbidden' - ? 400 - : 502 - return json(res, status, { - ok: false, - error: { - code: raised.code, - message: raised.message, - requested: raised.requested ?? requestedMemory ?? null, - container_memory: raised.container_memory ?? null, - current: raised.current ?? null, - }, - }) + if (hasWeight) { + const nextWeight = body.weight === null ? 1 : body.weight + const vm = patchVmRecord(cfg.paths.project, id, (record) => ({ + policy: { ...(record.policy || {}), weight: nextWeight }, + })) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) } - memoryRaiseResult = { - previous: raised.previous, - memory: raised.memory, - container_memory: raised.container_memory, + let memoryRaiseResult = null + if (hasMemory) { + const requestedMemory = + body.machine && typeof body.machine === 'object' && !Array.isArray(body.machine) + ? body.machine.memory + : body.memory + const liveVm = getVm(cfg.paths.project, id) || currentVm + const raiseMemory = ctx.raiseDockerSlotMemory || raiseDockerSlotMemory + const raised = await vmEffect(handles.get(id), () => + raiseMemory(cfg.paths.project, liveVm, requestedMemory, { + routing: ctx.routingConfig, + }), + ) + if (!raised.ok) { + const status = + raised.code === 'vm_not_found' + ? 404 + : raised.code === 'invalid_memory' || + raised.code === 'memory_not_increased' || + raised.code === 'kvm_memory_raise_forbidden' + ? 400 + : 502 + return json(res, status, { + ok: false, + error: { + code: raised.code, + message: raised.message, + requested: raised.requested ?? requestedMemory ?? null, + container_memory: raised.container_memory ?? null, + current: raised.current ?? null, + }, + }) + } + memoryRaiseResult = { + previous: raised.previous, + memory: raised.memory, + container_memory: raised.container_memory, + } } - } - const detail = await panel.buildVmDetail({ - cfg, - accountQuota, - id, - routingConfig: ctx.routingConfig, - poolScheduler: ctx.poolScheduler, - requestLog, - proxyPool, - kernelHealth: panelKernelHealth, - clusterNodes: clusterNodesOf(ctx), - role: panelIdentity(req).role, - }) - if (slotPolicyResult && detail?.data) { - detail.data.engine_switch = { - configured_engine: slotPolicyResult.configured_engine, - resolved_engine: slotPolicyResult.resolved_engine, - active_engine: slotPolicyResult.active_engine, - runtime: slotPolicyResult.runtime, + const detail = await panel.buildVmDetail({ + cfg, + accountQuota, + id, + routingConfig: ctx.routingConfig, + poolScheduler: ctx.poolScheduler, + requestLog, + proxyPool, + kernelHealth: panelKernelHealth, + clusterNodes: clusterNodesOf(ctx), + role: panelIdentity(req).role, + }) + if (slotPolicyResult && detail?.data) { + detail.data.engine_switch = { + configured_engine: slotPolicyResult.configured_engine, + resolved_engine: slotPolicyResult.resolved_engine, + active_engine: slotPolicyResult.active_engine, + runtime: slotPolicyResult.runtime, + } } - } - if (timezoneSync && detail?.data) detail.data.timezone_sync = timezoneSync - if (memoryRaiseResult && detail?.data) detail.data.memory_raise = memoryRaiseResult - return json(res, 200, detail) + if (timezoneSync && detail?.data) detail.data.timezone_sync = timezoneSync + if (memoryRaiseResult && detail?.data) detail.data.memory_raise = memoryRaiseResult + return json(res, 200, detail) + }) } // POST /api/panel/vms/:id/probe if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/probe$/.test(p)) { @@ -3218,71 +3271,89 @@ export function createPanelHandler(ctx) { if (req.method === 'POST' && p === '/api/panel/vms/schedule/hold') { const body = await readBody(req, 8192).catch(() => ({})) const reason = String(body?.reason || KERNEL_UPDATE_HOLD).slice(0, 64) - const ids = holdSchedulableSlots(cfg.paths.project, reason) - const drain = await waitForSlotDrain(ctx.poolScheduler, ids) - if (!drain.ok) { - const restored = resumeSchedulableSlots(cfg.paths.project, ids, reason) - return json(res, 409, { - ok: false, - error: { code: 'slots_busy', message: '在途请求未结束，已恢复调度' }, - data: { paused: ids.length, restored: restored.length, inflight: drain.inflight, drained: false }, - }) - } - return json(res, 200, panel.ok({ paused: ids.length, inflight: 0, drained: true })) + const candidates = listVms(cfg.paths.project) + .filter((vm) => vm?.schedulable !== false) + .map((vm) => vm.id) + return mutateVms(candidates, async () => { + const ids = [] + for (const id of candidates) { + if (getVm(cfg.paths.project, id)?.schedulable === false) continue + if (setVmSchedulable(cfg.paths.project, id, false, reason, { preserveStatus: true, source: 'force' })) + ids.push(id) + } + // Shared claims exclude every global execution; standalone retains its local drain behavior. + const drain = + isKubernetesMode() || ctx.vmOperations + ? { ok: true, inflight: 0 } + : await waitForSlotDrain(ctx.poolScheduler, ids) + if (!drain.ok) { + const restored = resumeSchedulableSlots(cfg.paths.project, ids, reason) + return json(res, 409, { + ok: false, + error: { code: 'slots_busy', message: '在途请求未结束，已恢复调度' }, + data: { paused: ids.length, restored: restored.length, inflight: drain.inflight, drained: false }, + }) + } + return json(res, 200, panel.ok({ paused: ids.length, inflight: 0, drained: true })) + }) } if (req.method === 'POST' && p === '/api/panel/vms/schedule/resume') { const body = await readBody(req, 8192).catch(() => ({})) const reason = String(body?.reason || KERNEL_UPDATE_HOLD).slice(0, 64) - const ids = (listVms(cfg.paths.project) || []) + const ids = listVms(cfg.paths.project) .filter((vm) => vm?.schedule_disabled_reason === reason) .map((vm) => vm.id) - const restored = resumeSchedulableSlots(cfg.paths.project, ids, reason) - return json(res, 200, panel.ok({ restored: restored.length })) + return mutateVms(ids, async () => { + const restored = resumeSchedulableSlots(cfg.paths.project, ids, reason) + return json(res, 200, panel.ok({ restored: restored.length })) + }) } // POST /api/panel/vms/:id/schedulable — sub2api-style schedule toggle if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/schedulable$/.test(p)) { const id = p.split('/')[4] - if (!getVm(cfg.paths.project, id)) { - const e = makeError({ - type: ErrorType.NOT_FOUND, - code: ErrorCode.VM_NOT_FOUND, - message: 'vm not found', - status: 404, - }) - return json(res, e.status, { ok: false, error: e.body.error }) - } - const body = await readBody(req, 8192).catch(() => ({})) - if (typeof body?.schedulable !== 'boolean') { - return json(res, 400, { ok: false, error: { message: 'schedulable required' } }) - } - const currentVm = getVm(cfg.paths.project, id) - const credentialFailure = - /oauth_revoked|oauth_invalid_grant|invalid_grant|refresh_token_missing|token has been revoked/i.test( - `${currentVm?.claude?.refresh_error || ''} ${currentVm?.schedule_disabled_reason || ''}`, - ) - if (body.schedulable && credentialFailure) { - return json(res, 409, { - ok: false, - error: { code: 'credential_unavailable', message: '凭证已失效或吊销，请重新导入凭证后再开启调度' }, + return mutateVms([id], async () => { + if (!getVm(cfg.paths.project, id)) { + const e = makeError({ + type: ErrorType.NOT_FOUND, + code: ErrorCode.VM_NOT_FOUND, + message: 'vm not found', + status: 404, + }) + return json(res, e.status, { ok: false, error: e.body.error }) + } + const body = await readBody(req, 8192).catch(() => ({})) + if (typeof body?.schedulable !== 'boolean') { + return json(res, 400, { ok: false, error: { message: 'schedulable required' } }) + } + const currentVm = getVm(cfg.paths.project, id) + const credentialFailure = + /oauth_revoked|oauth_invalid_grant|invalid_grant|refresh_token_missing|token has been revoked/i.test( + `${currentVm?.claude?.refresh_error || ''} ${currentVm?.schedule_disabled_reason || ''}`, + ) + if (body.schedulable && credentialFailure) { + return json(res, 409, { + ok: false, + error: { code: 'credential_unavailable', message: '凭证已失效或吊销，请重新导入凭证后再开启调度' }, + }) + } + const reason = body.schedulable ? null : body.reason || 'disabled' + const summary = setVmSchedulable(cfg.paths.project, id, body.schedulable, reason, { + preserveStatus: true, + source: 'manual', }) - } - const reason = body.schedulable ? null : body.reason || 'disabled' - const summary = setVmSchedulable(cfg.paths.project, id, body.schedulable, reason, { - preserveStatus: true, - source: 'manual', + return json( + res, + 200, + panel.ok({ + id, + schedulable: summary?.schedulable !== false, + schedule_manual: summary?.schedule_manual === true, + schedule_disabled_reason: summary?.schedule_disabled_reason || null, + status: summary?.status || null, + }), + ) }) - return json( - res, - 200, - panel.ok({ - id, - schedulable: summary?.schedulable !== false, - schedule_manual: summary?.schedule_manual === true, - schedule_disabled_reason: summary?.schedule_disabled_reason || null, - status: summary?.status || null, - }), - ) } // POST /api/panel/vms/:id/cooldown/clear — drop 401/quota leftover so the slot re-enters if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/cooldown\/clear$/.test(p)) { @@ -3333,61 +3404,81 @@ export function createPanelHandler(ctx) { // DELETE /api/panel/vms/:id — remove VM record, cli-home, unbind proxy if (req.method === 'DELETE' && /^\/api\/panel\/vms\/[^/]+$/.test(p)) { const id = p.split('/')[4] - if (!isValidVmId(id)) { - return json(res, 400, { ok: false, error: { message: 'invalid vm id' } }) - } - const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) - if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const active = getActiveVmId(cfg.paths.project) - if (active === id && listVms(cfg.paths.project).some((vm) => vm.id !== id)) { - return json(res, 409, { ok: false, error: { message: 'cannot delete active VM, switch active first' } }) - } - const vm = getVm(cfg.paths.project, id) - if (denyIfUserCannotDeleteVm(req, res, vm, json)) return true - try { - if (vm) { - const gone = await destroySlot(vm) - // A node slot that could not be removed keeps running there with the account's credential. - if (!gone.ok && slotHost(vm).kind === 'node') { - return json(res, 409, { - ok: false, - error: { code: gone.code || 'remote_destroy_failed', message: gone.error || 'remote destroy failed' }, - }) + return mutateVms([id], async (handles) => { + if (!isValidVmId(id)) { + return json(res, 400, { ok: false, error: { message: 'invalid vm id' } }) + } + const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) + if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const active = getActiveVmId(cfg.paths.project) + if (active === id && listVms(cfg.paths.project).some((vm) => vm.id !== id)) { + return json(res, 409, { ok: false, error: { message: 'cannot delete active VM, switch active first' } }) + } + const vm = getVm(cfg.paths.project, id) + if (denyIfUserCannotDeleteVm(req, res, vm, json)) return true + try { + if (vm) { + const gone = await vmEffect(handles.get(id), () => (ctx.vmMutationOps?.destroySlot || destroySlot)(vm)) + // A node slot that could not be removed keeps running there with the account's credential. + if (!gone.ok) { + return json(res, 409, { + ok: false, + error: { code: gone.code || 'remote_destroy_failed', message: gone.error || 'remote destroy failed' }, + }) + } } + } catch (error) { + await handles.get(id)?.markUncertain?.() + return json(res, 502, { + ok: false, + error: { code: error.code || 'vm_destroy_unknown', message: String(error.message || error) }, + }) } - } catch {} - // detach this VM only — do not unbind other slots sharing the SOCKS5 - try { - await proxyPool.unbindVm(id) - } catch {} - // remove cli-home - try { - fs.rmSync(path.join(cfg.paths.project, 'vms', id), { recursive: true, force: true }) - } catch {} - fs.unlinkSync(vmPath) - invalidateVmSlotReadCache(cfg.paths.project, id) - try { - await vmTestSchedule.remove(id) - } catch {} - if (active === id) { - clearActiveVm(cfg.paths.project) + // detach this VM only — do not unbind other slots sharing the SOCKS5 try { - await clearActiveVmFromDb() - } catch {} + await proxyPool.unbindVm(id) + } catch (error) { + if (isKubernetesMode() || ctx.vmOperations) throw error + } + // remove cli-home try { - reloadActiveVm(cfg) + fs.rmSync(path.join(cfg.paths.project, 'vms', id), { recursive: true, force: true }) + } catch (error) { + if (isKubernetesMode() || ctx.vmOperations) throw error + } + fs.unlinkSync(vmPath) + invalidateVmSlotReadCache(cfg.paths.project, id) + try { + await vmTestSchedule.remove(id) + } catch (error) { + if (isKubernetesMode() || ctx.vmOperations) throw error + } + if (active === id) { + clearActiveVm(cfg.paths.project) + try { + await clearActiveVmFromDb() + } catch (error) { + if (isKubernetesMode() || ctx.vmOperations) throw error + } + try { + reloadActiveVm(cfg) + } catch (error) { + if (isKubernetesMode() || ctx.vmOperations) throw error + } + } + // drop the DB credential mirror row too + try { + await removeVmFromDb(id) + } catch (error) { + if (isKubernetesMode() || ctx.vmOperations) throw error + } + // optional chat side files + try { + const chat = path.join(cfg.paths.project, 'vms', `${id}-chat.json`) + if (fs.existsSync(chat)) fs.unlinkSync(chat) } catch {} - } - // drop the DB credential mirror row too - try { - await removeVmFromDb(id) - } catch {} - // optional chat side files - try { - const chat = path.join(cfg.paths.project, 'vms', `${id}-chat.json`) - if (fs.existsSync(chat)) fs.unlinkSync(chat) - } catch {} - return json(res, 200, panel.ok({ deleted: id })) + return json(res, 200, panel.ok({ deleted: id })) + }) } // POST /api/panel/vms/:id/reset — destroy container + home, recreate same slot if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/reset$/.test(p)) { @@ -3988,375 +4079,407 @@ export function createPanelHandler(ctx) { const rawId = body.id || 'vm-' + padVm(idx) const id = String(rawId).replace(/[^a-zA-Z0-9_-]/g, '') if (!isValidVmId(id)) return json(res, 400, { ok: false, error: { message: 'invalid id' } }) - const vmsDir = path.join(cfg.paths.project, 'vms') - fs.mkdirSync(vmsDir, { recursive: true }) - const vmPath = path.join(vmsDir, id + '.json') - if (fs.existsSync(vmPath)) { - return json(res, 409, { ok: false, error: { message: 'vm id exists' } }) - } - const ident = panelIdentity(req) - if (ident.role === 'user') { - const ownerId = normalizeOwnerId(req.panelUserId) - const quota = clampVmCreateQuota((await panelUsers.getById(ownerId))?.vm_create_quota, 0) - const used = countUserCreatedVms(existing, ownerId) - if (!ownerId || quota <= 0 || used >= quota) { - return json(res, 403, { - ok: false, - error: { code: 'vm_create_quota_exceeded', message: '自建虚拟机数量已达上限' }, - }) - } - } - const startNow = body.start !== false && body.status !== 'stopped' - const wantKernel = body.kernel && OS_CATALOG[body.kernel] ? body.kernel : kernelForIndex(idx) - const nodeId = body.node_id ? String(body.node_id) : null - const vmDefaults = normalizeVmConfig(ctx.routingConfig?.vm) - const requestedRuntime = body.runtime_type - let runtime - if (requestedRuntime == null || requestedRuntime === '') { - runtime = vmDefaults.default_runtime - } else { - const kind = String(requestedRuntime).trim().toLowerCase() - if (kind !== 'docker' && kind !== 'kvm') { - return json(res, 400, { - ok: false, - error: { - type: 'invalid_request_error', - code: 'invalid_runtime_type', - message: 'runtime_type 必须是 docker 或 kvm', - }, - }) + return mutateVms([id], async (handles) => { + const existing = listVms(cfg.paths.project) + const aliases = physicalVmAliases(id) + const committed = + isKubernetesMode() || ctx.vmOperations + ? await getDb().prepare('SELECT id FROM vms WHERE id = ANY($1::text[]) LIMIT 1').get(aliases) + : null + if (committed || existing.some((vm) => aliases.includes(vm.id))) { + return json(res, 409, { ok: false, error: { code: 'vm_id_exists', message: 'vm identity exists' } }) } - runtime = kind - } - let machine - try { - machine = resolveMachineSpec({ config: vmDefaults, overrides: body.machine, runtime }) - } catch (error) { - if (error?.status === 400 || error?.code === 'invalid_machine') { - return json(res, 400, { - ok: false, - error: { type: 'invalid_request_error', code: 'invalid_machine', message: error.message }, - }) + const vmsDir = path.join(cfg.paths.project, 'vms') + fs.mkdirSync(vmsDir, { recursive: true }) + const vmPath = path.join(vmsDir, id + '.json') + if (fs.existsSync(vmPath)) { + return json(res, 409, { ok: false, error: { message: 'vm id exists' } }) } - throw error - } - if (nodeId) { - if (ident.role !== 'admin') { - return json(res, 403, { - ok: false, - error: { code: 'placement_forbidden', message: '只有管理员可以把虚拟机放到集群节点' }, - }) + const ident = panelIdentity(req) + if (ident.role === 'user') { + const ownerId = normalizeOwnerId(req.panelUserId) + const quota = clampVmCreateQuota((await panelUsers.getById(ownerId))?.vm_create_quota, 0) + const used = countUserCreatedVms(existing, ownerId) + if (!ownerId || quota <= 0 || used >= quota) { + return json(res, 403, { + ok: false, + error: { code: 'vm_create_quota_exceeded', message: '自建虚拟机数量已达上限' }, + }) + } } - const kindProbe = { node_id: nodeId } - stampVmKind(kindProbe, body) - if (isCodexVm(kindProbe) && !slotHost(kindProbe).supports('codex')) { - return json(res, 400, { - ok: false, - error: { code: 'remote_unsupported', message: '集群节点上的虚拟机暂不支持此操作' }, - }) + const startNow = body.start !== false && body.status !== 'stopped' + const wantKernel = body.kernel && OS_CATALOG[body.kernel] ? body.kernel : kernelForIndex(idx) + const nodeId = body.node_id ? String(body.node_id) : null + const vmDefaults = normalizeVmConfig(ctx.routingConfig?.vm) + const requestedRuntime = body.runtime_type + let runtime + if (requestedRuntime == null || requestedRuntime === '') { + runtime = vmDefaults.default_runtime + } else { + const kind = String(requestedRuntime).trim().toLowerCase() + if (kind !== 'docker' && kind !== 'kvm') { + return json(res, 400, { + ok: false, + error: { + type: 'invalid_request_error', + code: 'invalid_runtime_type', + message: 'runtime_type 必须是 docker 或 kvm', + }, + }) + } + runtime = kind } - let preflight + let machine try { - preflight = await preflightNode(nodeId, { kernel: wantKernel, runtime }) - } catch (e) { - return json(res, e?.status || 500, { - ok: false, - error: { code: e?.code || 'placement_failed', message: String(e?.message || e) }, - }) - } - if (!preflight.ok) { - return json(res, 409, { - ok: false, - error: { - code: 'placement_preflight_failed', - message: '目标节点预检未通过', - checks: preflight.checks, - }, - }) + machine = resolveMachineSpec({ config: vmDefaults, overrides: body.machine, runtime }) + } catch (error) { + if (error?.status === 400 || error?.code === 'invalid_machine') { + return json(res, 400, { + ok: false, + error: { type: 'invalid_request_error', code: 'invalid_machine', message: error.message }, + }) + } + throw error } - } - // An explicit exit (e.g. local egress for a node: that VPS's own route) replaces auto-allocation. - let pickedProxyId = typeof body.proxy_id === 'string' ? body.proxy_id.trim() : '' - if (pickedProxyId && isLocalEgressId(pickedProxyId)) { - // Local egress is per VPS: a generic pick resolves to the target machine's own row. - const resolved = await proxyPool.resolveLocalTarget(pickedProxyId, nodeId || null) - if (!resolved.ok) { - return json(res, 409, { - ok: false, - error: { code: resolved.error, message: LOCAL_PROXY_MISMATCH_MESSAGE }, - }) + if (nodeId) { + if (ident.role !== 'admin') { + return json(res, 403, { + ok: false, + error: { code: 'placement_forbidden', message: '只有管理员可以把虚拟机放到集群节点' }, + }) + } + const kindProbe = { node_id: nodeId } + stampVmKind(kindProbe, body) + if (isCodexVm(kindProbe) && !slotHost(kindProbe).supports('codex')) { + return json(res, 400, { + ok: false, + error: { code: 'remote_unsupported', message: '集群节点上的虚拟机暂不支持此操作' }, + }) + } + let preflight + try { + preflight = await preflightNode(nodeId, { kernel: wantKernel, runtime }) + } catch (e) { + return json(res, e?.status || 500, { + ok: false, + error: { code: e?.code || 'placement_failed', message: String(e?.message || e) }, + }) + } + if (!preflight.ok) { + return json(res, 409, { + ok: false, + error: { + code: 'placement_preflight_failed', + message: '目标节点预检未通过', + checks: preflight.checks, + }, + }) + } } - pickedProxyId = resolved.id - } - if (pickedProxyId) { - const row = proxyPool.snapshot().proxies.find((proxy) => proxy.id === pickedProxyId) - const owner = ident.role === 'user' ? normalizeOwnerId(req.panelUserId) : null - if (!row || !canBindProxyToVm(row, { owner_user_id: owner }, { role: ident.role })) { - return json(res, 404, { - ok: false, - error: { code: 'proxy_not_found', message: '所选出口不存在或不能绑到此槽位' }, - }) + // An explicit exit (e.g. local egress for a node: that VPS's own route) replaces auto-allocation. + let pickedProxyId = typeof body.proxy_id === 'string' ? body.proxy_id.trim() : '' + if (pickedProxyId && isLocalEgressId(pickedProxyId)) { + // Local egress is per VPS: a generic pick resolves to the target machine's own row. + const resolved = await proxyPool.resolveLocalTarget(pickedProxyId, nodeId || null) + if (!resolved.ok) { + return json(res, 409, { + ok: false, + error: { code: resolved.error, message: LOCAL_PROXY_MISMATCH_MESSAGE }, + }) + } + pickedProxyId = resolved.id } - const used = Array.isArray(row.bound_vm_ids) ? row.bound_vm_ids.length : 0 - if (!row.enabled || row.status === 'dead' || row.blocked_reason || used >= (row.bind_limit || 5)) { - return json(res, 409, { - ok: false, - error: { code: 'proxy_unavailable', message: '所选出口已失效或已绑满' }, - }) + if (pickedProxyId) { + const row = proxyPool.snapshot().proxies.find((proxy) => proxy.id === pickedProxyId) + const owner = ident.role === 'user' ? normalizeOwnerId(req.panelUserId) : null + if (!row || !canBindProxyToVm(row, { owner_user_id: owner }, { role: ident.role })) { + return json(res, 404, { + ok: false, + error: { code: 'proxy_not_found', message: '所选出口不存在或不能绑到此槽位' }, + }) + } + const used = Array.isArray(row.bound_vm_ids) ? row.bound_vm_ids.length : 0 + if (!row.enabled || row.status === 'dead' || row.blocked_reason || used >= (row.bind_limit || 5)) { + return json(res, 409, { + ok: false, + error: { code: 'proxy_unavailable', message: '所选出口已失效或已绑满' }, + }) + } } - } - if (runtime === RUNTIME_KVM && !nodeId) { - const probe = typeof ctx.probeLocalKvm === 'function' ? ctx.probeLocalKvm : probeLocalKvm - const kvm = await probe({ routing: ctx.routingConfig }) - if (!kvm?.ok) { - return json(res, 409, { - ok: false, - error: { code: 'kvm_unavailable', message: kvm?.error || '本机 KVM 不可用' }, - }) + if (runtime === RUNTIME_KVM && !nodeId) { + const probe = typeof ctx.probeLocalKvm === 'function' ? ctx.probeLocalKvm : probeLocalKvm + const kvm = await probe({ routing: ctx.routingConfig }) + if (!kvm?.ok) { + return json(res, 409, { + ok: false, + error: { code: 'kvm_unavailable', message: kvm?.error || '本机 KVM 不可用' }, + }) + } } - } - const requestedTimezone = validTimezone(body.timezone) - const requestedLocale = body.locale == null || body.locale === '' ? STANDARD_LOCALE : String(body.locale) - if (!SLOT_LOCALES.includes(requestedLocale)) { - return json(res, 400, { - ok: false, - error: { code: 'invalid_locale', message: `语言必须是 ${SLOT_LOCALES.join('、')}` }, - }) - } - let generated - try { - generated = assignComputerFingerprint( - cfg.paths.project, - { - id, - kernel: wantKernel, - timezone: requestedTimezone, - locale: requestedLocale, - runtime: { type: runtime }, - }, - { taken: takenFingerprintKeys(existing) }, - ) - } catch (e) { - return json(res, 400, { - ok: false, - error: { - code: e.code || 'computer_create_failed', - message: String(e.message || e), - }, - }) - } - const vm = { - id, - name: body.name || padVm(idx), - status: startNow ? 'running' : body.status || 'stopped', - kernel: wantKernel, - timezone: requestedTimezone || null, - // Only an explicit zone is a pin. An empty zone stays unset so the - // next SOCKS5 bind can write the exit node's detected timezone. - timezone_source: requestedTimezone ? 'manual' : 'auto', - locale: generated.locale || STANDARD_LOCALE, - region: body.region || body.zone || null, - note: body.note || `${(OS_CATALOG[wantKernel] || {}).pretty || wantKernel} · Go slot worker`, - proxy: null, - policy: { - maxConcurrency: 0, - concurrencyOverride: false, - maxRpm: 0, - rpmOverride: false, - maxSessions: 0, - sessionsOverride: false, - weight: Math.max(1, Math.min(100, Number(body.weight ?? 1))), - inflight: 0, - }, - claude: {}, - fingerprint: applyGeneratedFingerprint({}, generated), - stats: {}, - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - schedulable: false, - schedule_disabled_reason: 'no_credential', - proxy_cli_enabled: body.proxy_cli_enabled === true, - proxy_required: false, - seed_policy: standardSeedPolicy(), - runtime: { type: runtime }, - machine, - } - if (ident.role === 'user') { - vm.owner_user_id = normalizeOwnerId(req.panelUserId) - vm.origin = VM_ORIGIN.userCreated - } else { - vm.owner_user_id = null - vm.origin = VM_ORIGIN.platform - } - if (nodeId) vm.node_id = nodeId - stampVmKind(vm, body) - const openaiPolicy = normalizeOpenAIQuotaPolicy(ctx.routingConfig?.codex?.quota) - const explicitConcurrency = - Object.prototype.hasOwnProperty.call(body, 'max_concurrency') || - Object.prototype.hasOwnProperty.call(body, 'maxConcurrency') - const explicitRpm = - Object.prototype.hasOwnProperty.call(body, 'max_rpm') || Object.prototype.hasOwnProperty.call(body, 'maxRpm') - const explicitSessions = Object.prototype.hasOwnProperty.call(body, 'max_sessions') - const readField = (snake, camel) => - Object.prototype.hasOwnProperty.call(body, snake) ? body[snake] : body[camel] - const hasField = (snake, camel) => - Object.prototype.hasOwnProperty.call(body, snake) || Object.prototype.hasOwnProperty.call(body, camel) - const rawConcurrency = readField('max_concurrency', 'maxConcurrency') - const rawRpm = readField('max_rpm', 'maxRpm') - const rawSessions = body.max_sessions - const validInteger = (value, min, max) => - typeof value === 'number' && Number.isFinite(value) && Number.isInteger(value) && value >= min && value <= max - const invalidCreate = (value, min, max) => value !== null && !validInteger(value, min, max) - if (isCodexVm(vm)) { - if ( - (explicitConcurrency && invalidCreate(rawConcurrency, 1, 256)) || - (explicitRpm && invalidCreate(rawRpm, 0, 1e6)) || - (explicitSessions && invalidCreate(rawSessions, 0, 256)) - ) { + const requestedTimezone = validTimezone(body.timezone) + const requestedLocale = body.locale == null || body.locale === '' ? STANDARD_LOCALE : String(body.locale) + if (!SLOT_LOCALES.includes(requestedLocale)) { return json(res, 400, { ok: false, - error: { - code: 'invalid_create_schedule', - message: 'OpenAI scheduling values must be finite integers in range', - }, + error: { code: 'invalid_locale', message: `语言必须是 ${SLOT_LOCALES.join('、')}` }, }) } - vm.policy.maxConcurrency = - explicitConcurrency && rawConcurrency !== null ? rawConcurrency : openaiPolicy.max_concurrency - vm.policy.concurrencyOverride = explicitConcurrency && rawConcurrency !== null - vm.policy.maxRpm = explicitRpm && rawRpm !== null ? rawRpm : openaiPolicy.max_rpm - vm.policy.rpmOverride = explicitRpm && rawRpm !== null - vm.policy.maxSessions = explicitSessions && rawSessions !== null ? rawSessions : openaiPolicy.max_sessions - vm.policy.sessionsOverride = explicitSessions && rawSessions !== null - } else { - if ( - (explicitConcurrency && invalidCreate(rawConcurrency, 0, 256)) || - (explicitRpm && invalidCreate(rawRpm, 0, 1e6)) || - explicitSessions - ) { + let generated + try { + generated = assignComputerFingerprint( + cfg.paths.project, + { + id, + kernel: wantKernel, + timezone: requestedTimezone, + locale: requestedLocale, + runtime: { type: runtime }, + }, + { taken: takenFingerprintKeys(existing) }, + ) + } catch (e) { return json(res, 400, { ok: false, error: { - code: 'invalid_create_schedule', - message: 'Claude scheduling values are invalid or unsupported', + code: e.code || 'computer_create_failed', + message: String(e.message || e), }, }) } - vm.policy.maxConcurrency = explicitConcurrency - ? rawConcurrency - : Number(ctx.routingConfig?.concurrency?.default_max_per_account ?? 2) - vm.policy.concurrencyOverride = explicitConcurrency - vm.policy.maxRpm = explicitRpm ? rawRpm : Number(ctx.routingConfig?.concurrency?.default_max_rpm ?? 0) - vm.policy.rpmOverride = explicitRpm - } - atomicWriteJson(vmPath, vm, { mode: 0o600 }) - writeGuestMachineIdFile(cfg.paths.project, id, generated.guest_machine_id) - try { - seedFreshCliHome(cfg.paths.project, vm) - } catch (e) {} - let allocated = null - let proxyError = null - const wantProxy = body.auto_allocate_proxy === true || startNow || !!pickedProxyId - // px-local 没有 SOCKS URL，但它是合法出口；不要把它当成"未绑定"。 - const hasExit = (v) => !!(v?.proxy?.url || isLocalEgressProxy(v?.proxy)) - if (wantProxy && !hasExit(vm)) { - if (pickedProxyId) { - const bound = await proxyPool.bind(pickedProxyId, id) - if (bound.ok) allocated = bound.proxy - else proxyError = bound.error || 'proxy_bind_failed' + const vm = { + id, + name: body.name || padVm(idx), + status: startNow ? 'running' : body.status || 'stopped', + kernel: wantKernel, + timezone: requestedTimezone || null, + // Only an explicit zone is a pin. An empty zone stays unset so the + // next SOCKS5 bind can write the exit node's detected timezone. + timezone_source: requestedTimezone ? 'manual' : 'auto', + locale: generated.locale || STANDARD_LOCALE, + region: body.region || body.zone || null, + note: body.note || `${(OS_CATALOG[wantKernel] || {}).pretty || wantKernel} · Go slot worker`, + proxy: null, + policy: { + maxConcurrency: 0, + concurrencyOverride: false, + maxRpm: 0, + rpmOverride: false, + maxSessions: 0, + sessionsOverride: false, + weight: Math.max(1, Math.min(100, Number(body.weight ?? 1))), + inflight: 0, + }, + claude: {}, + fingerprint: applyGeneratedFingerprint({}, generated), + stats: {}, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + schedulable: false, + schedule_disabled_reason: 'no_credential', + proxy_cli_enabled: body.proxy_cli_enabled === true, + proxy_required: false, + seed_policy: standardSeedPolicy(), + runtime: { type: runtime }, + machine, + } + if (ident.role === 'user') { + vm.owner_user_id = normalizeOwnerId(req.panelUserId) + vm.origin = VM_ORIGIN.userCreated } else { - try { - allocated = await proxyPool.allocateForVm(id, { - ownerUserId: vm.owner_user_id || null, - role: ident.role, - }) - } catch (e) {} + vm.owner_user_id = null + vm.origin = VM_ORIGIN.platform } - if (allocated) { - bindVmProxy(cfg.paths.project, id, proxyPool.getProxyForVm(id)) - adoptVmProxyBinding(vm, getVm(cfg.paths.project, id)) + if (nodeId) vm.node_id = nodeId + stampVmKind(vm, body) + const openaiPolicy = normalizeOpenAIQuotaPolicy(ctx.routingConfig?.codex?.quota) + const explicitConcurrency = + Object.prototype.hasOwnProperty.call(body, 'max_concurrency') || + Object.prototype.hasOwnProperty.call(body, 'maxConcurrency') + const explicitRpm = + Object.prototype.hasOwnProperty.call(body, 'max_rpm') || + Object.prototype.hasOwnProperty.call(body, 'maxRpm') + const explicitSessions = Object.prototype.hasOwnProperty.call(body, 'max_sessions') + const readField = (snake, camel) => + Object.prototype.hasOwnProperty.call(body, snake) ? body[snake] : body[camel] + const hasField = (snake, camel) => + Object.prototype.hasOwnProperty.call(body, snake) || Object.prototype.hasOwnProperty.call(body, camel) + const rawConcurrency = readField('max_concurrency', 'maxConcurrency') + const rawRpm = readField('max_rpm', 'maxRpm') + const rawSessions = body.max_sessions + const validInteger = (value, min, max) => + typeof value === 'number' && + Number.isFinite(value) && + Number.isInteger(value) && + value >= min && + value <= max + const invalidCreate = (value, min, max) => value !== null && !validInteger(value, min, max) + if (isCodexVm(vm)) { + if ( + (explicitConcurrency && invalidCreate(rawConcurrency, 1, 256)) || + (explicitRpm && invalidCreate(rawRpm, 0, 1e6)) || + (explicitSessions && invalidCreate(rawSessions, 0, 256)) + ) { + return json(res, 400, { + ok: false, + error: { + code: 'invalid_create_schedule', + message: 'OpenAI scheduling values must be finite integers in range', + }, + }) + } + vm.policy.maxConcurrency = + explicitConcurrency && rawConcurrency !== null ? rawConcurrency : openaiPolicy.max_concurrency + vm.policy.concurrencyOverride = explicitConcurrency && rawConcurrency !== null + vm.policy.maxRpm = explicitRpm && rawRpm !== null ? rawRpm : openaiPolicy.max_rpm + vm.policy.rpmOverride = explicitRpm && rawRpm !== null + vm.policy.maxSessions = explicitSessions && rawSessions !== null ? rawSessions : openaiPolicy.max_sessions + vm.policy.sessionsOverride = explicitSessions && rawSessions !== null + } else { + if ( + (explicitConcurrency && invalidCreate(rawConcurrency, 0, 256)) || + (explicitRpm && invalidCreate(rawRpm, 0, 1e6)) || + explicitSessions + ) { + return json(res, 400, { + ok: false, + error: { + code: 'invalid_create_schedule', + message: 'Claude scheduling values are invalid or unsupported', + }, + }) + } + vm.policy.maxConcurrency = explicitConcurrency + ? rawConcurrency + : Number(ctx.routingConfig?.concurrency?.default_max_per_account ?? 2) + vm.policy.concurrencyOverride = explicitConcurrency + vm.policy.maxRpm = explicitRpm ? rawRpm : Number(ctx.routingConfig?.concurrency?.default_max_rpm ?? 0) + vm.policy.rpmOverride = explicitRpm } - } - // vm is written again below; carry the exit's zone so that write keeps it. - const zoned = await fillTimezoneFromExit(id) - if (zoned?.applied) { - vm.timezone = zoned.timezone - vm.timezone_source = 'proxy_geo' - if (vm.fingerprint && typeof vm.fingerprint === 'object') vm.fingerprint.timezone = zoned.timezone - } - if (body.activate === true) { - try { - activateVmSlot(id) - } catch (e) {} - } - let startError = null - if (startNow && hasExit(vm)) { - let boot + atomicWriteJson(vmPath, vm, { mode: 0o600 }) + writeGuestMachineIdFile(cfg.paths.project, id, generated.guest_machine_id) try { - boot = await startSlotReady(vm, cfg.paths.project, { routing: ctx.routingConfig }) + seedFreshCliHome(cfg.paths.project, vm) } catch (e) { - boot = { ok: false, error: String(e?.message || e) } + if (isKubernetesMode() || ctx.vmOperations) throw e } - if (!boot.ok) { - // Slot JSON is already on disk. 500 here makes the console treat - // create as a no-op, so the new row never refetches into the list. - patchVmRecord(cfg.paths.project, id, { - status: 'error', - schedulable: false, - schedule_disabled_reason: boot.error || 'runtime start failed', - }) - startError = boot.error || 'runtime start failed' - } else { - patchVmRecord(cfg.paths.project, id, { status: 'running' }) + let allocated = null + let proxyError = null + const wantProxy = body.auto_allocate_proxy === true || startNow || !!pickedProxyId + // px-local 没有 SOCKS URL，但它是合法出口；不要把它当成"未绑定"。 + const hasExit = (v) => !!(v?.proxy?.url || isLocalEgressProxy(v?.proxy)) + if (wantProxy && !hasExit(vm)) { + if (pickedProxyId) { + const bound = await proxyPool.bind(pickedProxyId, id) + if (bound.ok) allocated = bound.proxy + else proxyError = bound.error || 'proxy_bind_failed' + } else { + try { + allocated = await proxyPool.allocateForVm(id, { + ownerUserId: vm.owner_user_id || null, + role: ident.role, + }) + } catch (e) { + if (isKubernetesMode() || ctx.vmOperations) throw e + } + } + if (allocated) { + bindVmProxy(cfg.paths.project, id, proxyPool.getProxyForVm(id)) + adoptVmProxyBinding(vm, getVm(cfg.paths.project, id)) + } } - } else if (startNow) { - patchVmRecord(cfg.paths.project, id, { status: 'stopped' }) - } - const saved = getVm(cfg.paths.project, id) || vm - return json( - res, - 200, - panel.ok({ - vm: panel.publicVmBootView(summarizeVm(saved, cfg.paths.project, ctx.routingConfig)), - allocated_proxy: panel.publicAllocatedProxy(proxyPool, allocated, { - nodeId: saved.node_id || nodeId || null, - role: ident.role, - nodes: clusterNodesOf(ctx), + // vm is written again below; carry the exit's zone so that write keeps it. + const zoned = await fillTimezoneFromExit(id) + if (zoned?.applied) { + vm.timezone = zoned.timezone + vm.timezone_source = 'proxy_geo' + if (vm.fingerprint && typeof vm.fingerprint === 'object') vm.fingerprint.timezone = zoned.timezone + } + if (body.activate === true) { + try { + activateVmSlot(id) + } catch (e) { + if (isKubernetesMode() || ctx.vmOperations) throw e + } + } + let startError = null + if (startNow && hasExit(vm)) { + let boot + try { + boot = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.startSlotReady || startSlotReady)(vm, cfg.paths.project, { + routing: ctx.routingConfig, + }), + ) + } catch (e) { + boot = { ok: false, error: String(e?.message || e) } + } + if (!boot.ok) { + // Slot JSON is already on disk. 500 here makes the console treat + // create as a no-op, so the new row never refetches into the list. + patchVmRecord(cfg.paths.project, id, { + status: 'error', + schedulable: false, + schedule_disabled_reason: boot.error || 'runtime start failed', + }) + startError = boot.error || 'runtime start failed' + } else { + patchVmRecord(cfg.paths.project, id, { status: 'running' }) + } + } else if (startNow) { + patchVmRecord(cfg.paths.project, id, { status: 'stopped' }) + } + const saved = getVm(cfg.paths.project, id) || vm + return json( + res, + 200, + panel.ok({ + vm: panel.publicVmBootView(summarizeVm(saved, cfg.paths.project, ctx.routingConfig)), + allocated_proxy: panel.publicAllocatedProxy(proxyPool, allocated, { + nodeId: saved.node_id || nodeId || null, + role: ident.role, + nodes: clusterNodesOf(ctx), + }), + ...(startError ? { start_error: startError } : {}), + ...(proxyError ? { proxy_error: proxyError } : {}), }), - ...(startError ? { start_error: startError } : {}), - ...(proxyError ? { proxy_error: proxyError } : {}), - }), - ) + ) + }) } // POST /api/panel/vms/:id/start if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/start$/.test(p)) { const id = p.split('/')[4] - const started = await powerOnSlot({ - projectRoot: cfg.paths.project, - id, - proxyPool, - routing: ctx.routingConfig, - fillTimezone: fillTimezoneFromExit, - }) - if (!started.ok) { - return json(res, started.status, { ok: false, error: { code: started.code, message: started.error } }) - } - const { vm, bound, boot } = started - return json( - res, - 200, - panel.ok({ - vm: panel.publicVmBootView(summarizeVm(vm, cfg.paths.project, ctx.routingConfig)), - allocated_proxy: panel.publicAllocatedProxy(proxyPool, bound, { - nodeId: vm.node_id || null, - role: panelIdentity(req).role, - nodes: clusterNodesOf(ctx), + return mutateVms([id], async (handles) => { + if (!getVm(cfg.paths.project, id)) + return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'vm not found' } }) + const started = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.powerOnSlot || powerOnSlot)({ + projectRoot: cfg.paths.project, + id, + proxyPool, + routing: ctx.routingConfig, + fillTimezone: fillTimezoneFromExit, }), - runtime: panel.publicRuntimeView(vm.runtime) || GATEWAY_CAPABILITIES.runtime, - kernel: GATEWAY_CAPABILITIES.kernel, - boot: panel.publicSlotBoot(boot), - }), - ) + ) + if (!started.ok) { + return json(res, started.status, { ok: false, error: { code: started.code, message: started.error } }) + } + const { vm, bound, boot } = started + return json( + res, + 200, + panel.ok({ + vm: panel.publicVmBootView(summarizeVm(vm, cfg.paths.project, ctx.routingConfig)), + allocated_proxy: panel.publicAllocatedProxy(proxyPool, bound, { + nodeId: vm.node_id || null, + role: panelIdentity(req).role, + nodes: clusterNodesOf(ctx), + }), + runtime: panel.publicRuntimeView(vm.runtime) || GATEWAY_CAPABILITIES.runtime, + kernel: GATEWAY_CAPABILITIES.kernel, + boot: panel.publicSlotBoot(boot), + }), + ) + }) } // GET /api/panel/vms/:id/diagnose — read-only check of every service the slot hop needs if (req.method === 'GET' && /^\/api\/panel\/vms\/[^/]+\/diagnose$/.test(p)) { @@ -4372,43 +4495,52 @@ export function createPanelHandler(ctx) { } // POST /api/panel/vms/:id/recover — power cycle (stop → start) plus a fresh diagnosis if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/recover$/.test(p)) { - const recovered = await recoverSlot({ - projectRoot: cfg.paths.project, - id: p.split('/')[4], - proxyPool, - routing: ctx.routingConfig, - fillTimezone: fillTimezoneFromExit, - watchdog: ctx.kernelWatchdog, - internal: panelIdentity(req).role === 'admin', + const id = p.split('/')[4] + return mutateVms([id], async (handles) => { + if (!getVm(cfg.paths.project, id)) + return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'vm not found' } }) + const recovered = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.recoverSlot || recoverSlot)({ + projectRoot: cfg.paths.project, + id: p.split('/')[4], + proxyPool, + routing: ctx.routingConfig, + fillTimezone: fillTimezoneFromExit, + watchdog: ctx.kernelWatchdog, + internal: panelIdentity(req).role === 'admin', + }), + ) + if (!recovered.ok) { + return json(res, recovered.status, { ok: false, error: { code: recovered.code, message: recovered.error } }) + } + return json(res, 200, panel.ok(recovered.report)) }) - if (!recovered.ok) { - return json(res, recovered.status, { ok: false, error: { code: recovered.code, message: recovered.error } }) - } - return json(res, 200, panel.ok(recovered.report)) } // POST /api/panel/vms/:id/stop if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/stop$/.test(p)) { const id = p.split('/')[4] - const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) - if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) - const halt = await stopSlot(vm) - if (!halt.ok) return json(res, 500, { ok: false, error: { message: halt.error || 'runtime stop failed' } }) - vm.status = 'stopped' - vm.schedulable = false - vm.schedule_disabled_reason = 'stopped' - vm.updated_at = new Date().toISOString() - atomicWriteJson(vmPath, vm, { mode: 0o600 }) - return json( - res, - 200, - panel.ok({ - vm: summarizeVm(vm, cfg.paths.project, ctx.routingConfig), - runtime: panel.publicRuntimeView(vm.runtime) || GATEWAY_CAPABILITIES.runtime, - kernel: GATEWAY_CAPABILITIES.kernel, - halt: panel.publicSlotBoot(halt), - }), - ) + return mutateVms([id], async (handles) => { + const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) + if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) + const halt = await vmEffect(handles.get(id), () => (ctx.vmMutationOps?.stopSlot || stopSlot)(vm)) + if (!halt.ok) return json(res, 500, { ok: false, error: { message: halt.error || 'runtime stop failed' } }) + vm.status = 'stopped' + vm.schedulable = false + vm.schedule_disabled_reason = 'stopped' + vm.updated_at = new Date().toISOString() + atomicWriteJson(vmPath, vm, { mode: 0o600 }) + return json( + res, + 200, + panel.ok({ + vm: summarizeVm(vm, cfg.paths.project, ctx.routingConfig), + runtime: panel.publicRuntimeView(vm.runtime) || GATEWAY_CAPABILITIES.runtime, + kernel: GATEWAY_CAPABILITIES.kernel, + halt: panel.publicSlotBoot(halt), + }), + ) + }) } // POST /api/panel/vms/import @@ -6005,6 +6137,17 @@ export function createPanelHandler(ctx) { res._panelResponseState = state try { result = await handlePanel(req, res, url) + } catch (error) { + if ( + ['vm_operation_active', 'vm_execution_active', 'vm_physical_target_conflict', 'invalid_vm_operation'].includes( + error.code, + ) + ) { + state.response = [res, error.status || 409, { ok: false, error: { code: error.code, message: error.message } }] + result = true + } else if (error.panelPersistenceFailure) { + return persistenceFailure(res, error) + } else throw error } finally { delete res._panelResponseState } diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index fb446e7e2..19efc4ace 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -108,6 +108,12 @@ export class ExecutionAdmissionsRepo { .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') .get(physicalVmAliases(vmId)) if (operation) return denied('vm_operation_active', 0, vmLimit) + // A candidate may have been read before a completed management hold. + // Check every physical alias under the same VM locks after the hold commits. + const held = await this.db + .prepare('SELECT id FROM vms WHERE id = ANY($1::text[]) AND schedulable IS DISTINCT FROM 1 LIMIT 1') + .get(physicalVmAliases(vmId)) + if (held) return denied('vm_not_schedulable', 0, vmLimit) const alias = await this.db .prepare(`SELECT id FROM execution_admissions WHERE vm_id = ANY($1::text[]) AND vm_id <> $2 AND released_at IS NULL LIMIT 1`) diff --git a/src/lib/vm/vm-mutations.mjs b/src/lib/vm/vm-mutations.mjs new file mode 100644 index 000000000..cd1185e42 --- /dev/null +++ b/src/lib/vm/vm-mutations.mjs @@ -0,0 +1,39 @@ +import path from 'node:path' +import { isValidVmId, withVmLock } from './vm-file.mjs' +import { physicalContainerName } from './vm-physical-target.mjs' + +/** Acquire the complete target set before a batch can change any VM. */ +export function withVmMutations(projectRoot, ids, callback, { operations, kind = 'mutation' } = {}) { + const ordered = [...new Set(ids)] + for (const id of ordered) { + if (!isValidVmId(id)) + throw Object.assign(new Error('invalid VM operation ID'), { code: 'invalid_vm_operation', status: 400 }) + } + ordered.sort((a, b) => physicalContainerName(a).localeCompare(physicalContainerName(b)) || a.localeCompare(b)) + const handles = new Map() + let entered = false + const acquire = (index) => { + if (index === ordered.length) { + entered = true + return callback(handles) + } + const id = ordered[index] + return withVmLock( + path.join(projectRoot, 'vms', `${id}.json`), + async (operation) => { + handles.set(id, operation) + try { + return await acquire(index + 1) + } catch (error) { + // Nothing was sent or written when a later target refused its claim. + if (!entered) error.vmOperationNotStarted = true + throw error + } finally { + handles.delete(id) + } + }, + { operations, vmId: id, kind }, + ) + } + return acquire(0) +} diff --git a/test/unit/panel-vm-delete.test.mjs b/test/unit/panel-vm-delete.test.mjs index 48d604f90..c5ec304d0 100644 --- a/test/unit/panel-vm-delete.test.mjs +++ b/test/unit/panel-vm-delete.test.mjs @@ -35,6 +35,7 @@ function handler(project) { return true }, readBody: async () => ({}), + vmMutationOps: { destroySlot: async () => ({ ok: true, action: 'absent' }) }, proxyPool: { unbindVm() {} }, }) return { handlePanel, response, cfg } diff --git a/test/unit/panel-vm-shared-mutations.test.mjs b/test/unit/panel-vm-shared-mutations.test.mjs index bd4944fc5..80823e60b 100644 --- a/test/unit/panel-vm-shared-mutations.test.mjs +++ b/test/unit/panel-vm-shared-mutations.test.mjs @@ -9,6 +9,8 @@ import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { initVmDbSync, stopVmWatch } from '../../src/lib/vm/vm-db-sync.mjs' +import { setVmWriteHook } from '../../src/lib/vm/vm-file.mjs' async function fixture(t) { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-shared-vm-')) @@ -166,3 +168,139 @@ test('a normal non-ok stop result preserves its durable claim', async (t) => { assert.equal(h.response.status, 500) assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' })).ok, false) }) + +test('a completed stop keeps its claim until SQL/file mirror writes finish', async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + let finish + let flushing + const pending = new Promise((resolve) => { + finish = resolve + }) + const started = new Promise((resolve) => { + flushing = resolve + }) + let calls = 0 + const h = f.handler( + {}, + { + vmMutationOps: { stopSlot: async () => ({ ok: true, action: 'stopped' }) }, + accountQuota: { + writes: { + flush: async () => { + if (++calls === 2) { + flushing() + await pending + } + }, + }, + }, + }, + ) + const request = h.request('POST', '/api/panel/vms/vm-synthetic/stop') + await started + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' })).ok, false) + finish() + await request + assert.equal(h.response.status, 200) + const next = await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' }) + assert.equal(next.ok, true) + await next.release() +}) + +test('create HTTP success with a failed start keeps the VM operation unknown', async (t) => { + const f = await fixture(t) + const proxy = { id: 'synthetic-proxy', url: 'socks5://synthetic.invalid:1080' } + const h = f.handler( + { id: 'vm-synthetic', start: true, timezone: 'UTC' }, + { + proxyPool: { allocateForVm: async () => proxy, getProxyForVm: () => proxy, snapshot: () => ({ proxies: [] }) }, + vmMutationOps: { + startSlotReady: async () => ({ + ok: false, + error: 'connection dropped after start', + vmOperationUncertain: true, + }), + }, + }, + ) + await h.request('POST', '/api/panel/vms/create') + assert.equal(h.response.status, 200, JSON.stringify(h.response.body)) + assert.equal(h.response.body.data.start_error, 'connection dropped after start') + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' })).ok, false) +}) + +test('delete keeps files and claim when local destroy returns an unknown result', async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + const h = f.handler( + {}, + { + vmMutationOps: { + destroySlot: async () => ({ ok: false, code: 'synthetic_unknown', vmOperationUncertain: true }), + }, + }, + ) + await h.request('DELETE', '/api/panel/vms/vm-synthetic') + assert.equal(h.response.status, 409) + assert.equal(fs.existsSync(path.join(f.project, 'vms', 'vm-synthetic.json')), true) + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' })).ok, false) +}) + +test('an explicit not-started stop failure can release its claim', async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + const h = f.handler( + {}, + { vmMutationOps: { stopSlot: async () => ({ ok: false, code: 'synthetic_not_started', notStarted: true }) } }, + ) + await h.request('POST', '/api/panel/vms/vm-synthetic/stop') + assert.equal(h.response.status, 500) + const next = await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' }) + assert.equal(next.ok, true) + await next.release() +}) + +test('a successful panel hold publishes PG state before releasing its global idle claims', async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + await initVmDbSync(f.project, { watch: false }) + t.after(() => { + stopVmWatch() + setVmWriteHook(null) + }) + const h = f.handler({ reason: 'synthetic_maintenance' }) + await h.request('POST', '/api/panel/vms/schedule/hold') + assert.equal(h.response.status, 200, JSON.stringify(h.response.body)) + assert.equal(h.response.body.data.drained, true) + assert.equal((await f.peer.prepare('SELECT schedulable FROM vms WHERE id = $1').get('vm-synthetic')).schedulable, 0) + const stale = await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + vmId: 'vm-synthetic', + accountId: 'synthetic', + accountLimit: 2, + vmLimit: 2, + rpmLimit: 0, + fableLimit: 0, + }) + assert.equal(stale.ok, false) + assert.equal(stale.code, 'vm_not_schedulable') +}) + +test('delete retains ownership and files when required proxy persistence is unconfirmed', async (t) => { + const f = await fixture(t) + f.seed('vm-synthetic') + const h = f.handler( + {}, + { + vmMutationOps: { destroySlot: async () => ({ ok: true, action: 'removed' }) }, + proxyPool: { + unbindVm: async () => { + throw new Error('SQL acknowledgement lost') + }, + }, + }, + ) + await assert.rejects(h.request('DELETE', '/api/panel/vms/vm-synthetic'), /SQL acknowledgement lost/) + assert.equal(fs.existsSync(path.join(f.project, 'vms', 'vm-synthetic.json')), true) + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' })).ok, false) +}) From 0e23ca05797b9fd94f7b170bd8f8da8533f81ffc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:01:17 +0800 Subject: [PATCH 217/473] test: use synthetic computer identity fixture for panel mutation tests --- test/unit/panel-vm-shared-mutations.test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/panel-vm-shared-mutations.test.mjs b/test/unit/panel-vm-shared-mutations.test.mjs index 80823e60b..7ac6297c9 100644 --- a/test/unit/panel-vm-shared-mutations.test.mjs +++ b/test/unit/panel-vm-shared-mutations.test.mjs @@ -1,3 +1,4 @@ +import '../fixtures/kin-computer-test-env.mjs' import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' From 3c7986635118810690058551e2a6ad96846359c0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:01:25 +0800 Subject: [PATCH 218/473] feat(slotgate): persist executor generations and fail closed on unknown actions --- worker/internal/slotgate/authority.go | 720 ++++++++++++++++++ .../internal/slotgate/authority_lock_other.go | 11 + .../internal/slotgate/authority_lock_unix.go | 41 + worker/internal/slotgate/authority_test.go | 149 ++++ 4 files changed, 921 insertions(+) create mode 100644 worker/internal/slotgate/authority.go create mode 100644 worker/internal/slotgate/authority_lock_other.go create mode 100644 worker/internal/slotgate/authority_lock_unix.go diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go new file mode 100644 index 000000000..8f5ab182b --- /dev/null +++ b/worker/internal/slotgate/authority.go @@ -0,0 +1,720 @@ +package slotgate + +// Authority is the durable node-side state machine for executor incarnations. +// It is intentionally not wired into KSG1: that protocol and unrestricted +// SSH/Docker/SFTP access must be removed before this can provide fencing. +// Callers here are trusted node adapters, not remote protocol clients. + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ( + ErrAuthorityLocked = errors.New("authority_locked") + ErrAuthorityUnavailable = errors.New("authority_unavailable") + ErrAuthorityPhase = errors.New("authority_phase") + ErrFenceMismatch = errors.New("fence_mismatch") + ErrCommandExists = errors.New("command_exists") + ErrCommandOwner = errors.New("command_owner") + ErrCommandDispatched = errors.New("command_dispatched") + ErrPendingMutation = errors.New("pending_mutation") + ErrInvalidAuthority = errors.New("invalid_authority") + ErrTerminationProof = errors.New("termination_unconfirmed") +) + +type AuthorityPhase string + +const ( + PhaseVacant AuthorityPhase = "vacant" + PhaseActive AuthorityPhase = "active" + PhaseRevoking AuthorityPhase = "revoking" +) + +type CommandKind string +type CommandStatus string + +const ( + CommandForward CommandKind = "forward" + CommandMutation CommandKind = "mutation" + commandLaunch CommandKind = "launch" + CommandPending CommandStatus = "pending" + CommandUnknown CommandStatus = "unknown" + CommandCompleted CommandStatus = "completed" + CommandTerminated CommandStatus = "terminated" +) + +// ExecutorIdentity names an incarnation, never a reusable container name or +// PID alone. ProcessScope must identify a node-owned scope containing every +// descendant; a verifier must check the whole scope, not only its initial PID. +type ExecutorIdentity struct { + Kind string `json:"kind"` + NodeBootID string `json:"node_boot_id"` + ContainerID string `json:"container_id,omitempty"` + StartedAt string `json:"started_at,omitempty"` + PID int `json:"pid,omitempty"` + ProcessStart uint64 `json:"process_start,omitempty"` + ProcessScope string `json:"process_scope,omitempty"` +} + +// Fence is captured when work is created. A delayed operation must never be +// relabeled with a fresh State() value. Controllers share an active incarnation; +// generation is independent of each controller's admission owner ID. +type Fence struct { + Generation uint64 `json:"generation"` + Incarnation string `json:"incarnation"` + Executor ExecutorIdentity `json:"executor"` +} + +type CommandRecord struct { + ID string `json:"id"` + Kind CommandKind `json:"kind"` + Status CommandStatus `json:"status"` + Fence Fence `json:"fence"` + Runtime string `json:"runtime"` + Token string `json:"token"` + Dispatched bool `json:"dispatched"` + TerminationID string `json:"termination_id,omitempty"` +} + +type AuthorityState struct { + Fence Fence `json:"fence"` + Phase AuthorityPhase `json:"phase"` + Commands map[string]CommandRecord `json:"commands"` + LastTermination *TerminationProof `json:"last_termination,omitempty"` + Retired []ExecutorIdentity `json:"retired,omitempty"` +} + +type CommandHandle struct { + VMID string + ID string + Fence Fence + runtime string + token string +} + +// TerminationProof can only originate from the trusted node verifier. Neither +// a client-provided boolean, socket EOF, cancel ACK nor aggregate health is a +// proof. Container verifiers must also exclude restart of the revoked target. +type TerminationProof struct { + ID string `json:"id"` + Executor ExecutorIdentity `json:"executor"` + Scope string `json:"scope"` + ObservedAt string `json:"observed_at"` +} + +type TerminationVerifier func(context.Context, ExecutorIdentity) (TerminationProof, error) + +type authorityJournal struct { + Version int `json:"version"` + Slots map[string]AuthorityState `json:"slots"` +} + +type Authority struct { + mu sync.Mutex + root string + lock *os.File + runtime string + verify TerminationVerifier + journal authorityJournal + unavailable bool + closed bool +} + +const authorityMarker = "slotgate-authority-v1\n" + +// OpenAuthority requires a private node-owned directory outside slot mounts. +// The process lock is held until Close; a second daemon cannot serve an older +// in-memory generation. Missing/corrupt existing state is never repaired here. +func OpenAuthority(root string, verify TerminationVerifier) (*Authority, error) { + if !filepath.IsAbs(root) { + return nil, ErrInvalidAuthority + } + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, err + } + info, err := os.Lstat(root) + if err != nil { + return nil, err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o022 != 0 { + return nil, ErrInvalidAuthority + } + lock, err := lockAuthority(filepath.Join(root, "authority.lock")) + if err != nil { + return nil, err + } + runtime, err := randomAuthorityID() + if err != nil { + lock.Close() + return nil, err + } + a := &Authority{root: root, lock: lock, runtime: runtime, verify: verify, journal: authorityJournal{Version: 1, Slots: map[string]AuthorityState{}}} + fail := func(err error) (*Authority, error) { a.Close(); return nil, err } + lockInfo, err := lock.Stat() + if err != nil { + return fail(err) + } + initialized := lockInfo.Size() != 0 + if initialized { + marker := make([]byte, len(authorityMarker)) + if lockInfo.Size() != int64(len(marker)) { + return fail(ErrInvalidAuthority) + } + if _, err = lock.ReadAt(marker, 0); err != nil { + return fail(err) + } + if string(marker) != authorityMarker { + return fail(ErrInvalidAuthority) + } + } + file, err := openAuthorityFile(filepath.Join(root, "authority.json"), os.O_RDONLY, 0) + if errors.Is(err, os.ErrNotExist) { + if initialized { + return fail(ErrInvalidAuthority) + } + if err = a.persist(a.journal); err != nil { + return fail(err) + } + if err = a.markInitialized(); err != nil { + return fail(err) + } + return a, nil + } + if err != nil { + return fail(err) + } + decode := json.NewDecoder(file) + decode.DisallowUnknownFields() + err = decode.Decode(&a.journal) + if err == nil { + var extra any + if trailing := decode.Decode(&extra); trailing != io.EOF { + err = ErrInvalidAuthority + } + } + closeErr := file.Close() + if err != nil { + return fail(fmt.Errorf("%w: %v", ErrInvalidAuthority, err)) + } + if closeErr != nil { + return fail(closeErr) + } + if err = validateJournal(a.journal); err != nil { + return fail(err) + } + changed := false + for vm, state := range a.journal.Slots { + for id, command := range state.Commands { + if command.Status == CommandPending || command.Status == CommandUnknown { + command.Status = CommandUnknown + state.Commands[id] = command + state.Phase = PhaseRevoking + changed = true + } + } + a.journal.Slots[vm] = state + } + if changed { + if err = a.persist(a.journal); err != nil { + return fail(err) + } + } + if !initialized { + if err = a.markInitialized(); err != nil { + return fail(err) + } + } + return a, nil +} + +func (a *Authority) markInitialized() error { + if _, err := a.lock.WriteAt([]byte(authorityMarker), 0); err != nil { + return err + } + return a.lock.Sync() +} + +func (a *Authority) Close() error { + a.mu.Lock() + defer a.mu.Unlock() + if a.closed { + return nil + } + a.closed = true + return a.lock.Close() +} + +func (a *Authority) State(vm string) (AuthorityState, error) { + a.mu.Lock() + defer a.mu.Unlock() + if err := a.available(); err != nil { + return AuthorityState{}, err + } + state, exists := a.journal.Slots[vm] + if !exists { + return AuthorityState{}, ErrInvalidAuthority + } + return cloneState(state), nil +} + +// CreateSlot only allocates a vacant generation. It does not discover or adopt +// an existing executor, which would require separate bootstrap proof. +func (a *Authority) CreateSlot(vm string) (AuthorityState, error) { + a.mu.Lock() + defer a.mu.Unlock() + if err := a.available(); err != nil { + return AuthorityState{}, err + } + if !vmIDRe.MatchString(vm) { + return AuthorityState{}, ErrInvalidAuthority + } + if _, exists := a.journal.Slots[vm]; exists { + return AuthorityState{}, ErrFenceMismatch + } + incarnation, err := randomAuthorityID() + if err != nil { + return AuthorityState{}, err + } + state := AuthorityState{Fence: Fence{Generation: 1, Incarnation: incarnation}, Phase: PhaseVacant, Commands: map[string]CommandRecord{}} + return a.commit(vm, state) +} + +func (a *Authority) BeginLaunch(vm string, expected Fence, commandID string) (CommandHandle, error) { + return a.begin(vm, expected, commandID, commandLaunch, PhaseVacant) +} + +// BeginCommand must commit before any bytes or mutation are sent. Command IDs +// remain tombstoned across generations, preventing retry by relabeling an ID. +func (a *Authority) BeginCommand(vm string, expected Fence, commandID string, kind CommandKind) (CommandHandle, error) { + if kind != CommandForward && kind != CommandMutation { + return CommandHandle{}, ErrInvalidAuthority + } + return a.begin(vm, expected, commandID, kind, PhaseActive) +} + +// DispatchCommand checks the captured incarnation at the actual send boundary. +// dispatch must synchronously start the operation against the supplied exact +// executor; it must not enqueue later work or resolve a reusable slot path. +// The lock covers this initial send, not the job's execution or response. It +// must return before calling any other Authority method. A send error leaves a +// durable pending command and cannot be retried with this command ID. +func (a *Authority) DispatchCommand(handle CommandHandle, dispatch func(ExecutorIdentity) error) error { + a.mu.Lock() + defer a.mu.Unlock() + state, command, err := a.command(handle) + if err != nil { + return err + } + phase := PhaseActive + if command.Kind == commandLaunch { + phase = PhaseVacant + } + if state.Phase != phase || command.Status != CommandPending { + return ErrAuthorityPhase + } + if command.Dispatched { + return ErrCommandDispatched + } + if dispatch == nil { + return ErrInvalidAuthority + } + command.Dispatched = true + state.Commands[handle.ID] = command + if _, err = a.commit(handle.VMID, state); err != nil { + return err + } + return dispatch(handle.Fence.Executor) +} + +func (a *Authority) begin(vm string, expected Fence, id string, kind CommandKind, phase AuthorityPhase) (CommandHandle, error) { + a.mu.Lock() + defer a.mu.Unlock() + state, err := a.match(vm, expected) + if err != nil { + return CommandHandle{}, err + } + if state.Phase != phase { + return CommandHandle{}, ErrAuthorityPhase + } + if !validAuthorityID(id) { + return CommandHandle{}, ErrInvalidAuthority + } + if _, exists := state.Commands[id]; exists { + return CommandHandle{}, ErrCommandExists + } + if kind == commandLaunch && hasPendingMutation(state) { + return CommandHandle{}, ErrPendingMutation + } + token, err := randomAuthorityID() + if err != nil { + return CommandHandle{}, err + } + state.Commands[id] = CommandRecord{ID: id, Kind: kind, Status: CommandPending, Fence: expected, Runtime: a.runtime, Token: token} + if _, err = a.commit(vm, state); err != nil { + return CommandHandle{}, err + } + return CommandHandle{VMID: vm, ID: id, Fence: expected, runtime: a.runtime, token: token}, nil +} + +// BindExecutor records the exact identity observed after this daemon's launch. +// A launch from an earlier daemon is unknown and cannot be adopted implicitly. +func (a *Authority) BindExecutor(handle CommandHandle, executor ExecutorIdentity) (AuthorityState, error) { + a.mu.Lock() + defer a.mu.Unlock() + if err := validateExecutor(executor); err != nil { + return AuthorityState{}, err + } + state, command, err := a.command(handle) + if err != nil { + return AuthorityState{}, err + } + if state.Phase != PhaseVacant || command.Kind != commandLaunch || command.Status != CommandPending || !command.Dispatched { + return AuthorityState{}, ErrAuthorityPhase + } + for _, retired := range state.Retired { + if executor == retired { + return AuthorityState{}, ErrFenceMismatch + } + } + command.Status = CommandCompleted + state.Commands[handle.ID] = command + state.Fence.Executor = executor + state.Phase = PhaseActive + return a.commit(handle.VMID, state) +} + +// CompleteCommand is for a trusted node adapter's confirmed completion only. +// A timeout or lost connection must leave the command pending. +func (a *Authority) CompleteCommand(handle CommandHandle) error { + a.mu.Lock() + defer a.mu.Unlock() + state, command, err := a.command(handle) + if err != nil { + return err + } + if command.Status == CommandCompleted { + return nil + } + if command.Status != CommandPending { + return ErrCommandOwner + } + command.Status = CommandCompleted + state.Commands[handle.ID] = command + _, err = a.commit(handle.VMID, state) + return err +} + +func (a *Authority) Revoke(vm string, expected Fence) (AuthorityState, error) { + a.mu.Lock() + defer a.mu.Unlock() + state, err := a.match(vm, expected) + if err != nil { + return AuthorityState{}, err + } + if state.Phase == PhaseRevoking { + return state, nil + } + state.Phase = PhaseRevoking + return a.commit(vm, state) +} + +// Advance verifies termination outside the journal lock so unrelated VMs keep +// serving. It rechecks the captured fence after verification. Unknown mutations +// block advancement even if the old executor is gone: a delayed daemon action +// could otherwise restart it. No automatic repair for those actions is exposed. +func (a *Authority) Advance(ctx context.Context, vm string, expected Fence) (AuthorityState, error) { + a.mu.Lock() + _, err := a.canAdvance(vm, expected) + a.mu.Unlock() + if err != nil { + return AuthorityState{}, err + } + if a.verify == nil { + return AuthorityState{}, ErrTerminationProof + } + proof, err := a.verify(ctx, expected.Executor) + if err != nil { + return AuthorityState{}, err + } + if err = ctx.Err(); err != nil { + return AuthorityState{}, err + } + if err = validateProof(proof, expected.Executor); err != nil { + return AuthorityState{}, err + } + incarnation, err := randomAuthorityID() + if err != nil { + return AuthorityState{}, err + } + a.mu.Lock() + defer a.mu.Unlock() + state, err := a.canAdvance(vm, expected) + if err != nil { + return AuthorityState{}, err + } + if expected.Generation == ^uint64(0) { + return AuthorityState{}, ErrInvalidAuthority + } + for id, command := range state.Commands { + if pendingCommand(command) { + command.Status = CommandTerminated + command.TerminationID = proof.ID + state.Commands[id] = command + } + } + state.Fence = Fence{Generation: expected.Generation + 1, Incarnation: incarnation} + state.Retired = append(state.Retired, expected.Executor) + state.Phase = PhaseVacant + state.LastTermination = &proof + return a.commit(vm, state) +} + +func (a *Authority) canAdvance(vm string, expected Fence) (AuthorityState, error) { + state, err := a.match(vm, expected) + if err != nil { + return AuthorityState{}, err + } + if state.Phase != PhaseRevoking { + return AuthorityState{}, ErrAuthorityPhase + } + if hasPendingMutation(state) { + return AuthorityState{}, ErrPendingMutation + } + if err = validateExecutor(state.Fence.Executor); err != nil { + return AuthorityState{}, ErrTerminationProof + } + return state, nil +} + +func (a *Authority) command(handle CommandHandle) (AuthorityState, CommandRecord, error) { + if err := a.available(); err != nil { + return AuthorityState{}, CommandRecord{}, err + } + if handle.runtime != a.runtime { + return AuthorityState{}, CommandRecord{}, ErrCommandOwner + } + state, err := a.match(handle.VMID, handle.Fence) + if err != nil { + return AuthorityState{}, CommandRecord{}, err + } + command, exists := state.Commands[handle.ID] + if !exists || command.Runtime != a.runtime || command.Token != handle.token || command.Fence != handle.Fence { + return AuthorityState{}, CommandRecord{}, ErrCommandOwner + } + return state, command, nil +} + +func (a *Authority) match(vm string, expected Fence) (AuthorityState, error) { + if err := a.available(); err != nil { + return AuthorityState{}, err + } + state, exists := a.journal.Slots[vm] + if !exists || state.Fence != expected { + return AuthorityState{}, ErrFenceMismatch + } + return cloneState(state), nil +} + +func (a *Authority) available() error { + if a.closed || a.unavailable { + return ErrAuthorityUnavailable + } + return nil +} + +func (a *Authority) commit(vm string, state AuthorityState) (AuthorityState, error) { + next := authorityJournal{Version: 1, Slots: make(map[string]AuthorityState, len(a.journal.Slots)+1)} + for key, value := range a.journal.Slots { + next.Slots[key] = value + } + next.Slots[vm] = state + if err := a.persist(next); err != nil { + return AuthorityState{}, err + } + a.journal = next + return cloneState(state), nil +} + +// fsync the temporary file, atomically replace the journal, then fsync its +// directory. Any persistence error poisons this daemon, including an uncertain +// post-rename outcome. It must reopen and validate disk before authorizing work. +func (a *Authority) persist(next authorityJournal) error { + data, err := json.Marshal(next) + if err != nil { + a.unavailable = true + return err + } + file, err := os.CreateTemp(a.root, ".authority-*") + if err != nil { + a.unavailable = true + return err + } + defer os.Remove(file.Name()) + if _, err = file.Write(append(data, '\n')); err == nil { + err = file.Sync() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err == nil { + err = os.Rename(file.Name(), filepath.Join(a.root, "authority.json")) + } + if err == nil { + var dir *os.File + dir, err = os.Open(a.root) + if err == nil { + err = dir.Sync() + if closeErr := dir.Close(); err == nil { + err = closeErr + } + } + } + if err != nil { + a.unavailable = true + } + return err +} + +func cloneState(state AuthorityState) AuthorityState { + commands := make(map[string]CommandRecord, len(state.Commands)) + for id, command := range state.Commands { + commands[id] = command + } + state.Commands = commands + state.Retired = append([]ExecutorIdentity(nil), state.Retired...) + if state.LastTermination != nil { + proof := *state.LastTermination + state.LastTermination = &proof + } + return state +} + +func pendingCommand(command CommandRecord) bool { + return command.Status == CommandPending || command.Status == CommandUnknown +} + +func hasPendingMutation(state AuthorityState) bool { + for _, command := range state.Commands { + if pendingCommand(command) && command.Kind != CommandForward { + return true + } + } + return false +} + +func validAuthorityID(id string) bool { + return strings.TrimSpace(id) == id && id != "" && len(id) <= 256 +} + +func randomAuthorityID() (string, error) { + var bytes [16]byte + if _, err := rand.Read(bytes[:]); err != nil { + return "", err + } + return hex.EncodeToString(bytes[:]), nil +} + +func validateExecutor(e ExecutorIdentity) error { + if !validAuthorityID(e.NodeBootID) { + return ErrInvalidAuthority + } + switch e.Kind { + case "container": + id, err := hex.DecodeString(e.ContainerID) + if err != nil || len(id) != 32 || e.PID != 0 || e.ProcessStart != 0 || e.ProcessScope != "" { + return ErrInvalidAuthority + } + if _, err = time.Parse(time.RFC3339Nano, e.StartedAt); err != nil { + return ErrInvalidAuthority + } + case "process": + if e.PID <= 0 || e.ProcessStart == 0 || !validAuthorityID(e.ProcessScope) || e.ContainerID != "" || e.StartedAt != "" { + return ErrInvalidAuthority + } + default: + return ErrInvalidAuthority + } + return nil +} + +func validateProof(proof TerminationProof, expected ExecutorIdentity) error { + if proof.Executor != expected || !validAuthorityID(proof.ID) { + return ErrTerminationProof + } + if _, err := time.Parse(time.RFC3339Nano, proof.ObservedAt); err != nil { + return ErrTerminationProof + } + if expected.Kind == "container" && (proof.Scope == "container_removed" || proof.Scope == "container_stopped") { + return nil + } + if expected.Kind == "process" && proof.Scope == "process_scope_empty" { + return nil + } + return ErrTerminationProof +} + +func validateJournal(journal authorityJournal) error { + if journal.Version != 1 || journal.Slots == nil { + return ErrInvalidAuthority + } + for vm, state := range journal.Slots { + if !vmIDRe.MatchString(vm) || state.Fence.Generation == 0 || !validAuthorityID(state.Fence.Incarnation) || state.Commands == nil { + return ErrInvalidAuthority + } + if state.Phase != PhaseActive && state.Phase != PhaseVacant && state.Phase != PhaseRevoking { + return ErrInvalidAuthority + } + if state.Phase == PhaseActive || state.Fence.Executor.Kind != "" { + if err := validateExecutor(state.Fence.Executor); err != nil { + return err + } + } else if state.Fence.Executor != (ExecutorIdentity{}) { + return ErrInvalidAuthority + } + if state.Phase == PhaseVacant && state.Fence.Executor != (ExecutorIdentity{}) { + return ErrInvalidAuthority + } + for id, command := range state.Commands { + if id != command.ID || !validAuthorityID(id) || !validAuthorityID(command.Runtime) || !validAuthorityID(command.Token) || command.Fence.Generation == 0 || command.Fence.Generation > state.Fence.Generation || !validAuthorityID(command.Fence.Incarnation) { + return ErrInvalidAuthority + } + if command.Kind != CommandForward && command.Kind != CommandMutation && command.Kind != commandLaunch { + return ErrInvalidAuthority + } + if command.Status != CommandPending && command.Status != CommandUnknown && command.Status != CommandCompleted && command.Status != CommandTerminated { + return ErrInvalidAuthority + } + if pendingCommand(command) && command.Fence != state.Fence { + return ErrInvalidAuthority + } + } + if state.LastTermination != nil { + if err := validateExecutor(state.LastTermination.Executor); err != nil { + return err + } + if err := validateProof(*state.LastTermination, state.LastTermination.Executor); err != nil { + return err + } + } + for _, retired := range state.Retired { + if err := validateExecutor(retired); err != nil { + return err + } + if state.Fence.Executor == retired { + return ErrInvalidAuthority + } + } + } + return nil +} diff --git a/worker/internal/slotgate/authority_lock_other.go b/worker/internal/slotgate/authority_lock_other.go new file mode 100644 index 000000000..5fb61edde --- /dev/null +++ b/worker/internal/slotgate/authority_lock_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin + +package slotgate + +import "os" + +// A platform without the node authority's process lock must fail closed. +func openAuthorityFile(string, int, os.FileMode) (*os.File, error) { + return nil, ErrAuthorityUnavailable +} +func lockAuthority(string) (*os.File, error) { return nil, ErrAuthorityUnavailable } diff --git a/worker/internal/slotgate/authority_lock_unix.go b/worker/internal/slotgate/authority_lock_unix.go new file mode 100644 index 000000000..52e7db45f --- /dev/null +++ b/worker/internal/slotgate/authority_lock_unix.go @@ -0,0 +1,41 @@ +//go:build linux || darwin + +package slotgate + +import ( + "errors" + "os" + "syscall" +) + +func openAuthorityFile(path string, flags int, mode os.FileMode) (*os.File, error) { + fd, err := syscall.Open(path, flags|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, uint32(mode.Perm())) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), path) + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o022 != 0 { + file.Close() + if err != nil { + return nil, err + } + return nil, ErrInvalidAuthority + } + return file, nil +} + +func lockAuthority(path string) (*os.File, error) { + file, err := openAuthorityFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + if err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + file.Close() + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return nil, ErrAuthorityLocked + } + return nil, err + } + return file, nil +} diff --git a/worker/internal/slotgate/authority_test.go b/worker/internal/slotgate/authority_test.go index f60523e34..82dc91c71 100644 --- a/worker/internal/slotgate/authority_test.go +++ b/worker/internal/slotgate/authority_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -38,6 +39,9 @@ func launchForTest(t *testing.T, a *Authority, vm string, executor ExecutorIdent if err != nil { t.Fatal(err) } + if err = a.DispatchCommand(launch, func(ExecutorIdentity) error { return nil }); err != nil { + t.Fatal(err) + } s, err = a.BindExecutor(launch, executor) if err != nil { t.Fatal(err) @@ -103,6 +107,12 @@ func TestAuthorityRevocationRequiresProofAndRejectsCapturedOldCommands(t *testin if err != nil { t.Fatal(err) } + if err = a.DispatchCommand(launch, func(ExecutorIdentity) error { return nil }); err != nil { + t.Fatal(err) + } + if _, err = a.BindExecutor(launch, testExecutor("a")); !errors.Is(err, ErrFenceMismatch) { + t.Fatalf("retired executor was rebound: %v", err) + } active, err := a.BindExecutor(launch, testExecutor("b")) if err != nil { t.Fatal(err) @@ -275,12 +285,151 @@ func TestAuthoritySerializesSameNodeJournalAcrossProcesses(t *testing.T) { a := authorityForTest(t, root, terminatedExecutor) _, err := OpenAuthority(root, terminatedExecutor) requireAuthorityError(t, err, ErrAuthorityLocked) + probe := exec.Command(os.Args[0], "-test.run=^TestAuthorityProcessProbe$") + probe.Env = append(os.Environ(), "SLOTGATE_AUTHORITY_PROBE_ROOT="+root) + if output, err := probe.CombinedOutput(); err != nil { + t.Fatalf("another process acquired authority: %v\n%s", err, output) + } if err = a.Close(); err != nil { t.Fatal(err) } authorityForTest(t, root, terminatedExecutor) } +func TestAuthorityProcessProbe(t *testing.T) { + root := os.Getenv("SLOTGATE_AUTHORITY_PROBE_ROOT") + if root == "" { + t.Skip("subprocess-only lock probe") + } + if os.Getenv("SLOTGATE_AUTHORITY_PROBE_MODE") == "crash-mutation" { + a, err := OpenAuthority(root, terminatedExecutor) + if err != nil { + t.Fatal(err) + } + s, err := a.State("vm-1") + if err != nil { + t.Fatal(err) + } + command, err := a.BeginCommand("vm-1", s.Fence, "crashed-mutation", CommandMutation) + if err != nil { + t.Fatal(err) + } + if err = a.DispatchCommand(command, func(ExecutorIdentity) error { return nil }); err != nil { + t.Fatal(err) + } + os.Exit(0) // Intentionally bypass Close, modeling loss of the node daemon. + } + _, err := OpenAuthority(root, terminatedExecutor) + requireAuthorityError(t, err, ErrAuthorityLocked) +} + +func TestAuthorityProcessCrashCannotForgetDispatchedMutation(t *testing.T) { + root := t.TempDir() + a := authorityForTest(t, root, terminatedExecutor) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + if err := a.Close(); err != nil { + t.Fatal(err) + } + probe := exec.Command(os.Args[0], "-test.run=^TestAuthorityProcessProbe$") + probe.Env = append(os.Environ(), "SLOTGATE_AUTHORITY_PROBE_ROOT="+root, "SLOTGATE_AUTHORITY_PROBE_MODE=crash-mutation") + if output, err := probe.CombinedOutput(); err != nil { + t.Fatalf("crash helper: %v\n%s", err, output) + } + reopened := authorityForTest(t, root, terminatedExecutor) + got, err := reopened.State("vm-1") + if err != nil || got.Phase != PhaseRevoking || !got.Commands["crashed-mutation"].Dispatched || got.Commands["crashed-mutation"].Status != CommandUnknown { + t.Fatalf("crashed mutation was forgotten: %+v %v", got, err) + } + _, err = reopened.Advance(context.Background(), "vm-1", s.Fence) + requireAuthorityError(t, err, ErrPendingMutation) +} + +func TestAuthorityProcessIdentityRequiresBirthAndWholeScopeProof(t *testing.T) { + a := authorityForTest(t, t.TempDir(), func(_ context.Context, e ExecutorIdentity) (TerminationProof, error) { + return TerminationProof{ID: "process-proof", Executor: e, Scope: "process_scope_empty", ObservedAt: "2026-10-21T00:01:00Z"}, nil + }) + executor := ExecutorIdentity{Kind: "process", NodeBootID: "test-node-boot", PID: 42, ProcessStart: 12345, ProcessScope: "/vm2api/incarnation-a"} + s := launchForTest(t, a, "vm-1", executor) + wrong := s.Fence + wrong.Executor.ProcessStart++ + _, err := a.Revoke("vm-1", wrong) + requireAuthorityError(t, err, ErrFenceMismatch) + if _, err = a.Revoke("vm-1", s.Fence); err != nil { + t.Fatal(err) + } + if _, err = a.Advance(context.Background(), "vm-1", s.Fence); err != nil { + t.Fatal(err) + } + for _, invalid := range []ExecutorIdentity{ + {Kind: "process", NodeBootID: "test-node-boot", PID: 42}, + {Kind: "process", NodeBootID: "test-node-boot", PID: 42, ProcessStart: 12345}, + } { + if err = validateExecutor(invalid); err == nil { + t.Fatalf("PID alone was accepted: %+v", invalid) + } + } +} + +func TestAuthorityMissingInitializedJournalCannotResetGeneration(t *testing.T) { + root := t.TempDir() + a := authorityForTest(t, root, terminatedExecutor) + launchForTest(t, a, "vm-1", testExecutor("a")) + if err := a.Close(); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(root, "authority.json")); err != nil { + t.Fatal(err) + } + if reopened, err := OpenAuthority(root, terminatedExecutor); err == nil { + reopened.Close() + t.Fatal("lost initialized journal silently reset generations") + } +} + +func TestAuthorityDispatchChecksEpochAtSendAndNeverRetriesUnknownSend(t *testing.T) { + a := authorityForTest(t, t.TempDir(), terminatedExecutor) + s := launchForTest(t, a, "vm-1", testExecutor("a")) + queued, err := a.BeginCommand("vm-1", s.Fence, "queued-before-revoke", CommandForward) + if err != nil { + t.Fatal(err) + } + if _, err = a.Revoke("vm-1", s.Fence); err != nil { + t.Fatal(err) + } + sends := 0 + send := func(ExecutorIdentity) error { sends++; return nil } + requireAuthorityError(t, a.DispatchCommand(queued, send), ErrAuthorityPhase) + if _, err = a.Advance(context.Background(), "vm-1", s.Fence); err != nil { + t.Fatal(err) + } + requireAuthorityError(t, a.DispatchCommand(queued, send), ErrFenceMismatch) + if sends != 0 { + t.Fatal("captured old ticket sent after revocation") + } + + b := authorityForTest(t, t.TempDir(), terminatedExecutor) + active := launchForTest(t, b, "vm-2", testExecutor("b")) + command, err := b.BeginCommand("vm-2", active.Fence, "transport-unknown", CommandForward) + if err != nil { + t.Fatal(err) + } + transportError := errors.New("lost response after send") + err = b.DispatchCommand(command, func(identity ExecutorIdentity) error { + if identity != active.Fence.Executor { + t.Fatal("send used a different executor") + } + return transportError + }) + if err != transportError { + t.Fatalf("send error changed: %v", err) + } + requireAuthorityError(t, b.DispatchCommand(command, send), ErrCommandDispatched) + state, err := b.State("vm-2") + if err != nil || state.Commands[command.ID].Status != CommandPending { + t.Fatalf("unknown send was released: %+v %v", state, err) + } +} + func TestAuthorityInvalidJournalIsNotReinitialized(t *testing.T) { root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "authority.json"), []byte(`{"version":1,"slots":`), 0o600); err != nil { From 5e6212f360813a88d5985599500aaa6176edfc09 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:06:21 +0800 Subject: [PATCH 219/473] test: reproduce relay SQL scope loss and denied admission fallback --- test/unit/cluster-maintenance-scopes.test.mjs | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/test/unit/cluster-maintenance-scopes.test.mjs b/test/unit/cluster-maintenance-scopes.test.mjs index 750fa27ee..af0460b3c 100644 --- a/test/unit/cluster-maintenance-scopes.test.mjs +++ b/test/unit/cluster-maintenance-scopes.test.mjs @@ -1,5 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' import { getDb } from '../../src/lib/db/database.mjs' import { withDatabaseActivities } from '../maintenance-fixture.mjs' import { ClusterManager } from '../../src/lib/cluster/cluster-manager.mjs' @@ -27,7 +28,10 @@ async function fixture(t) { onLinkState: async () => sql('state'), onNodesChanged: async () => sql('nodes'), }) - manager.remoteHost = async () => sql('host') + manager.remoteHost = async () => { + await sql('host') + return { root: '/synthetic' } + } manager._checkHealth = async () => sql('health') manager._ensureGate = async () => sql('gate') t.mock.method(SshLink.prototype, 'start', () => {}) @@ -74,3 +78,40 @@ test('node change hooks await asynchronous persistence in a fresh activity', asy await fx.manager._nodesChanged() assert.deepEqual(fx.calls, ['nodes']) }) + +test('slot relay callbacks do not reuse the closed startup activity', async (t) => { + const fx = await fixture(t) + fx.manager._dialDirect = async () => { + await getDb().query('SELECT 1') + fx.calls.push('dial') + return null + } + await fx.manager.ensureSlotRelays('node-synthetic', 'vm-synthetic') + const relay = fx.manager.slotRelays.get('vm-synthetic').kernel + assert.equal(await relay.dialDirect(), null) + assert.equal(await relay.remotePath(), '/synthetic/vms/vm-synthetic/run/kernel.sock') + assert.deepEqual(fx.calls, ['dial', 'host']) +}) + +test('a refused direct admission cannot fall back to raw SSH', async () => { + let ssh = 0 + const relay = new SocketRelay({ + socketPath: '/tmp/synthetic-relay-unused', + remotePath: '/synthetic', + getClient: () => { + ssh++ + return {} + }, + dialDirect: async () => { + throw Object.assign(new Error('maintenance'), { code: 'database_maintenance' }) + }, + }) + const local = Object.assign(new EventEmitter(), { + destroy() { + this.destroyed = true + }, + }) + await relay._relay(local) + assert.equal(local.destroyed, true) + assert.equal(ssh, 0) +}) From 8430bd83ab6819c6b5e6309d4376bdf005401f72 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:07:19 +0800 Subject: [PATCH 220/473] test: reproduce proxy mutations bypassing current VM operation claims --- test/unit/proxy-vm-mutations.test.mjs | 144 ++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 test/unit/proxy-vm-mutations.test.mjs diff --git a/test/unit/proxy-vm-mutations.test.mjs b/test/unit/proxy-vm-mutations.test.mjs new file mode 100644 index 000000000..beaf2653a --- /dev/null +++ b/test/unit/proxy-vm-mutations.test.mjs @@ -0,0 +1,144 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { withVmMutations } from '../../src/lib/vm/vm-mutations.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' + +async function fixture(t, { bound = ['vm-a'] } = {}) { + const savedMode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'standalone' + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const first = await new ProxyPool({ db }).init() + const second = await new ProxyPool({ db: peer }).init() + const imported = await first.importLines('192.0.2.1:1080') + const id = imported.items[0].id + for (const vmId of bound) await first.bind(id, vmId) + await second.refresh() + for (const pool of [first, second]) pool.stopScheduler() + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(async () => { + first.stopScheduler() + second.stopScheduler() + if (savedMode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = savedMode + await peer.close() + }) + return { + db, + peer, + first, + second, + id, + operations: new SharedVmOperations(db, { ownerId: 'same-controller-owner' }), + otherOperations: new SharedVmOperations(peer, { ownerId: 'same-controller-owner' }), + } +} + +test('a bound proxy connection edit requires current VM operation claims', async (t) => { + const f = await fixture(t) + await assert.rejects(f.first.update(f.id, { port: 2080 }), { code: 'vm_mutation_claim_required' }) + assert.equal((await f.first.repo.getById(f.id)).port, 1080) +}) + +test('the current exact claim permits a bound proxy edit and nested VM mutation', async (t) => { + const f = await fixture(t) + await f.operations.run('vm-a', async () => { + assert.equal((await f.first.update(f.id, { port: 2080 })).ok, true) + await f.otherOperations.run('vm-a', async () => { + assert.equal((await f.second.update(f.id, { label: 'synthetic' })).ok, true) + }) + }) + assert.equal((await f.first.repo.getById(f.id)).port, 2080) +}) + +test('the same owner ID in another request cannot authorize a foreign active claim', async (t) => { + const f = await fixture(t) + let finish + let entered + const pending = new Promise((resolve) => { + finish = resolve + }) + const ready = new Promise((resolve) => { + entered = resolve + }) + const editing = f.operations.run('vm-a', async () => { + entered() + await pending + }) + await ready + try { + await f.otherOperations.run('vm-new', async () => { + await assert.rejects(f.second.bind(f.id, 'vm-new'), { code: 'vm_operation_active' }) + }) + } finally { + finish() + await editing + } + assert.deepEqual(f.first.snapshot().proxies[0].bound_vm_ids, ['vm-a']) +}) + +test('the PostgreSQL binding set is rechecked after a stale replica selected its lock set', async (t) => { + const f = await fixture(t) + await f.otherOperations.run('vm-b', () => f.second.bind(f.id, 'vm-b')) + await f.operations.run('vm-a', async () => { + await assert.rejects(f.first.update(f.id, { port: 2080 }), { code: 'vm_mutation_claim_required' }) + }) + assert.equal((await f.first.repo.getById(f.id)).port, 1080) +}) + +test('a stale empty-proxy removal cannot delete a newly bound VM outside its claim set', async (t) => { + const f = await fixture(t, { bound: [] }) + await f.otherOperations.run('vm-new', () => f.second.bind(f.id, 'vm-new')) + await assert.rejects(f.first.remove(f.id), { code: 'vm_mutation_claim_required' }) + assert.ok(await f.first.repo.getById(f.id)) +}) + +test('physical alias operations block a new binding even across node identities', async (t) => { + const f = await fixture(t) + const alias = await f.operations.tryAcquire({ vmId: 'a', nodeId: 'node-a' }) + await alias.markUncertain() + await f.otherOperations.run('vm-new', async () => { + await assert.rejects(f.second.bind(f.id, 'vm-new'), { code: 'vm_operation_active' }) + }) +}) + +test('an uncertain current operation cannot send another proxy mutation', async (t) => { + const f = await fixture(t) + await f.operations.run('vm-a', async (operation) => { + await operation.markUncertain() + await assert.rejects(f.first.update(f.id, { port: 2080 })) + }) + assert.equal((await f.first.repo.getById(f.id)).port, 1080) +}) + +test('detached work cannot reuse a callback claim after it has returned', async (t) => { + const f = await fixture(t) + let run + const pending = new Promise((resolve) => { + run = resolve + }) + let descendant + await f.operations.run('vm-a', async () => { + descendant = pending.then(() => f.first.update(f.id, { port: 2080 })) + }) + run() + await assert.rejects(descendant, { code: 'vm_mutation_claim_required' }) +}) + +test('global proxy settings require the complete current binding set', async (t) => { + const f = await fixture(t, { bound: ['vm-a', 'vm-b'] }) + await f.operations.run('vm-a', async () => { + await assert.rejects(f.first.updateConfig({ dns_primary: '1.1.1.1' }), { code: 'vm_mutation_claim_required' }) + }) + await withVmMutations( + '/synthetic', + ['vm-b', 'vm-a'], + async () => { + assert.equal((await f.first.updateConfig({ dns_primary: '1.1.1.1' })).ok, true) + }, + { operations: f.operations }, + ) +}) From baf6e200425a5000db7fa78537390e0a63ea9129 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:07:40 +0800 Subject: [PATCH 221/473] fix: admit relay lookups and reject unauthorized SSH fallback --- src/lib/cluster/cluster-manager.mjs | 10 +++++++--- src/lib/cluster/socket-relay.mjs | 10 +++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/lib/cluster/cluster-manager.mjs b/src/lib/cluster/cluster-manager.mjs index 5acd86425..51f51ed64 100644 --- a/src/lib/cluster/cluster-manager.mjs +++ b/src/lib/cluster/cluster-manager.mjs @@ -882,7 +882,11 @@ export class ClusterManager { if (!link) throw new ClusterError(404, 'node_not_found', '节点不存在') const paths = this.slotRelayPaths(nodeId, vmId) const getClient = () => (link.state === 'ready' ? link.client : null) - const remote = (name) => async () => `${(await this.remoteHost(nodeId)).root}/vms/${vmId}/run/${name}` + const remote = (name) => () => + this._background( + 'cluster_slot_path', + async () => `${(await this.remoteHost(nodeId)).root}/vms/${vmId}/run/${name}`, + ) const entry = { nodeId, paths, @@ -890,13 +894,13 @@ export class ClusterManager { socketPath: paths.kernel, remotePath: remote('kernel.sock'), getClient, - dialDirect: () => this._dialDirect(nodeId, vmId, 'kernel'), + dialDirect: () => this._background('cluster_slot_dial', () => this._dialDirect(nodeId, vmId, 'kernel')), }), worker: new SocketRelay({ socketPath: paths.worker, remotePath: remote('worker.sock'), getClient, - dialDirect: () => this._dialDirect(nodeId, vmId, 'worker'), + dialDirect: () => this._background('cluster_slot_dial', () => this._dialDirect(nodeId, vmId, 'worker')), }), } entry.ready = Promise.all([entry.kernel.start(), entry.worker.start()]) diff --git a/src/lib/cluster/socket-relay.mjs b/src/lib/cluster/socket-relay.mjs index 19264ba2e..4e4796b50 100644 --- a/src/lib/cluster/socket-relay.mjs +++ b/src/lib/cluster/socket-relay.mjs @@ -96,7 +96,15 @@ export class SocketRelay { local.once('close', () => this.active.delete(local)) local.on('error', () => {}) if (this.dialDirect) { - const direct = await this.dialDirect().catch(() => null) + let direct + try { + direct = await this.dialDirect() + } catch { + // Only an explicit null authorizes SSH fallback. Admission failures + // must not acquire the same remote privileges through another path. + local.destroy() + return + } if (direct) { this._pipeDirect(local, direct) return From 5fa35a22325d2a913fe6562ec8ed508e1fe73b79 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:10:12 +0800 Subject: [PATCH 222/473] test(slotgate): require authenticated fenced forwarding and Docker termination proof --- worker/internal/slotgate/ksg2_test.go | 379 ++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 worker/internal/slotgate/ksg2_test.go diff --git a/worker/internal/slotgate/ksg2_test.go b/worker/internal/slotgate/ksg2_test.go new file mode 100644 index 000000000..b506a2373 --- /dev/null +++ b/worker/internal/slotgate/ksg2_test.go @@ -0,0 +1,379 @@ +package slotgate + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +type fakeEngine struct { + mu sync.Mutex + executor ExecutorIdentity + absent bool + status int + keepAfterRemove bool + calls []string +} + +func (f *fakeEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, r.Method+" "+r.URL.Path) + if f.status != 0 { + w.WriteHeader(f.status) + return + } + if r.URL.Path == "/info" { + fmt.Fprint(w, `{"ID":"test-daemon"}`) + return + } + if r.Method == "GET" { + if f.absent { + w.WriteHeader(404) + fmt.Fprint(w, `{"message":"No such container"}`) + return + } + json.NewEncoder(w).Encode(map[string]any{"Id": f.executor.ContainerID, "State": map[string]any{"Running": true, "Restarting": false, "StartedAt": f.executor.StartedAt}}) + return + } + if r.Method == "DELETE" && !f.keepAfterRemove { + f.absent = true + } + w.WriteHeader(204) +} + +func unixHTTP(t *testing.T, socket string, handler http.Handler) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(socket), 0o700); err != nil { + t.Fatal(err) + } + ln, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + server := &http.Server{Handler: handler} + go server.Serve(ln) + t.Cleanup(func() { server.Close() }) +} + +func engineForTest(t *testing.T) (string, *fakeEngine, ExecutorIdentity) { + t.Helper() + boot, err := nodeBootID() + if err != nil { + t.Fatal(err) + } + executor := testExecutor("a") + executor.NodeBootID = boot + executor.DockerDaemonID = "test-daemon" + fake := &fakeEngine{executor: executor} + socket := filepath.Join(shortRoot(t), "docker.sock") + unixHTTP(t, socket, fake) + return socket, fake, executor +} + +func fencedGate(t *testing.T) (string, string, *Server, AuthorityState, *fakeEngine) { + t.Helper() + socket, engine, executor := engineForTest(t) + root := shortRoot(t) + s, err := New(Config{Listen: "127.0.0.1:0", Root: root, Token: testToken, TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: filepath.Join(root, "authority"), DockerSocket: socket}) + if err != nil { + t.Fatal(err) + } + state := launchForTest(t, s.authority, "v", executor) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.ServeListener(ctx, tls.NewListener(ln, s.TLSConfig())) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + return ln.Addr().String(), root, s, state, engine +} + +type controlReply struct { + OK bool `json:"ok"` + Error string `json:"error"` + Ticket string `json:"ticket"` + State *struct { + Fence Fence `json:"fence"` + Phase AuthorityPhase `json:"phase"` + } `json:"state"` +} + +func gate2Connect(t *testing.T, addr, token string, request gate2Request, httpBytes string) (net.Conn, *bufio.Reader) { + t.Helper() + c, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS13}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Close() }) + c.SetDeadline(time.Now().Add(5 * time.Second)) + data, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + if _, err = fmt.Fprintf(c, "KSG2 %s\n%s\n%s", token, data, httpBytes); err != nil { + t.Fatal(err) + } + return c, bufio.NewReader(c) +} + +func gate2Control(t *testing.T, addr string, request gate2Request) controlReply { + t.Helper() + c, reader := gate2Connect(t, addr, testToken, request, "") + defer c.Close() + line, err := reader.ReadBytes('\n') + if err != nil { + t.Fatal(err) + } + var reply controlReply + if err = json.Unmarshal(line, &reply); err != nil { + t.Fatalf("control reply %q: %v", line, err) + } + return reply +} + +func forwardRequest(t *testing.T, addr string, state AuthorityState, id string) gate2Request { + t.Helper() + req := gate2Request{Op: "prepare", VMID: "v", Fence: state.Fence, CommandID: id, Kind: "kernel"} + reply := gate2Control(t, addr, req) + if !reply.OK || reply.Ticket == "" { + t.Fatalf("prepare failed: %+v", reply) + } + req.Op = "forward" + req.Ticket = reply.Ticket + return req +} + +func TestKSG2FencedModeRejectsKSG1AndUnauthenticatedState(t *testing.T) { + addr, _, _, state, _ := fencedGate(t) + c, reader := gate2Connect(t, addr, "wrong-token", gate2Request{Op: "state", VMID: "v"}, "") + line, _ := reader.ReadString('\n') + c.Close() + if line != "ERR auth\n" { + t.Fatalf("unauthenticated state response: %q", line) + } + c, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS13}) + if err != nil { + t.Fatal(err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(3 * time.Second)) + fmt.Fprintf(c, "KSG1 %s v kernel\n", testToken) + line, _ = bufio.NewReader(c).ReadString('\n') + if line != "ERR protocol\n" { + t.Fatalf("fenced mode accepted KSG1: %q", line) + } + reply := gate2Control(t, addr, gate2Request{Op: "state", VMID: "v"}) + if !reply.OK || reply.State == nil || reply.State.Fence != state.Fence { + t.Fatalf("authenticated state: %+v", reply) + } + reply = gate2Control(t, addr, gate2Request{Op: "complete", VMID: "v", Fence: state.Fence}) + if reply.OK { + t.Fatal("client completed a node command without a verifier") + } +} + +func TestKSG2ParsesExactlyOneHTTPRequestAndRetainsForwardPending(t *testing.T) { + addr, root, s, state, _ := fencedGate(t) + var mu sync.Mutex + requests := 0 + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests++ + mu.Unlock() + body, _ := io.ReadAll(r.Body) + if r.URL.Path != "/messages" || string(body) != "first" { + t.Errorf("forwarded request: %s %q", r.URL.Path, body) + } + fmt.Fprint(w, `{"ok":true}`) + })) + req := forwardRequest(t, addr, state, "one-http-request") + c, reader := gate2Connect(t, addr, testToken, req, "POST /messages HTTP/1.1\r\nHost: kernel\r\nContent-Length: 5\r\n\r\nfirstGET /must-not-run HTTP/1.1\r\nHost: kernel\r\n\r\n") + line, err := reader.ReadString('\n') + if err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, response.Body) + response.Body.Close() + c.Close() + mu.Lock() + count := requests + mu.Unlock() + if count != 1 { + t.Fatalf("HTTP requests forwarded = %d, want 1", count) + } + current, err := s.authority.State("v") + if err != nil || current.Commands[req.CommandID].Status != CommandPending || !current.Commands[req.CommandID].Dispatched { + t.Fatalf("EOF released forward: %+v %v", current, err) + } + c, reader = gate2Connect(t, addr, testToken, req, "GET /messages HTTP/1.1\r\nHost: kernel\r\n\r\n") + line, _ = reader.ReadString('\n') + c.Close() + if line != "ERR command_dispatched\n" { + t.Fatalf("ticket replay: %q", line) + } +} + +func TestKSG2MissingOrStaleFenceTicketCannotSendAndNoLegacySocketFallback(t *testing.T) { + addr, root, s, state, _ := fencedGate(t) + legacyRequests := 0 + unixHTTP(t, filepath.Join(root, "vms", "v", "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { legacyRequests++; fmt.Fprint(w, "legacy") })) + req := forwardRequest(t, addr, state, "legacy-path-forbidden") + missing := req + missing.Ticket = "" + c, reader := gate2Connect(t, addr, testToken, missing, "GET / HTTP/1.1\r\nHost: kernel\r\n\r\n") + line, _ := reader.ReadString('\n') + c.Close() + if line != "ERR ticket_invalid\n" { + t.Fatalf("missing ticket: %q", line) + } + c, reader = gate2Connect(t, addr, testToken, req, "GET / HTTP/1.1\r\nHost: kernel\r\n\r\n") + line, _ = reader.ReadString('\n') + c.Close() + if line != "ERR no_socket\n" { + t.Fatalf("incarnation socket unavailable: %q", line) + } + if legacyRequests != 0 { + t.Fatalf("fell back to legacy socket: %d", legacyRequests) + } + late := forwardRequest(t, addr, state, "late-command") + if _, err := s.authority.Revoke("v", state.Fence); err != nil { + t.Fatal(err) + } + c, reader = gate2Connect(t, addr, testToken, late, "GET / HTTP/1.1\r\nHost: kernel\r\n\r\n") + line, _ = reader.ReadString('\n') + c.Close() + if line != "ERR authority_phase\n" { + t.Fatalf("late command dispatched: %q", line) + } + reply := gate2Control(t, addr, gate2Request{Op: "advance", VMID: "v", Fence: state.Fence}) + if !reply.OK || reply.State == nil || reply.State.Fence.Generation != state.Fence.Generation+1 { + t.Fatalf("verified advance: %+v", reply) + } + c, reader = gate2Connect(t, addr, testToken, late, "GET / HTTP/1.1\r\nHost: kernel\r\n\r\n") + line, _ = reader.ReadString('\n') + c.Close() + if line != "ERR fence_mismatch\n" { + t.Fatalf("old generation dispatched after takeover: %q", line) + } +} + +func TestKSG2MalformedHTTPAndWrongTargetTicketAreNotDispatched(t *testing.T) { + addr, _, s, state, _ := fencedGate(t) + req := forwardRequest(t, addr, state, "invalid-http") + c, reader := gate2Connect(t, addr, testToken, req, "not-an-http-request\r\n\r\n") + line, _ := reader.ReadString('\n') + c.Close() + if line != "ERR http_request\n" { + t.Fatalf("malformed HTTP response: %q", line) + } + current, _ := s.authority.State("v") + if current.Commands[req.CommandID].Dispatched { + t.Fatal("malformed HTTP was dispatched") + } + req.Kind = "worker" + c, reader = gate2Connect(t, addr, testToken, req, "GET / HTTP/1.1\r\nHost: kernel\r\n\r\n") + line, _ = reader.ReadString('\n') + c.Close() + if line != "ERR ticket_invalid\n" { + t.Fatalf("ticket reused on a different target: %q", line) + } +} + +func TestDockerVerifierRequiresMatchingImmutableIdentityAndTrustedAbsence(t *testing.T) { + socket, engine, executor := engineForTest(t) + verifier, err := newDockerVerifier(socket, executor.NodeBootID) + if err != nil { + t.Fatal(err) + } + wrong := executor + wrong.StartedAt = "2026-10-22T00:00:00Z" + if _, err = verifier(context.Background(), wrong); err == nil { + t.Fatal("different container incarnation was terminated") + } + engine.mu.Lock() + calls := append([]string(nil), engine.calls...) + engine.calls = nil + engine.mu.Unlock() + for _, call := range calls { + if !strings.HasPrefix(call, "GET ") { + t.Fatalf("mutated mismatching target: %s", call) + } + } + proof, err := verifier(context.Background(), executor) + if err != nil || proof.Executor != executor || proof.Scope != "container_removed" { + t.Fatalf("termination proof: %+v %v", proof, err) + } + engine.mu.Lock() + calls = append([]string(nil), engine.calls...) + engine.mu.Unlock() + if !strings.Contains(strings.Join(calls, "\n"), "DELETE /containers/"+executor.ContainerID) { + t.Fatalf("immutable ID not removed: %v", calls) + } + if _, err = verifier(context.Background(), executor); err != nil { + t.Fatalf("trusted 404 for tracked old identity: %v", err) + } +} + +func TestDockerVerifierErrorsCannotAdvanceOrReleasePendingForward(t *testing.T) { + for _, status := range []int{401, 500} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + addr, _, s, state, engine := fencedGate(t) + req := forwardRequest(t, addr, state, "pending") + if _, err := s.authority.Revoke("v", state.Fence); err != nil { + t.Fatal(err) + } + engine.mu.Lock() + engine.status = status + engine.mu.Unlock() + reply := gate2Control(t, addr, gate2Request{Op: "advance", VMID: "v", Fence: state.Fence}) + if reply.OK { + t.Fatal("Engine error advanced generation") + } + current, _ := s.authority.State("v") + if current.Fence != state.Fence || current.Commands[req.CommandID].Status != CommandPending { + t.Fatalf("Engine error cleared pending: %+v", current) + } + }) + } + socket, engine, executor := engineForTest(t) + verifier, _ := newDockerVerifier(socket, executor.NodeBootID) + wrong := executor + wrong.DockerDaemonID = "different-daemon" + if _, err := verifier(context.Background(), wrong); err == nil { + t.Fatal("different daemon 404 could become termination proof") + } + wrong = executor + wrong.NodeBootID = "another-boot" + if _, err := verifier(context.Background(), wrong); err == nil { + t.Fatal("different host boot accepted") + } + engine.mu.Lock() + engine.keepAfterRemove = true + engine.mu.Unlock() + if _, err := verifier(context.Background(), executor); err == nil { + t.Fatal("remove response alone became termination proof") + } +} From 39fb7543e4b25eb34b2e943847ed0b19204f704a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:12:25 +0800 Subject: [PATCH 223/473] test: reproduce shared watchdog mutation and drain races --- test/unit/shared-kernel-watchdog.test.mjs | 113 ++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 test/unit/shared-kernel-watchdog.test.mjs diff --git a/test/unit/shared-kernel-watchdog.test.mjs b/test/unit/shared-kernel-watchdog.test.mjs new file mode 100644 index 000000000..17c53c448 --- /dev/null +++ b/test/unit/shared-kernel-watchdog.test.mjs @@ -0,0 +1,113 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { createKernelWatchdog } from '../../src/lib/transport/kernel-watchdog.mjs' + +function fixture(t, overrides = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'shared-watchdog-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + fs.writeFileSync(path.join(root, 'kernel.json'), JSON.stringify({ slots_per_worker: 1 })) + const vm = { id: 'vm-shared-watchdog', status: 'running', runtime: { worker_run_dir: root } } + const db = getDb() + return { + vm, + create: (extra = {}) => + createKernelWatchdog({ + listTargets: () => [vm], + probeVm: () => vm, + health: async () => ({ ok: true, healthy: true, ready_slots: 20 }), + observeRuntime: async () => ({ state: 'running' }), + sharedTasks: new SharedBackgroundTasks(db), + operations: new SharedVmOperations(db), + ensure: async () => ({ ok: true }), + ...overrides, + ...extra, + }), + } +} + +function reserve() { + return new ExecutionAdmissionsRepo(getDb()).tryReserve({ + accountId: 'watchdog-account', + vmId: 'vm-shared-watchdog', + accountLimit: 2, + vmLimit: 2, + fableLimit: 0, + rpmLimit: 0, + }) +} + +test('watchdog recovery cannot mutate a globally occupied VM', async (t) => { + let restarts = 0 + const fx = fixture(t, { + ensure: async () => { + restarts++ + return { ok: true } + }, + }) + const active = await reserve() + assert.equal(active.ok, true) + await fx.create().tick() + assert.equal(restarts, 0) + await active.release() +}) + +test('two controller watchdogs execute one shared VM check plan', async (t) => { + let restarts = 0 + const fx = fixture(t, { + ensure: async () => { + restarts++ + return { ok: true } + }, + }) + await Promise.all([fx.create().tick(), fx.create().tick()]) + assert.equal(restarts, 1) +}) + +test('an unconfirmed watchdog recovery retains VM and background occupancy', async (t) => { + const fx = fixture(t, { ensure: async () => ({ ok: false, error: 'termination unknown' }) }) + await fx.create().tick() + assert.equal((await reserve()).code, 'vm_operation_active') + const row = await getDb().prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.ok(row.unknown_at) + assert.equal(row.finished_at, null) +}) + +test('watchdog flush waits for off-path telemetry work to finish', async (t) => { + let enter + let finish + const started = new Promise((resolve) => { + enter = resolve + }) + const done = new Promise((resolve) => { + finish = resolve + }) + const fx = fixture(t, { + ensureTelemetry: async () => { + enter() + await done + return { ok: true } + }, + }) + const watchdog = fx.create() + const ticking = watchdog.tick() + await started + await Promise.race([ticking, new Promise((resolve) => setTimeout(resolve, 20))]) + let flushed = false + const flushing = watchdog.flush().then(() => { + flushed = true + }) + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(flushed, false) + } finally { + finish() + await Promise.all([ticking, flushing]) + } +}) From 54b2d2667951b5cc637d0b4ec254372880428afb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:15:37 +0800 Subject: [PATCH 224/473] feat: coordinate watchdog work and await outstanding recovery effects --- src/lib/transport/kernel-watchdog.mjs | 89 ++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/src/lib/transport/kernel-watchdog.mjs b/src/lib/transport/kernel-watchdog.mjs index 6fe2888bc..0ac27fe2e 100644 --- a/src/lib/transport/kernel-watchdog.mjs +++ b/src/lib/transport/kernel-watchdog.mjs @@ -26,6 +26,10 @@ import { resolveInferenceEngine } from '../vm/slot-engine.mjs' import { slotHost } from '../vm/slot-host.mjs' import { dockerRequest } from '../cluster/docker-remote.mjs' import { clusterManager, vmNodeId } from '../cluster/placement.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { SharedVmOperations } from '../vm/shared-vm-operations.mjs' +import { physicalContainerName } from '../vm/vm-physical-target.mjs' +import { flushVmDbSync } from '../vm/vm-db-sync.mjs' /** A failed observation is not evidence of a dead container. Use the slot's own host, including KVM runners. */ export async function observeWatchdogContainer( @@ -162,6 +166,8 @@ export function createKernelWatchdog({ healBridges = null, probeVm = watchdogProbeVm, now = () => Date.now(), + sharedTasks, + operations = null, } = {}) { let config = normalizeKernelWatchdogConfig(initial) let timer = null @@ -176,19 +182,37 @@ export function createKernelWatchdog({ // vmId -> this tick's pending check/restart, so hold() can wait for it to settle. const inflight = new Map() let healingBridges = false + const tasks = backgroundTasksFor(sharedTasks) + const mutations = operations || (tasks ? new SharedVmOperations(tasks.db) : null) + const sideEffects = new Set() + + function trackSideEffect(operation) { + sideEffects.add(operation) + operation.then( + () => sideEffects.delete(operation), + () => sideEffects.delete(operation), + ) + return operation + } function healBridgesOffPath(vms) { if (typeof healBridges !== 'function' || healingBridges) return healingBridges = true - Promise.resolve() - .then(() => healBridges(vms)) + const work = () => healBridges(vms) + const pending = tasks + ? tasks.run({ taskKey: 'kernel_bridge_heal', intervalMs: config.interval_sec * 1000 }, async () => ({ + terminal: true, + value: await work(), + })) + : Promise.resolve().then(work) + trackSideEffect(pending) .catch((error) => console.warn(`[kernel-watchdog] bridge heal failed: ${error?.message || error}`)) .finally(() => { healingBridges = false }) } - async function healTelemetry(vm, at) { + async function healTelemetry(vm, at, markUnknown = () => {}) { if (typeof ensureTelemetry !== 'function' || at < (telemetryDue.get(vm.id) || 0)) return telemetryDue.set(vm.id, at + telemetryEveryMs) try { @@ -197,13 +221,15 @@ export function createKernelWatchdog({ console.warn(`[kernel-watchdog] ${vm.id} telemetry sidecar was missing; relaunched`) if (result?.ok === false) console.warn(`[kernel-watchdog] ${vm.id} telemetry sidecar relaunch failed: ${result.error}`) + if (tasks && result?.ok !== true) markUnknown() } catch (error) { + if (tasks) markUnknown() console.warn(`[kernel-watchdog] ${vm.id} telemetry sidecar check failed: ${error?.message || error}`) } } /** L4 gate: bounded container restarts with growing waits; spending them is L5. */ - function admitRestart(vm, reason, at) { + async function admitRestart(vm, reason, at) { if (kernelFaults.has(vm.id)) return false const state = restarts.get(vm.id) || { attempts: [], nextAt: at + config.restart_backoff_sec[0] * 1000 } state.attempts = state.attempts.filter((t) => at - t < config.restart_window_sec * 1000) @@ -216,7 +242,7 @@ export function createKernelWatchdog({ console.error(`[kernel-watchdog] ${vm.id} faulted; scheduling skips it: ${fault}`) if (typeof onFault === 'function') { try { - onFault(vm, fault) + await onFault(vm, fault) } catch (error) { console.warn(`[kernel-watchdog] fault notify failed: ${error?.message || error}`) } @@ -230,20 +256,21 @@ export function createKernelWatchdog({ } /** One VM's check. `held` is re-read after every await: an operator cycle may start mid-check. */ - async function reviveDownContainer(vm, state) { + async function reviveDownContainer(vm, state, markUnknown = () => {}) { if (vm.status !== 'running' || typeof revive !== 'function') return - if (!admitRestart(vm, `container ${state}`, now())) return + if (!(await admitRestart(vm, `container ${state}`, now()))) return const revived = await revive(vm) + if (tasks && revived?.ok !== true) markUnknown() if (!revived?.ok) console.warn(`[kernel-watchdog] ${vm.id} boot failed: ${revived?.error || revived?.code}`) } - async function checkVm(vm, routing) { + async function checkVm(vm, routing, markUnknown = () => {}) { const probed = probeVm(vm) if (!probed) { // The kernel cannot be probed without its relay; only a container that is really down is revived. const observed = await observeRuntime(vm) if (held.has(vm.id) || !['missing', 'stopped', 'restarting'].includes(observed?.state)) return - return reviveDownContainer(vm, observed.state) + return reviveDownContainer(vm, observed.state, markUnknown) } const exec = { vmId: vm.id, @@ -261,7 +288,10 @@ export function createKernelWatchdog({ // Config convergence, not a failure: outside the restart budget. if (kernelSlotMismatch(exec)) { const observed = await observeRuntime(vm) - if (!held.has(vm.id) && observed?.state === 'running') await ensure(exec, { timeoutMs: config.timeout_ms }) + if (!held.has(vm.id) && observed?.state === 'running') { + const result = await ensure(exec, { timeoutMs: config.timeout_ms }) + if (tasks && result?.ok !== true) markUnknown() + } } return } @@ -270,10 +300,11 @@ export function createKernelWatchdog({ if (held.has(vm.id)) return // Permission/transport failures and deliberately paused containers never authorize recovery. if (!['running', 'missing', 'stopped', 'restarting'].includes(observed?.state)) return - if (observed.state !== 'running') return reviveDownContainer(vm, observed.state) - if (!admitRestart(vm, reason, now())) return + if (observed.state !== 'running') return reviveDownContainer(vm, observed.state, markUnknown) + if (!(await admitRestart(vm, reason, now()))) return console.warn(`[kernel-watchdog] ${vm.id} restarting container: ${reason}`) const result = await ensure(exec, { timeoutMs: config.timeout_ms }) + if (tasks && result?.ok !== true) markUnknown() if (held.has(vm.id)) return // The container itself is gone or exited: a kernel restart cannot help, a full boot can. if (result?.reason === 'container_unavailable' && typeof revive === 'function' && vm.status === 'running') { @@ -282,6 +313,7 @@ export function createKernelWatchdog({ if (held.has(vm.id) || !['missing', 'stopped', 'restarting'].includes(latest?.state)) return console.warn(`[kernel-watchdog] ${vm.id} container unavailable; booting slot`) const revived = await revive(vm).catch((error) => ({ ok: false, error: error?.message || String(error) })) + if (tasks && revived?.ok !== true) markUnknown() if (!revived?.ok) console.warn(`[kernel-watchdog] ${vm.id} boot failed: ${revived?.error || revived?.code}`) } } @@ -301,9 +333,35 @@ export function createKernelWatchdog({ // missing sidecar costs telemetry, not inference. Off the tick's critical // path so a slow docker exec cannot delay container restarts. The due time // is set before the check runs, so a slot never has two checks in flight. - if (isTelemetryHealTarget(vm)) void healTelemetry(vm, now()) - if (!isKernelWatchdogTarget(vm, routing)) continue - const work = checkVm(vm, routing) + if (!tasks && isTelemetryHealTarget(vm)) void trackSideEffect(healTelemetry(vm, now())) + if (!isKernelWatchdogTarget(vm, routing) && !(tasks && isTelemetryHealTarget(vm))) continue + const work = tasks + ? tasks.run( + { taskKey: `kernel_watchdog:${physicalContainerName(vm.id)}`, intervalMs: config.interval_sec * 1000 }, + async () => { + let unknown = false + try { + await mutations.run( + vm.id, + async () => { + const markUnknown = () => { + unknown = true + } + if (isTelemetryHealTarget(vm)) await healTelemetry(vm, now(), markUnknown) + if (isKernelWatchdogTarget(vm, routing)) await checkVm(vm, routing, markUnknown) + await flushVmDbSync() + return { vmOperationUncertain: unknown } + }, + { kind: 'kernel_watchdog' }, + ) + } catch (error) { + if (['vm_execution_active', 'vm_operation_active'].includes(error.code)) error.taskNotStarted = true + throw error + } + return { terminal: !unknown } + }, + ) + : checkVm(vm, routing) inflight.set(vm.id, work) try { await work @@ -334,6 +392,7 @@ export function createKernelWatchdog({ }, async flush() { while (inTick) await new Promise((resolve) => setTimeout(resolve, 10)) + while (sideEffects.size) await Promise.allSettled([...sideEffects]) }, stop() { running = false From 8cdd02aec4b0cb7a855ee3af05208da7cb1b768d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:17:43 +0800 Subject: [PATCH 225/473] fix: validate current VM claims inside proxy mutation transactions --- src/lib/vm/proxy-pool.mjs | 59 ++++++++++++++++++++++++++- src/lib/vm/shared-vm-operations.mjs | 15 +++++++ test/unit/proxy-vm-mutations.test.mjs | 4 +- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/lib/vm/proxy-pool.mjs b/src/lib/vm/proxy-pool.mjs index ed578c053..f9ea75eec 100644 --- a/src/lib/vm/proxy-pool.mjs +++ b/src/lib/vm/proxy-pool.mjs @@ -14,6 +14,10 @@ import net from 'node:net' import crypto from 'node:crypto' import { resolveStoreDb, withTransaction, getActivityCoordinator } from '../db/database.mjs' import { ProxiesRepo, PROXY_COLUMNS, PROXY_HEALTH_COLUMNS } from '../db/repos/proxies-repo.mjs' +import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { currentVmOperationIds } from './shared-vm-operations.mjs' +import { physicalVmAliases } from './vm-physical-target.mjs' import { canBindProxyToVm, normalizeOwnerId, proxyOwnerId, vmOwnerId } from '../admin/resource-owner.mjs' import { validTimezone } from '../core/timezone.mjs' import { lookupProxyGeo, lookupProxyGeoV6 } from './proxy-geo.mjs' @@ -425,7 +429,7 @@ export class ProxyPool extends EventEmitter { } /** Each control mutation starts from PG and writes only its changed rows/fields. */ - _mutate(change) { + _mutate(change, { observation = false } = {}) { const operation = withTransaction(this.db, async () => { await this.repo.lockMutations() const snapshot = await this.repo.snapshot() @@ -443,6 +447,7 @@ export class ProxyPool extends EventEmitter { this._mutation = null } if (result?.ok === false) return result + if (!observation && isKubernetesMode()) await this._assertVmMutationClaims(before, working) const previous = new Map(before.proxies.map((proxy) => [proxy.id, proxy])) for (const proxy of working.proxies) { const old = previous.get(proxy.id) @@ -472,6 +477,56 @@ export class ProxyPool extends EventEmitter { return this._track(operation) } + async _assertVmMutationClaims(before, after) { + const configChanged = + Object.keys(this._changes(before.config, after.config, Object.keys(DEFAULT_CONFIG))).length > 0 + const previous = new Map(before.proxies.map((proxy) => [proxy.id, proxy])) + const next = new Map(after.proxies.map((proxy) => [proxy.id, proxy])) + const involved = new Set() + const required = new Set() + for (const id of new Set([...previous.keys(), ...next.keys()])) { + const old = previous.get(id) + const updated = next.get(id) + const patch = old && updated ? this._changes(this._stored(old), this._stored(updated)) : null + if (!configChanged && old && updated && !Object.keys(patch).length) continue + const oldIds = boundVmIdsOf(old) + const newIds = boundVmIdsOf(updated) + for (const vmId of [...oldIds, ...newIds]) involved.add(vmId) + // A binding-only change owns its added/removed targets. Configuration and + // deletion affect every existing member of the shared proxy. + const bindingOnly = + !configChanged && + old && + updated && + Object.keys(patch).every((key) => ['bound_vm_id', 'updated_at'].includes(key)) + for (const vmId of [...oldIds, ...newIds]) { + if (!bindingOnly || oldIds.includes(vmId) !== newIds.includes(vmId)) required.add(vmId) + } + } + if (!involved.size) return + // Serialize these checks with acquisition of each logical/physical VM claim. + await new ExecutionAdmissionsRepo(this.db).lockResources({ vmIds: [...involved] }) + const aliases = [...new Set([...involved].flatMap(physicalVmAliases))] + const rows = await this.db + .prepare(`SELECT id, vm_id, uncertain_at FROM vm_operations + WHERE vm_id = ANY($1::text[]) AND released_at IS NULL`) + .all(aliases) + const current = currentVmOperationIds(this.db) + if (rows.some((row) => !current.has(row.id) || row.uncertain_at)) { + throw Object.assign(new Error('A proxy VM operation is active'), { code: 'vm_operation_active', status: 409 }) + } + if ( + [...required].some( + (vmId) => !rows.some((row) => current.has(row.id) && physicalVmAliases(vmId).includes(row.vm_id)), + ) + ) { + throw Object.assign(new Error('Current VM operation claims do not cover the proxy binding set'), { + code: 'vm_mutation_claim_required', + status: 409, + }) + } + } + /** Compatibility for explicit state edits; production operations use the methods below. */ save() { if (this._mutation) return @@ -553,7 +608,7 @@ export class ProxyPool extends EventEmitter { return this._mutate(() => this._updateConfig(...args)) } async reportRuntimeFailure(...args) { - return this._mutate(() => this._reportRuntimeFailure(...args)) + return this._mutate(() => this._reportRuntimeFailure(...args), { observation: true }) } /** diff --git a/src/lib/vm/shared-vm-operations.mjs b/src/lib/vm/shared-vm-operations.mjs index 21164ea3e..2805decf8 100644 --- a/src/lib/vm/shared-vm-operations.mjs +++ b/src/lib/vm/shared-vm-operations.mjs @@ -6,6 +6,21 @@ import { physicalVmAliases } from './vm-physical-target.mjs' const callbacks = new AsyncLocalStorage() +/** Exact active claims inherited by this callback, never merely a controller owner ID. */ +export function currentVmOperationIds(db) { + return new Set( + (callbacks.getStore() || []) + .filter( + (context) => + context.active && + !context.operation.uncertain && + context.db.schema === db.schema && + context.db.connectionString === db.connectionString, + ) + .map((context) => context.operation.operationId), + ) +} + function operationError(code, status = 409) { return Object.assign(new Error(code.replaceAll('_', ' ')), { code, status }) } diff --git a/test/unit/proxy-vm-mutations.test.mjs b/test/unit/proxy-vm-mutations.test.mjs index beaf2653a..2fb30ecc0 100644 --- a/test/unit/proxy-vm-mutations.test.mjs +++ b/test/unit/proxy-vm-mutations.test.mjs @@ -131,13 +131,13 @@ test('detached work cannot reuse a callback claim after it has returned', async test('global proxy settings require the complete current binding set', async (t) => { const f = await fixture(t, { bound: ['vm-a', 'vm-b'] }) await f.operations.run('vm-a', async () => { - await assert.rejects(f.first.updateConfig({ dns_primary: '1.1.1.1' }), { code: 'vm_mutation_claim_required' }) + await assert.rejects(f.first.updateConfig({ dns_primary: '1.1.1.1:53' }), { code: 'vm_mutation_claim_required' }) }) await withVmMutations( '/synthetic', ['vm-b', 'vm-a'], async () => { - assert.equal((await f.first.updateConfig({ dns_primary: '1.1.1.1' })).ok, true) + assert.equal((await f.first.updateConfig({ dns_primary: '1.1.1.1:53' })).ok, true) }, { operations: f.operations }, ) From f91f7f101407760c59140e9f13767487cebe2dbd Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:18:39 +0800 Subject: [PATCH 226/473] test: distinguish Docker exec creation refusal from unknown execution --- test/unit/shared-cli-node-guard.test.mjs | 30 ++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/test/unit/shared-cli-node-guard.test.mjs b/test/unit/shared-cli-node-guard.test.mjs index 55339c3a9..2d5588a52 100644 --- a/test/unit/shared-cli-node-guard.test.mjs +++ b/test/unit/shared-cli-node-guard.test.mjs @@ -87,7 +87,7 @@ test('CLI guard keeps the VM unavailable until its actual command has ended', as await next.release() }) -async function engine(t, states) { +async function engine(t, states, { createStatus = 201, startStatus = 200 } = {}) { const execId = 'a'.repeat(64) const calls = [] const server = http.createServer(async (req, res) => { @@ -97,10 +97,12 @@ async function engine(t, states) { calls.push(req.url) let body if (req.url.endsWith('/exec')) { - res.statusCode = 201 + res.statusCode = createStatus body = { Id: execId } - } else if (req.url.endsWith('/start')) body = {} - else body = states.length > 1 ? states.shift() : states[0] + } else if (req.url.endsWith('/start')) { + res.statusCode = startStatus + body = {} + } else body = states.length > 1 ? states.shift() : states[0] res.setHeader('content-type', 'application/json') res.end(JSON.stringify(body)) }) @@ -140,3 +142,23 @@ test('guard executor timeout leaves a running Docker exec unconfirmed', async (t assert.equal(result.ended, false) assert.equal(result.ok, false) }) + +for (const createStatus of [404, 409]) { + test(`a complete Docker create-exec ${createStatus} rejection is proven not started`, async (t) => { + const fx = await engine(t, [], { createStatus }) + await guard(() => runCliNodeGuardCommand(fx.connect, 'kin-synthetic', ['synthetic'])).tick() + assert.deepEqual(fx.calls, ['/containers/kin-synthetic/exec']) + const next = await reserve() + assert.equal(next.ok, true) + await next.release() + const row = await getDb().prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.equal(row.unknown_at, null) + assert.ok(row.finished_at) + }) +} + +test('a start-exec 404 remains unknown after an exec was created', async (t) => { + const fx = await engine(t, [], { startStatus: 404 }) + await guard(() => runCliNodeGuardCommand(fx.connect, 'kin-synthetic', ['synthetic'])).tick() + assert.equal((await reserve()).code, 'vm_operation_active') +}) From 5918156264512d3f828cd788d901382a0cb0ce10 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:18:57 +0800 Subject: [PATCH 227/473] fix: release CLI cleanup claims after explicit exec creation rejection --- src/lib/vm/cli-node-guard.mjs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/lib/vm/cli-node-guard.mjs b/src/lib/vm/cli-node-guard.mjs index 9a3cf89b5..2e8a64004 100644 --- a/src/lib/vm/cli-node-guard.mjs +++ b/src/lib/vm/cli-node-guard.mjs @@ -76,15 +76,25 @@ exit 0 /** A start response does not prove that the cleanup command has exited. */ export async function runCliNodeGuardCommand(connect, container, cmd, { user, timeoutMs = 15000, pollMs = 100 } = {}) { - const created = await dockerJson( - connect, - { - method: 'POST', - path: `/containers/${encodeURIComponent(container)}/exec`, - body: { Cmd: cmd, AttachStdout: false, AttachStderr: false, ...(user ? { User: user } : {}) }, - }, - [201], - ) + let created + try { + created = await dockerJson( + connect, + { + method: 'POST', + path: `/containers/${encodeURIComponent(container)}/exec`, + body: { Cmd: cmd, AttachStdout: false, AttachStderr: false, ...(user ? { User: user } : {}) }, + }, + [201], + ) + } catch (error) { + // The daemon explicitly refused creation; no exec ID was allocated or started. + if (error.code === 'docker_api_error' && [404, 409].includes(error.status)) { + error.vmOperationNotStarted = true + error.taskNotStarted = true + } + throw error + } const execId = created.Id await dockerJson( connect, From 41a1fd7acc92597485aa8302af15fbd45ef23d00 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:19:39 +0800 Subject: [PATCH 228/473] feat(slotgate): add opt-in fenced HTTP forwarding and Docker termination verification --- worker/internal/slotgate/authority.go | 17 +- worker/internal/slotgate/docker_verifier.go | 141 +++++++++++ worker/internal/slotgate/ksg2.go | 265 ++++++++++++++++++++ worker/internal/slotgate/ksg2_test.go | 70 ++++++ worker/internal/slotgate/slotgate.go | 61 ++++- 5 files changed, 537 insertions(+), 17 deletions(-) create mode 100644 worker/internal/slotgate/docker_verifier.go create mode 100644 worker/internal/slotgate/ksg2.go diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index 8f5ab182b..c49cda587 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -58,13 +58,14 @@ const ( // PID alone. ProcessScope must identify a node-owned scope containing every // descendant; a verifier must check the whole scope, not only its initial PID. type ExecutorIdentity struct { - Kind string `json:"kind"` - NodeBootID string `json:"node_boot_id"` - ContainerID string `json:"container_id,omitempty"` - StartedAt string `json:"started_at,omitempty"` - PID int `json:"pid,omitempty"` - ProcessStart uint64 `json:"process_start,omitempty"` - ProcessScope string `json:"process_scope,omitempty"` + Kind string `json:"kind"` + NodeBootID string `json:"node_boot_id"` + ContainerID string `json:"container_id,omitempty"` + DockerDaemonID string `json:"docker_daemon_id,omitempty"` + StartedAt string `json:"started_at,omitempty"` + PID int `json:"pid,omitempty"` + ProcessStart uint64 `json:"process_start,omitempty"` + ProcessScope string `json:"process_scope,omitempty"` } // Fence is captured when work is created. A delayed operation must never be @@ -639,7 +640,7 @@ func validateExecutor(e ExecutorIdentity) error { return ErrInvalidAuthority } case "process": - if e.PID <= 0 || e.ProcessStart == 0 || !validAuthorityID(e.ProcessScope) || e.ContainerID != "" || e.StartedAt != "" { + if e.PID <= 0 || e.ProcessStart == 0 || !validAuthorityID(e.ProcessScope) || e.ContainerID != "" || e.DockerDaemonID != "" || e.StartedAt != "" { return ErrInvalidAuthority } default: diff --git a/worker/internal/slotgate/docker_verifier.go b/worker/internal/slotgate/docker_verifier.go new file mode 100644 index 000000000..cacd8dce3 --- /dev/null +++ b/worker/internal/slotgate/docker_verifier.go @@ -0,0 +1,141 @@ +package slotgate + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +func nodeBootID() (string, error) { + data, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + return "", err + } + id := strings.TrimSpace(string(data)) + if !validAuthorityID(id) { + return "", ErrInvalidAuthority + } + return id, nil +} + +// newDockerVerifier speaks only to a configured, trusted local Engine socket. +// The old identity must already have been recorded by a trusted node launcher. +// A daemon ID mismatch, malformed response or denied request is not absence. +func newDockerVerifier(socket, boot string) (TerminationVerifier, error) { + if !filepath.IsAbs(socket) || !validAuthorityID(boot) { + return nil, ErrInvalidAuthority + } + transport := &http.Transport{DisableKeepAlives: true, DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socket) + }} + client := &http.Client{Transport: transport, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + call := func(ctx context.Context, method, path string, out any, allowed ...int) (int, error) { + req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil) + if err != nil { + return 0, err + } + res, err := client.Do(req) + if err != nil { + return 0, err + } + defer res.Body.Close() + ok := false + for _, status := range allowed { + if res.StatusCode == status { + ok = true + } + } + if !ok { + return res.StatusCode, fmt.Errorf("docker_status_%d", res.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(res.Body, (1<<20)+1)) + if err != nil { + return res.StatusCode, err + } + if len(data) > 1<<20 { + return res.StatusCode, ErrInvalidAuthority + } + if out != nil && res.StatusCode == 200 { + if err = json.Unmarshal(data, out); err != nil { + return res.StatusCode, err + } + } + return res.StatusCode, nil + } + type inspectState struct { + ID string `json:"Id"` + State *struct { + Running *bool `json:"Running"` + Restarting *bool `json:"Restarting"` + StartedAt string `json:"StartedAt"` + } `json:"State"` + } + return func(ctx context.Context, expected ExecutorIdentity) (TerminationProof, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := validateExecutor(expected); err != nil { + return TerminationProof{}, err + } + if expected.Kind != "container" || expected.NodeBootID != boot || !validAuthorityID(expected.DockerDaemonID) { + return TerminationProof{}, ErrTerminationProof + } + var daemon struct { + ID string `json:"ID"` + } + if _, err := call(ctx, "GET", "/info", &daemon, 200); err != nil { + return TerminationProof{}, err + } + if daemon.ID != expected.DockerDaemonID { + return TerminationProof{}, ErrTerminationProof + } + path := "/containers/" + expected.ContainerID + var info inspectState + status, err := call(ctx, "GET", path+"/json", &info, 200, 404) + if err != nil { + return TerminationProof{}, err + } + if status == 200 { + if info.ID != expected.ContainerID || info.State == nil || info.State.Running == nil || info.State.Restarting == nil || info.State.StartedAt != expected.StartedAt { + return TerminationProof{}, ErrTerminationProof + } + if *info.State.Running || *info.State.Restarting { + if _, err = call(ctx, "POST", path+"/stop?t=10", nil, 204, 304); err != nil { + return TerminationProof{}, err + } + } + // Removal prevents the revoked incarnation being restarted by name or + // automatic restart policy. Persistent bind-mounted slot data is kept. + if _, err = call(ctx, "DELETE", path+"?force=1", nil, 204, 404); err != nil { + return TerminationProof{}, err + } + var remaining inspectState + status, err = call(ctx, "GET", path+"/json", &remaining, 200, 404) + if err != nil || status != 404 { + if err == nil { + err = ErrTerminationProof + } + return TerminationProof{}, err + } + } + // Recheck the trusted daemon after the absence observation as well. + daemon.ID = "" + if _, err = call(ctx, "GET", "/info", &daemon, 200); err != nil { + return TerminationProof{}, err + } + if daemon.ID != expected.DockerDaemonID { + return TerminationProof{}, ErrTerminationProof + } + id, err := randomAuthorityID() + if err != nil { + return TerminationProof{}, err + } + return TerminationProof{ID: id, Executor: expected, Scope: "container_removed", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)}, nil + }, nil +} diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go new file mode 100644 index 000000000..a272a9f27 --- /dev/null +++ b/worker/internal/slotgate/ksg2.go @@ -0,0 +1,265 @@ +package slotgate + +import ( + "bufio" + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "path/filepath" + "strings" + "time" +) + +const gate2LineMax = 8192 +const gate2RequestMax = 16 << 20 + +// KSG2 \n followed by one bounded JSON command line. Only forward +// commands continue with one parsed HTTP/1 request and an HTTP response after +// OK\n. KSG1 raw splicing is never accepted when fenced mode is enabled. +type gate2Request struct { + Op string `json:"op"` + VMID string `json:"vm_id"` + Fence Fence `json:"fence"` + CommandID string `json:"command_id,omitempty"` + Kind string `json:"kind,omitempty"` + Ticket string `json:"ticket,omitempty"` +} + +type gate2State struct { + Fence Fence `json:"fence"` + Phase AuthorityPhase `json:"phase"` + Pending int `json:"pending"` + LastTermination *TerminationProof `json:"last_termination,omitempty"` +} + +func publicGateState(state AuthorityState) gate2State { + pending := 0 + for _, command := range state.Commands { + if pendingCommand(command) { + pending++ + } + } + // Do not disclose the command nonce or daemon runtime used to mint tickets. + return gate2State{Fence: state.Fence, Phase: state.Phase, Pending: pending, LastTermination: state.LastTermination} +} + +func gate2Line(reader *bufio.Reader) ([]byte, error) { + var line []byte + for { + part, err := reader.ReadSlice('\n') + line = append(line, part...) + if len(line) > gate2LineMax { + return nil, ErrInvalidAuthority + } + if errors.Is(err, bufio.ErrBufferFull) { + continue + } + return line, err + } +} + +func gate2Code(err error) string { + for _, known := range []error{ErrAuthorityUnavailable, ErrAuthorityPhase, ErrFenceMismatch, ErrCommandExists, ErrCommandOwner, ErrCommandDispatched, ErrPendingMutation, ErrInvalidAuthority, ErrTerminationProof} { + if errors.Is(err, known) { + return known.Error() + } + } + return "execution_unknown" +} + +func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio.Reader, line string) { + fields := strings.Fields(line) + if len(fields) != 2 || fields[0] != "KSG2" { + io.WriteString(conn, "ERR protocol\n") + return + } + token := sha256.Sum256([]byte(fields[1])) + if subtle.ConstantTimeCompare(token[:], s.token[:]) != 1 { + io.WriteString(conn, "ERR auth\n") + return + } + lineBytes, err := gate2Line(reader) + if err != nil { + io.WriteString(conn, "ERR bad_request\n") + return + } + var request gate2Request + decoder := json.NewDecoder(bytes.NewReader(lineBytes)) + decoder.DisallowUnknownFields() + if err = decoder.Decode(&request); err != nil { + io.WriteString(conn, "ERR bad_request\n") + return + } + if err = decoder.Decode(new(any)); err != io.EOF { + io.WriteString(conn, "ERR bad_request\n") + return + } + if !vmIDRe.MatchString(request.VMID) { + io.WriteString(conn, "ERR bad_request\n") + return + } + if request.Op == "forward" { + s.forwardFenced(ctx, conn, reader, request) + return + } + // No network mutation runs under the handshake/header deadline. Engine + // operations have their own bounded timeout and retain state on any error. + conn.SetDeadline(time.Now().Add(35 * time.Second)) + reply := map[string]any{"ok": false} + var state AuthorityState + switch request.Op { + case "state": + state, err = s.authority.State(request.VMID) + case "prepare": + if _, ok := kinds[request.Kind]; !ok { + err = ErrInvalidAuthority + break + } + var handle CommandHandle + handle, err = s.authority.BeginCommand(request.VMID, request.Fence, request.CommandID, CommandForward) + if err == nil { + reply["ticket"] = gate2Ticket(handle.token, request) + } + case "revoke": + state, err = s.authority.Revoke(request.VMID, request.Fence) + case "advance": + state, err = s.authority.Advance(ctx, request.VMID, request.Fence) + default: + err = ErrInvalidAuthority + } + if err != nil { + reply["error"] = gate2Code(err) + } else { + reply["ok"] = true + if request.Op != "prepare" { + reply["state"] = publicGateState(state) + } + } + json.NewEncoder(conn).Encode(reply) +} + +func gate2Ticket(nonce string, request gate2Request) string { + mac := hmac.New(sha256.New, []byte(nonce)) + // The opcode/ticket are excluded so prepare and forward share one grant. + data, _ := json.Marshal(struct { + VM string + Fence Fence + ID string + Kind string + }{request.VMID, request.Fence, request.CommandID, request.Kind}) + mac.Write(data) + return hex.EncodeToString(mac.Sum(nil)) +} + +func (s *Server) forwardHandle(request gate2Request) (CommandHandle, error) { + if _, ok := kinds[request.Kind]; !ok { + return CommandHandle{}, errors.New("ticket_invalid") + } + state, err := s.authority.State(request.VMID) + if err != nil { + return CommandHandle{}, err + } + if state.Fence != request.Fence { + return CommandHandle{}, ErrFenceMismatch + } + command, exists := state.Commands[request.CommandID] + if !exists || command.Kind != CommandForward || command.Fence != request.Fence { + return CommandHandle{}, errors.New("ticket_invalid") + } + wanted := gate2Ticket(command.Token, request) + if !hmac.Equal([]byte(wanted), []byte(request.Ticket)) { + return CommandHandle{}, errors.New("ticket_invalid") + } + return CommandHandle{VMID: request.VMID, ID: request.CommandID, Fence: request.Fence, runtime: command.Runtime, token: command.Token}, nil +} + +func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio.Reader, request gate2Request) { + handle, err := s.forwardHandle(request) + if err != nil { + code := gate2Code(err) + if err.Error() == "ticket_invalid" { + code = "ticket_invalid" + } + fmt.Fprintf(conn, "ERR %s\n", code) + return + } + // Parse and reserialize exactly one request. Never raw-splice client bytes: + // keep-alive, pipelined requests and delayed extra bytes cannot share a grant. + limited := &io.LimitedReader{R: reader, N: gate2RequestMax + 1} + req, err := http.ReadRequest(bufio.NewReader(limited)) + if err != nil { + io.WriteString(conn, "ERR http_request\n") + return + } + defer req.Body.Close() + if req.ProtoMajor != 1 || req.Method == http.MethodConnect || req.Header.Get("Upgrade") != "" { + io.WriteString(conn, "ERR http_request\n") + return + } + body, err := io.ReadAll(req.Body) + if err != nil || limited.N <= 0 { + io.WriteString(conn, "ERR http_request\n") + return + } + req.Body = io.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) + req.TransferEncoding = nil + req.Trailer = nil + req.Close = true + for _, header := range []string{"Connection", "Keep-Alive", "Proxy-Connection", "TE", "Trailer", "Upgrade"} { + req.Header.Del(header) + } + conn.SetDeadline(time.Time{}) + var target net.Conn + err = s.authority.DispatchCommand(handle, func(ExecutorIdentity) error { + var dialErr error + // The incarnation has a private socket namespace. No fallback to run/. + socket := filepath.Join(s.cfg.Root, "vms", request.VMID, "incarnations", request.Fence.Incarnation, "run", kinds[request.Kind]) + target, dialErr = dialSlotSocket(socket) + if dialErr != nil { + return errors.New("no_socket") + } + target.SetWriteDeadline(time.Now().Add(prefaceTimeout)) + if err := req.Write(target); err != nil { + return err + } + return target.SetWriteDeadline(time.Time{}) + }) + if target != nil { + defer target.Close() + } + if err != nil { + code := gate2Code(err) + if err.Error() == "no_socket" { + code = "no_socket" + } + fmt.Fprintf(conn, "ERR %s\n", code) + return + } + stop := context.AfterFunc(ctx, func() { target.Close() }) + defer stop() + if _, err = io.WriteString(conn, "OK\n"); err != nil { + return + } + response, err := http.ReadResponse(bufio.NewReader(target), req) + if err != nil { + return + } + defer response.Body.Close() + if response.StatusCode == http.StatusSwitchingProtocols { + return + } + response.Close = true + response.Write(conn) + // Even a full HTTP response remains pending in this batch. Classifying a + // kernel's normal terminal response is a separate trusted protocol adapter. +} diff --git a/worker/internal/slotgate/ksg2_test.go b/worker/internal/slotgate/ksg2_test.go index b506a2373..77917eb1f 100644 --- a/worker/internal/slotgate/ksg2_test.go +++ b/worker/internal/slotgate/ksg2_test.go @@ -23,6 +23,10 @@ type fakeEngine struct { absent bool status int keepAfterRemove bool + infoEmptyAfter int + infoCalls int + truncated404 bool + huge404 bool calls []string } @@ -35,12 +39,24 @@ func (f *fakeEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } if r.URL.Path == "/info" { + f.infoCalls++ + if f.infoEmptyAfter > 0 && f.infoCalls > f.infoEmptyAfter { + fmt.Fprint(w, `{}`) + return + } fmt.Fprint(w, `{"ID":"test-daemon"}`) return } if r.Method == "GET" { if f.absent { + if f.truncated404 { + w.Header().Set("Content-Length", "1000") + } w.WriteHeader(404) + if f.huge404 { + fmt.Fprint(w, strings.Repeat("x", (1<<20)+1)) + return + } fmt.Fprint(w, `{"message":"No such container"}`) return } @@ -189,6 +205,10 @@ func TestKSG2FencedModeRejectsKSG1AndUnauthenticatedState(t *testing.T) { if reply.OK { t.Fatal("client completed a node command without a verifier") } + reply = gate2Control(t, addr, gate2Request{Op: "prepare", VMID: "v", CommandID: "no-captured-fence", Kind: "kernel"}) + if reply.OK { + t.Fatal("prepare omitted its captured fence") + } } func TestKSG2ParsesExactlyOneHTTPRequestAndRetainsForwardPending(t *testing.T) { @@ -377,3 +397,53 @@ func TestDockerVerifierErrorsCannotAdvanceOrReleasePendingForward(t *testing.T) t.Fatal("remove response alone became termination proof") } } + +func TestDockerVerifierFinalDaemonObservationCannotReuseEarlierFields(t *testing.T) { + socket, engine, executor := engineForTest(t) + engine.mu.Lock() + engine.infoEmptyAfter = 1 + engine.mu.Unlock() + verifier, _ := newDockerVerifier(socket, executor.NodeBootID) + if _, err := verifier(context.Background(), executor); err == nil { + t.Fatal("missing final daemon ID reused an earlier observation") + } +} + +func TestDockerVerifierIncompleteOrOversized404IsUnknown(t *testing.T) { + for _, which := range []string{"truncated", "oversized"} { + t.Run(which, func(t *testing.T) { + socket, engine, executor := engineForTest(t) + engine.mu.Lock() + engine.absent = true + engine.truncated404 = which == "truncated" + engine.huge404 = which == "oversized" + engine.mu.Unlock() + verifier, _ := newDockerVerifier(socket, executor.NodeBootID) + if _, err := verifier(context.Background(), executor); err == nil { + t.Fatal("incomplete/unbounded Engine response became absence proof") + } + }) + } +} + +func TestKSG2TwoControllersCanForwardWhileFirstResponseIsStillPending(t *testing.T) { + addr, root, _, state, _ := fencedGate(t) + entered := make(chan struct{}, 2) + release := make(chan struct{}) + defer close(release) + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { entered <- struct{}{}; <-release; fmt.Fprint(w, "done") })) + for _, id := range []string{"controller-a", "controller-b"} { + req := forwardRequest(t, addr, state, id) + _, reader := gate2Connect(t, addr, testToken, req, "GET / HTTP/1.1\r\nHost: kernel\r\n\r\n") + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("send %s: %q %v", id, line, err) + } + } + for range 2 { + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("second controller waited for the first response") + } + } +} diff --git a/worker/internal/slotgate/slotgate.go b/worker/internal/slotgate/slotgate.go index 0f11bae23..a3f307f50 100644 --- a/worker/internal/slotgate/slotgate.go +++ b/worker/internal/slotgate/slotgate.go @@ -30,6 +30,11 @@ type Config struct { Token string `json:"token"` // TLSDir holds key.pem / cert.pem, created on first start. TLSDir string `json:"tls_dir"` + // Experimental node fencing. Raw SSH/Docker/SFTP permissions and controller + // admission wiring must be migrated before this can be enabled for HA. + Fenced bool `json:"fenced,omitempty"` + AuthorityDir string `json:"authority_dir,omitempty"` + DockerSocket string `json:"docker_socket,omitempty"` } // Preface: "KSG1 \n"; reply "OK\n" or "ERR \n". @@ -47,11 +52,12 @@ var ( ) type Server struct { - cfg Config - token [sha256.Size]byte - tls *tls.Config - slots chan struct{} - wg sync.WaitGroup + cfg Config + token [sha256.Size]byte + tls *tls.Config + slots chan struct{} + wg sync.WaitGroup + authority *Authority } func New(cfg Config) (*Server, error) { @@ -71,12 +77,35 @@ func New(cfg Config) (*Server, error) { if err != nil { return nil, fmt.Errorf("tls: %w", err) } - return &Server{ + srv := &Server{ cfg: cfg, token: sha256.Sum256([]byte(cfg.Token)), tls: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13}, slots: make(chan struct{}, MaxConns), - }, nil + } + if cfg.Fenced { + inside, err := filepath.Rel(filepath.Join(cfg.Root, "vms"), cfg.AuthorityDir) + if err != nil || inside == "." || (inside != ".." && !strings.HasPrefix(inside, ".."+string(filepath.Separator))) { + return nil, errors.New("authority_dir must be outside slot mounts") + } + boot, err := nodeBootID() + if err != nil { + return nil, err + } + socket := cfg.DockerSocket + if socket == "" { + socket = "/var/run/docker.sock" + } + verify, err := newDockerVerifier(socket, boot) + if err != nil { + return nil, err + } + srv.authority, err = OpenAuthority(cfg.AuthorityDir, verify) + if err != nil { + return nil, err + } + } + return srv, nil } // TLSConfig is the server side config, for callers that bring their own listener. @@ -86,6 +115,9 @@ func (s *Server) Serve(ctx context.Context) error { lc := net.ListenConfig{KeepAlive: keepAlive} ln, err := lc.Listen(ctx, "tcp", s.cfg.Listen) if err != nil { + if s.authority != nil { + s.authority.Close() + } return err } return s.ServeListener(ctx, tls.NewListener(ln, s.tls)) @@ -94,6 +126,9 @@ func (s *Server) Serve(ctx context.Context) error { func (s *Server) ServeListener(ctx context.Context, ln net.Listener) error { stop := context.AfterFunc(ctx, func() { ln.Close() }) defer stop() + if s.authority != nil { + defer s.authority.Close() + } defer s.wg.Wait() for { conn, err := ln.Accept() @@ -117,14 +152,18 @@ func (s *Server) ServeListener(ctx context.Context, ln net.Listener) error { go func() { defer s.wg.Done() defer func() { <-s.slots }() - s.handle(conn) + if s.cfg.Fenced { + stopConn := context.AfterFunc(ctx, func() { conn.Close() }) + defer stopConn() + } + s.handle(ctx, conn) }() } } // handle runs the TLS handshake and preface under one deadline, so a client // that stalls in either is dropped after prefaceTimeout. -func (s *Server) handle(conn net.Conn) { +func (s *Server) handle(ctx context.Context, conn net.Conn) { defer conn.Close() conn.SetDeadline(time.Now().Add(prefaceTimeout)) br := bufio.NewReaderSize(conn, prefaceMax) @@ -132,6 +171,10 @@ func (s *Server) handle(conn net.Conn) { if err != nil { return } + if s.cfg.Fenced { + s.handleFenced(ctx, conn, br, string(line)) + return + } vm, kind, code := s.parse(string(line)) if code != "" { fmt.Fprintf(conn, "ERR %s\n", code) From 0ffae4b04c2429f472bef5a42add4a00a1ed7fcc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:20:05 +0800 Subject: [PATCH 229/473] feat: coordinate backups and restores with durable global maintenance --- src/lib/admin/backup-service.mjs | 115 +++++++++++++++++++++-- src/lib/db/global-maintenance.mjs | 78 ++++++++++----- src/server.mjs | 16 +++- test/e2e/maintenance-backup.e2e.test.mjs | 95 +++++++++++++++++++ test/unit/maintenance-handoff.test.mjs | 83 +++++++++++++++- 5 files changed, 344 insertions(+), 43 deletions(-) create mode 100644 test/e2e/maintenance-backup.e2e.test.mjs diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index 2640513ef..a17fa8bb8 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -3,12 +3,13 @@ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' import { Worker } from 'node:worker_threads' -import { getDb, applyMigrations, quoteIdentifier } from '../db/database.mjs' +import { getDb, applyMigrations, quoteIdentifier, getActivityCoordinator } from '../db/database.mjs' import { sha256FileAsync, runPgTool, runProcess, extractArchiveMember } from './backup-core.mjs' import { SettingsRepo } from '../db/repos/settings-repo.mjs' import { BackupRepo } from '../db/repos/backup-repo.mjs' import { invalidateReadCache } from '../vm/vm-file.mjs' import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' // tarAvailability / sha256File / the runnable pipeline live in backup-core.mjs; // keep the historical export surface for tests and callers. @@ -74,12 +75,33 @@ export class BackupService { async _enterMaintenance(reason) { if (this.isMaintaining) return false this.isMaintaining = true - await this._maintenance?.(true, reason) + const coordinator = getActivityCoordinator(getDb()) + if (coordinator) { + try { + this._globalOperation = await coordinator.beginMaintenanceFromCurrentActivity({ + kind: reason, + onDraining: async (operation) => { + this._globalOperation = operation + await this._maintenance?.(true, reason, { operation }) + }, + }) + } catch (error) { + const state = await coordinator.getState() + if (state.phase === 'draining' && state.ownerId === coordinator.ownerId) { + await coordinator.cancelDrain(state.operationId) + this._globalOperation = null + await this._maintenance?.(false, reason) + this.isMaintaining = false + } + throw error + } + } else await this._maintenance?.(true, reason) return true } async _leaveMaintenance(reason) { - await this._maintenance?.(false, reason) + if (this._globalOperation && !this._globalOperation._resumed) await this._globalOperation.resume() + await this._maintenance?.(false, reason, { operation: this._globalOperation }) this.isMaintaining = false } @@ -148,15 +170,20 @@ export class BackupService { } /** Export a PostgreSQL snapshot while files are frozen, then dump in a worker. */ - async _createBackup({ kind = 'manual', note = null } = {}) { + async _createBackup({ kind = 'manual', note = null, maintenanceOperation = null } = {}) { if (this._running) throw Object.assign(new Error('backup already in progress'), { code: 'backup_in_progress' }) this._running = true const id = 'bak_' + crypto.randomBytes(6).toString('hex') const createdAt = new Date().toISOString() let client let entered = false + const previousOperation = this._globalOperation + let ownedOperation = null try { - entered = await this._enterMaintenance(kind === 'pre_restore' ? 'restore' : 'backup') + entered = maintenanceOperation + ? kind !== 'pre_restore' + : await this._enterMaintenance(kind === 'pre_restore' ? 'restore' : 'backup') + if (!previousOperation && !maintenanceOperation) ownedOperation = this._globalOperation const db = getDb() client = await db.pool.connect() await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') @@ -192,6 +219,7 @@ export class BackupService { } return stored } catch (error) { + if (this._globalOperation) this._globalOperation._completionUnknown = true try { if (!(await this._repo().get(id))) await this._repo().insert({ @@ -213,7 +241,14 @@ export class BackupService { } } this._running = false - if (entered) await this._leaveMaintenance('backup') + try { + if (entered) await this._leaveMaintenance('backup') + } finally { + if (ownedOperation) { + await ownedOperation.close() + this._globalOperation = null + } + } } } @@ -387,7 +422,7 @@ export class BackupService { fs.chmodSync(path.join(target, name), 0o600) } } - if (this.configDir && fs.existsSync(path.join(tmp, 'config'))) { + if (!isKubernetesMode() && this.configDir && fs.existsSync(path.join(tmp, 'config'))) { fs.mkdirSync(this.configDir, { recursive: true }) for (const name of fs.readdirSync(this.configDir)) { if (name.endsWith('.json') && !fs.lstatSync(path.join(this.configDir, name)).isDirectory()) @@ -402,6 +437,7 @@ export class BackupService { } async _applyArchive(archive) { + this._globalOperation?._assertHealthy() const db = getDb() // pg_restore --clean only removes objects present in the archive. Later // migrations can add tables (including login sessions) that must disappear @@ -457,6 +493,7 @@ export class BackupService { ['--no-psqlrc', '--quiet', '--single-transaction', '--set=ON_ERROR_STOP=1', '--file', cleanup, '--file', restore], db.connectionString, ) + this._globalOperation?._assertHealthy() this._restoreFiles(archive.tmp) await applyMigrations(getDb()) } @@ -501,23 +538,40 @@ export class BackupService { await this._enterMaintenance('restore') pre = await this.createBackup({ kind: 'pre_restore', note: `before restore of ${id}` }) previous = await this._extractArchive(pre) + if (this._globalOperation) await this._globalOperation.beginRestore() changed = true await getDb().withMaintenance(async () => { await this._applyArchive(archive) await this._registerOrphanArchives([pre, rec]) + if (this._globalOperation) await this._globalOperation.beginRebind() await this._rebindStores?.(getDb()) + if (this._globalOperation) { + const coordinator = getActivityCoordinator(getDb()) + await coordinator.acknowledgeEpoch((await coordinator.getState()).epoch) + } }) await this._leaveMaintenance('restore') fs.rmSync(this.maintenanceFile) this.isRestoring = false return { ok: true, restored: id, pre_restore: pre.id } } catch (error) { + if (this._globalOperation?.healthy && !this._globalOperation._resumed) { + await this._globalOperation.failBlocked(error.code || 'restore_failed') + } if (maintenanceStarted) { - this.isRestoring = true - this.isMaintaining = true + const coordinator = getActivityCoordinator(getDb()) + if (coordinator && !changed && (await coordinator.getState()).phase === 'open') { + fs.rmSync(this.maintenanceFile, { force: true }) + this.isRestoring = false + this.isMaintaining = false + } else { + this.isRestoring = true + this.isMaintaining = true + } } - if (changed && previous) { + if (changed && previous && !this._globalOperation?._resumed) { try { + this._globalOperation?._assertHealthy() await getDb().withMaintenance(async () => { await this._applyArchive(previous) await this._registerOrphanArchives([pre]) @@ -532,6 +586,10 @@ export class BackupService { throw error } finally { this._restoreRunning = false + if (this._globalOperation) { + await this._globalOperation.close() + this._globalOperation = null + } for (const item of [archive, previous]) if (item) fs.rmSync(item.tmp, { recursive: true, force: true }) } } @@ -580,6 +638,43 @@ export class BackupService { /** Auto-backup checker: every checkIntervalMs + a catch-up run at boot. */ runScheduled(generation = this._schedulerGeneration) { const pending = (async () => { + const coordinator = getActivityCoordinator(getDb()) + if (coordinator) { + const config = await coordinator.runActivity({ kind: 'backup_schedule_check' }, async () => { + const now = Number( + (await getDb().query('SELECT extract(epoch FROM clock_timestamp()) * 1000 AS now')).rows[0].now, + ) + const eligible = await this.shouldRunScheduled(now) + return { + ended: true, + value: eligible && generation === this._schedulerGeneration ? await this.getSchedule() : null, + } + }) + if (!config) return null + try { + const result = await coordinator.runScheduledMaintenance( + { taskKey: 'scheduled-backup', intervalMs: Math.round(config.interval_hours * 3600_000) }, + async (operation) => { + if (generation !== this._schedulerGeneration) { + await this._leaveMaintenance('backup') + return { ended: true, value: null } + } + const record = await this.createBackup({ kind: 'scheduled', maintenanceOperation: operation }) + return { ended: record?.status === 'ok', value: record } + }, + { + onDraining: async (operation) => { + this._globalOperation = operation + this.isMaintaining = true + await this._maintenance?.(true, 'backup', { operation }) + }, + }, + ) + return result.ran ? result.value : null + } finally { + this._globalOperation = null + } + } const now = this.sharedTasks ? await this.sharedTasks.now() : Date.now() if (!(await this.shouldRunScheduled(now)) || generation !== this._schedulerGeneration) return null const config = await this.getSchedule() diff --git a/src/lib/db/global-maintenance.mjs b/src/lib/db/global-maintenance.mjs index bfe7906af..107c96650 100644 --- a/src/lib/db/global-maintenance.mjs +++ b/src/lib/db/global-maintenance.mjs @@ -338,7 +338,7 @@ export class GlobalMaintenanceCoordinator { const scope = { coordinator: this, id, epoch, kind, active: true, remoteStarted: false, pending: new Set() } const assertActive = () => { this._assertHealthy() - if ((!scope.active && !scope.transferredTo?.healthy) || scope.handoffPending) { + if ((!scope.active && !scope.transferredTo?.healthy) || scope.handoffPending || scope.completing) { throw failure('maintenance_activity_closed', 'The activity scope has completed or is transferring') } } @@ -372,7 +372,7 @@ export class GlobalMaintenanceCoordinator { this._assertHealthy() for (let context = activities.getStore(); context; context = context.parent) { const { scope } = context - if ((!scope.active && !scope.transferredTo?.healthy) || scope.handoffPending) { + if ((!scope.active && !scope.transferredTo?.healthy) || scope.handoffPending || scope.completing) { throw failure('maintenance_activity_closed', 'An inherited activity scope has completed') } scope.operation?._assertHealthy() @@ -385,38 +385,55 @@ export class GlobalMaintenanceCoordinator { const scope = this.assertActivity() const token = Symbol('persistence') scope.pending.add(token) - return () => scope.pending.delete(token) + return () => { + scope.pending.delete(token) + if (!scope.pending.size) { + for (const resolve of scope.waiters || []) resolve() + scope.waiters?.clear() + } + } } async _complete(scope, execution) { return this._exclusiveLocal(async () => { this._assertHealthy() - const result = await this._transaction(async (client) => { - const row = ( - await client.query( - `SELECT * FROM ${this.qualified}.activities + if (execution?.ended === true && scope.pending.size) { + throw failure('maintenance_persistence_pending', 'Activity SQL must finish before confirming completion') + } + scope.completing = true + let result + try { + result = await this._transaction(async (client) => { + const row = ( + await client.query( + `SELECT * FROM ${this.qualified}.activities WHERE id = $1 AND owner_id = $2 FOR UPDATE`, - [scope.id, this.ownerId], - ) - ).rows[0] - if (!row) throw failure('maintenance_activity_missing', 'The durable activity is unavailable') - if (row.completed_at) return { ended: true, uncertain: false } - if (row.uncertain_at) return { ended: false, uncertain: true } - const ended = execution?.ended === true - await client.query( - `UPDATE ${this.qualified}.activities SET + [scope.id, this.ownerId], + ) + ).rows[0] + if (!row) throw failure('maintenance_activity_missing', 'The durable activity is unavailable') + if (row.completed_at) return { ended: true, uncertain: false } + if (row.uncertain_at) return { ended: false, uncertain: true } + const ended = execution?.ended === true + await client.query( + `UPDATE ${this.qualified}.activities SET completed_at = CASE WHEN $3 THEN clock_timestamp() ELSE NULL END, uncertain_at = CASE WHEN $3 THEN NULL ELSE clock_timestamp() END, reason = $4 WHERE id = $1 AND owner_id = $2`, - [scope.id, this.ownerId, ended, execution?.reason || (ended ? 'completed' : 'execution_unknown')], - ) - return { ended, uncertain: !ended } - }) + [scope.id, this.ownerId, ended, execution?.reason || (ended ? 'completed' : 'execution_unknown')], + ) + return { ended, uncertain: !ended } + }) + } catch (error) { + scope.completing = false + throw error + } if (scope.active) { scope.active = false this._refs -= 1 if (!this._refs) await this._releaseHolder() } + scope.completing = false return result }) } @@ -480,10 +497,10 @@ export class GlobalMaintenanceCoordinator { } } - async runScheduledMaintenance(plan, callback) { + async runScheduledMaintenance(plan, callback, { onDraining = null } = {}) { let operation try { - operation = await this._beginMaintenance({ kind: 'backup' }, null, plan) + operation = await this._beginMaintenance({ kind: 'backup', onDraining }, null, plan) } catch (error) { if (error.code === 'maintenance_already_completed') { return { ran: false, code: error.code, sharedResult: error.sharedResult } @@ -542,13 +559,18 @@ export class GlobalMaintenanceCoordinator { }) } - async _beginMaintenance({ kind, signal = null, timeoutMs = 60000 } = {}, sourceScope = null, schedule = null) { + async _beginMaintenance( + { kind, signal = null, timeoutMs = 60000, onDraining = null } = {}, + sourceScope = null, + schedule = null, + ) { this._assertHealthy() if (!['backup', 'restore'].includes(kind)) throw new Error('invalid maintenance kind') if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) throw new Error('invalid maintenance timeout') if (signal?.aborted) throw failure('maintenance_cancelled', 'Maintenance was cancelled before admission') const client = this._connection('maintenance') const operation = new MaintenanceOperation(this, client, kind) + operation.sourceActivityId = sourceScope?.id || null try { await client.connect() operation.backendPid = (await client.query('SELECT pg_backend_pid() AS pid')).rows[0].pid @@ -636,6 +658,7 @@ export class GlobalMaintenanceCoordinator { throw error } }) + if (onDraining) await operation.runAdmin(() => onDraining(operation)) // DRAINING is committed before waiting. Existing scopes never reacquire this lock for their SQL. await client.query("SELECT set_config('statement_timeout', $1, false)", [String(timeoutMs)]) const abort = () => { @@ -649,6 +672,7 @@ export class GlobalMaintenanceCoordinator { signal?.removeEventListener('abort', abort) } operation._assertHealthy() + operation._drained = true await client.query("SELECT set_config('statement_timeout', '60000', false)") await this._checkClaims(client, operation.operationId) return operation @@ -754,6 +778,12 @@ class MaintenanceOperation { try { return await activities.run({ scope, parent: activities.getStore() }, callback) } finally { + if (scope.pending.size) { + await new Promise((resolve) => { + scope.waiters ||= new Set() + scope.waiters.add(resolve) + }) + } scope.active = false } } @@ -795,6 +825,7 @@ class MaintenanceOperation { } async beginRestore() { + if (!this._drained) throw failure('maintenance_operation_changed', 'Drain all activities before replacing data') return this._transaction(async (row) => { if (this.kind !== 'restore' || row.phase !== 'draining') throw failure('maintenance_operation_changed', 'Restore is not in its pre-restore stage') @@ -817,6 +848,7 @@ class MaintenanceOperation { } async resume() { + if (!this._drained) throw failure('maintenance_operation_changed', 'Drain all activities before resuming') const state = await this._transaction(async (row) => { if ( (this.kind === 'restore' && row.phase !== 'rebinding') || diff --git a/src/server.mjs b/src/server.mjs index 09abb52f1..35361c446 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -641,9 +641,8 @@ backupService.onRestored(async (db) => { await loadSettingsState(db) await loadModelPolicy({ force: true }) await loadGptModelPolicy({ force: true }) - // A restore swaps the database pool and routing files. Rebuild every - // holder before reporting success; otherwise requests keep stale snapshots - // or silently write through statements owned by the closed connection. + // A restore replaces database objects. Rebuild every store and cache before + // reporting success so requests use the restored configuration and state. for (const store of [apiKeyStore, apiEndpointStore, panelUsers, accountQuota, stickyRouter, proxyPool, requestLog]) { await store.rebind(db) } @@ -683,14 +682,21 @@ backupService.onRestored(async (db) => { reloadActiveVm(cfg) await vmTestSchedule.reload() }) -backupService.onMaintenance(async (enabled) => { +backupService.onMaintenance(async (enabled, _reason, context = {}) => { if (enabled) { await routingRt.routingState?.stop() vmTestSchedule.stop() proxyPool.stopScheduler() requestLog.stopCleanupScheduler() for (const monitor of maintenanceMonitors()) await monitor?.stop?.() - while ([...activeRequests].some((request) => !/\/(?:backups)(?:\/|$)/.test(String(request.url || '')))) { + while ( + [...activeRequests].some((request) => { + if (context.operation?.sourceActivityId) { + return requestActivities.get(request)?.gate.activityId !== context.operation.sourceActivityId + } + return activityCoordinator || !/\/(?:backups)(?:\/|$)/.test(String(request.url || '')) + }) + ) { await new Promise((resolve) => setTimeout(resolve, 10)) } for (const monitor of maintenanceMonitors()) await monitor?.flush?.() diff --git a/test/e2e/maintenance-backup.e2e.test.mjs b/test/e2e/maintenance-backup.e2e.test.mjs new file mode 100644 index 000000000..0c1cd4c5a --- /dev/null +++ b/test/e2e/maintenance-backup.e2e.test.mjs @@ -0,0 +1,95 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import pg from 'pg' +import { startGateway, api } from '../harness.mjs' +import { coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' +import { quoteIdentifier } from '../../src/lib/db/database.mjs' + +async function fixture(t, scheduled = false) { + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-maintenance-backup-')) + fs.mkdirSync(path.join(project, 'vms')) + fs.writeFileSync(path.join(project, 'vms', 'active.json'), '{}') + const gw = await startGateway({ + project, + env: { VM2API_CONTROL_PLANE_MODE: 'kubernetes', ...(scheduled ? {} : { KIN_BACKUP_DISABLED: '1' }) }, + }) + const schema = coordinationSchemaFor(gw.schema) + t.after(async () => { + await gw.stop() + const cleanup = new pg.Client({ connectionString: gw.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(schema)} CASCADE`) + } finally { + await cleanup.end() + fs.rmSync(project, { recursive: true, force: true }) + } + }) + return { gw, q: quoteIdentifier(schema) } +} + +test('Kubernetes manual backup transfers its HTTP activity to private maintenance history', async (t) => { + const { gw, q } = await fixture(t) + const response = await api(gw, 'POST', '/api/panel/backups') + assert.equal(response.status, 201, response.text) + const history = (await gw.db.pool.query(`SELECT kind, source_activity_id, ended_at FROM ${q}.operations`)).rows + assert.equal(history.length, 1) + assert.equal(history[0].kind, 'backup') + assert.ok(history[0].source_activity_id) + assert.ok(history[0].ended_at) + assert.equal((await fetch(gw.baseUrl + '/readyz')).status, 200) +}) + +test('Kubernetes scheduled backup uses private claims and leaves no active business task in its snapshot', async (t) => { + const { gw, q } = await fixture(t, true) + let records = [] + for (let i = 0; i < 100; i++) { + records = (await gw.db.pool.query("SELECT id FROM backup_records WHERE kind='scheduled' AND status='ok'")).rows + if (records.length) break + await new Promise((resolve) => setTimeout(resolve, 100)) + } + assert.equal(records.length, 1, gw.stderr()) + assert.equal( + (await gw.db.pool.query("SELECT count(*) AS n FROM background_task_runs WHERE task_key='scheduled-backup'")).rows[0] + .n, + 0, + ) + const history = (await gw.db.pool.query(`SELECT ended_at FROM ${q}.operations WHERE task_key='scheduled-backup'`)) + .rows + assert.equal(history.length, 1) + assert.ok(history[0].ended_at) +}) + +test('Kubernetes restore advances the durable epoch and rebinds before reopening requests', async (t) => { + const { gw, q } = await fixture(t) + const before = await api(gw, 'GET', '/api/panel/routing') + assert.equal(before.status, 200, before.text) + const backup = await api(gw, 'POST', '/api/panel/backups') + assert.equal(backup.status, 201, backup.text) + const id = backup.json.item.id + const routingFile = path.join(gw.project, 'config', 'routing.json') + const seed = JSON.parse(fs.readFileSync(routingFile, 'utf8')) + seed.pool = { ...seed.pool, queue_max: 987 } + const updatedSeed = JSON.stringify(seed, null, 2) + fs.writeFileSync(routingFile, updatedSeed) + assert.equal((await api(gw, 'PUT', '/api/panel/routing', { body: { pool: { queue_max: 23 } } })).status, 200) + const restored = await api(gw, 'POST', `/api/panel/backups/${id}/restore`, { body: { confirm: true } }) + assert.equal(restored.status, 200, restored.text) + const state = (await gw.db.pool.query(`SELECT phase, epoch FROM ${q}.control WHERE id=1`)).rows[0] + assert.equal(state.phase, 'open') + assert.equal(state.epoch, 2) + assert.equal( + (await gw.db.pool.query(`SELECT count(*) AS n FROM ${q}.members WHERE stopped_at IS NULL AND bound_epoch<>2`)) + .rows[0].n, + 0, + ) + assert.ok((await gw.db.pool.query(`SELECT ended_at FROM ${q}.operations WHERE kind='restore'`)).rows[0].ended_at) + assert.equal(fs.readFileSync(routingFile, 'utf8'), updatedSeed) + const routing = await api(gw, 'GET', '/api/panel/routing') + assert.equal(routing.status, 200, routing.text) + assert.equal(routing.json.data.pool.queue_max, before.json.data.pool.queue_max) + assert.equal((await fetch(gw.baseUrl + '/readyz')).status, 200) +}) diff --git a/test/unit/maintenance-handoff.test.mjs b/test/unit/maintenance-handoff.test.mjs index 15768d74c..2f1120af1 100644 --- a/test/unit/maintenance-handoff.test.mjs +++ b/test/unit/maintenance-handoff.test.mjs @@ -1,7 +1,10 @@ import test from 'node:test' import assert from 'node:assert/strict' import pg from 'pg' -import { getDb, quoteIdentifier, registerActivityCoordinator } from '../../src/lib/db/database.mjs' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createDatabase, getDb, quoteIdentifier, registerActivityCoordinator } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { runPgTool } from '../../src/lib/admin/backup-core.mjs' @@ -127,10 +130,7 @@ test('a transferred restore owner death retains its phase and private unfinished null, ) await assert.rejects(coordinator.admit({ kind: 'http' }), { code: 'database_maintenance' }) - await assert.rejects( - async () => request.run(() => db.query('SELECT 1')), - { code: 'maintenance_activity_closed' }, - ) + await assert.rejects(async () => request.run(() => db.query('SELECT 1')), { code: 'maintenance_activity_closed' }) }) test('handoff never excludes another active or unknown background claim', async (t) => { @@ -152,3 +152,76 @@ test('handoff never excludes another active or unknown background claim', async await request.run(() => db.query('SELECT 1')) await request.complete({ ended: true }) }) + +test('maintenance admin scope waits for its detached SQL before releasing ownership', async (t) => { + const { db, coordinator, operations } = await fixture(t) + const operation = await coordinator.beginMaintenance({ kind: 'backup' }) + operations.push(operation) + const resource = 'synthetic-admin-sql:' + db.schema + const blocker = await db.pool.connect() + await blocker.query('SELECT pg_advisory_lock(hashtext(current_database()),hashtext($1))', [resource]) + let finished = false + let released = false + let query + const admin = operation + .runAdmin(() => { + query = db + .query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [resource]) + .then(() => { + finished = true + }) + return 'callback returned' + }) + .then((value) => { + released = true + return value + }) + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(released, false) + await blocker.query('SELECT pg_advisory_unlock(hashtext(current_database()),hashtext($1))', [resource]) + assert.equal(await admin, 'callback returned') + assert.equal(finished, true) + await operation.resume() + } finally { + await blocker.query('SELECT pg_advisory_unlock_all()') + blocker.release() + await query + await admin + } +}) + +test('new Kubernetes databases cannot run migrations through RESTORING or REBINDING', async (t) => { + const { db, coordinator, operations } = await fixture(t) + const migrationsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'maintenance-startup-')) + fs.writeFileSync( + path.join(migrationsDir, '999_should_not_run.sql'), + 'CREATE TABLE should_not_run_during_restore(id integer);', + ) + t.after(() => fs.rmSync(migrationsDir, { recursive: true, force: true })) + const operation = await coordinator.beginMaintenance({ kind: 'restore' }) + operations.push(operation) + await operation.beginRestore() + for (const phase of ['restoring', 'rebinding']) { + if (phase === 'rebinding') await operation.beginRebind() + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + try { + await assert.rejects( + createDatabase({ connectionString: db.connectionString, schema: db.schema, migrationsDir }), + { code: 'database_maintenance' }, + ) + } finally { + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + } + assert.equal( + (await db.pool.query('SELECT to_regclass($1) AS name', [`${db.schema}.should_not_run_during_restore`])).rows[0] + .name, + null, + ) + assert.equal((await coordinator.getState()).phase, phase) + } + await coordinator.acknowledgeEpoch(2) + await operation.resume() +}) From 46c9afedb6d53734791c4b9c6a105be75f0a1e43 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:23:31 +0800 Subject: [PATCH 230/473] test: reproduce unconfirmed egress owner and container termination --- test/unit/egress-stop-proof.test.mjs | 170 +++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 test/unit/egress-stop-proof.test.mjs diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs new file mode 100644 index 000000000..193e9f7ac --- /dev/null +++ b/test/unit/egress-stop-proof.test.mjs @@ -0,0 +1,170 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import vmContext from 'node:vm' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { egressRunDir, startEgressProcess, stopProxyEgress } from '../../src/lib/vm/egress.mjs' + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'egress-stop-proof-')) + const dir = egressRunDir(root, 'synthetic') + fs.mkdirSync(dir, { recursive: true }) + const script = path.join(root, 'kin-egress-fixture.mjs') + fs.writeFileSync( + script, + "process.on('SIGTERM', () => setTimeout(() => process.exit(0), 80)); setInterval(() => {}, 1000)\n", + ) + const bin = path.join(root, 'kin-egress-fixture.sh') + fs.writeFileSync(bin, `#!/bin/sh\nexec '${process.execPath}' '${script}' "$@"\n`, { mode: 0o700 }) + const pids = [] + t.after(() => { + for (const pid of pids) { + try { + process.kill(pid, 'SIGKILL') + } catch {} + } + fs.rmSync(root, { recursive: true, force: true }) + }) + const start = async () => { + const result = await startEgressProcess({ + projectRoot: root, + proxyId: 'synthetic', + proxyUrl: 'socks5h://192.0.2.1:1080', + listenHost: '127.0.0.1', + tcpPort: 0, + dnsPort: 0, + bin, + }) + if (result.pid) pids.push(result.pid) + return result + } + return { root, dir, bin, pids, start } +} + +const noDocker = { + runDocker: () => ({ ok: false, stderr: 'Error: No such network: synthetic' }), + runIptables: () => ({ ok: true, stdout: '' }), +} + +test('new egress starts atomically persist observable execution identity', { + skip: process.platform !== 'linux', +}, async (t) => { + const f = fixture(t) + const result = await f.start() + assert.equal(result.ok, true) + const receipt = JSON.parse(fs.readFileSync(path.join(f.dir, 'egress.owner.json'), 'utf8')) + assert.equal(receipt.pid, result.pid) + assert.equal(receipt.proxy_id, 'synthetic') + assert.equal(receipt.boot_id, fs.readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim()) + assert.equal(receipt.pid_namespace, fs.readlinkSync('/proc/self/ns/pid')) + assert.match(receipt.start_ticks, /^\d+$/) +}) + +test('receipt persistence failure after spawn is unknown and preserves its PID record', { + skip: process.platform !== 'linux', +}, async (t) => { + const f = fixture(t) + const rename = fs.renameSync + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === path.join(f.dir, 'egress.owner.json')) + throw Object.assign(new Error('synthetic receipt failure'), { code: 'EACCES' }) + return rename(from, to) + }) + const result = await f.start() + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal(Number(fs.readFileSync(path.join(f.dir, 'egress.pid'), 'utf8')), result.pid) +}) + +test('Kubernetes legacy or foreign-namespace PID records cannot prove the old helper ended', { + skip: process.platform !== 'linux', +}, async (t) => { + const mode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + }) + for (const foreign of [false, true]) { + const f = fixture(t) + const child = spawn(f.bin, ['-config', path.join(f.dir, 'egress.json')], { stdio: 'ignore' }) + const closed = once(child, 'close') + await once(child, 'spawn') + t.after(async () => { + child.kill('SIGKILL') + await closed + }) + fs.writeFileSync(path.join(f.dir, 'egress.pid'), String(child.pid)) + fs.writeFileSync(path.join(f.dir, 'egress.json'), '{}') + if (foreign) + fs.writeFileSync( + path.join(f.dir, 'egress.owner.json'), + JSON.stringify({ + pid: child.pid, + proxy_id: 'synthetic', + boot_id: 'another-host', + pid_namespace: 'pid:[foreign]', + start_ticks: '1', + }), + ) + const result = await stopProxyEgress(f.root, 'synthetic', noDocker) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal(fs.existsSync(path.join(f.dir, 'egress.pid')), true) + } +}) + +test('Docker network inspection failure cannot be treated as absent during proxy teardown', async (t) => { + const f = fixture(t) + const result = await stopProxyEgress(f.root, 'synthetic', { + ...noDocker, + runDocker: () => ({ ok: false, stderr: 'permission denied connecting to Docker daemon' }), + }) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) +}) + +function remoteFixture({ noop = false, permission = false } = {}) { + const source = fs.readFileSync(new URL('../../src/lib/cluster/remote-slot.mjs', import.meta.url), 'utf8') + const body = source + .slice(source.indexOf('export async function setRemoteProxyEgressEnabled('), source.indexOf('/** Where a network')) + .replace('export ', '') + let running = true + const ids = [] + const stop = vmContext.runInNewContext(`${body}; setRemoteProxyEgressEnabled`, { + nodeSession: async () => ({ docker: {} }), + vmNodeId: () => 'node-synthetic', + slotEgressNames: () => ({ egress: 'kin-egress-current' }), + proxyKey: () => 'legacy', + inspectContainerOrNull: async () => { + if (permission) throw new Error('Docker access denied') + return { Id: 'immutable-egress-id', State: { Running: running, Restarting: false, Pid: running ? 123 : 0 } } + }, + containerAction: async (_docker, id) => { + ids.push(id) + if (!noop) running = false + }, + remoteContainerStopped: (info) => + info.State?.Running === false && info.State?.Restarting === false && info.State?.Pid === 0, + failure: (error) => ({ ok: false, error: error.message }), + }) + return { stop, ids } +} + +test('remote egress API acknowledgement requires observed immutable container termination', async () => { + const f = remoteFixture({ noop: true }) + const result = await f.stop({ id: 'vm-synthetic', proxy: { id: 'synthetic' } }, '/synthetic', false) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) +}) + +test('remote egress stop addresses immutable IDs and permission failures remain unknown', async () => { + const f = remoteFixture() + assert.equal((await f.stop({ id: 'vm-synthetic', proxy: { id: 'synthetic' } }, '/synthetic', false)).ok, true) + assert.equal(f.ids[0], 'immutable-egress-id') + const unknown = await remoteFixture({ permission: true }).stop({ id: 'vm-synthetic', proxy: {} }, '/synthetic', false) + assert.equal(unknown.vmOperationUncertain, true) +}) From 7e943f8fdc39f1a0d104ae03f0a24f550951329b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:24:15 +0800 Subject: [PATCH 231/473] test: reproduce duplicate notifications across control planes --- test/unit/notify-shared.test.mjs | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 test/unit/notify-shared.test.mjs diff --git a/test/unit/notify-shared.test.mjs b/test/unit/notify-shared.test.mjs new file mode 100644 index 000000000..624d2e303 --- /dev/null +++ b/test/unit/notify-shared.test.mjs @@ -0,0 +1,107 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { createNotifyMonitor, summarizePoolAvailability } from '../../src/lib/admin/notify.mjs' + +const config = { + enabled: true, + interval_sec: 15, + cooldown_sec: 300, + events: { pool_empty: true, pool_low: false, pool_recovered: true, digest: false, revoked: false, invalid: false }, +} + +const empty = () => + summarizePoolAvailability([{ id: 'synthetic-vm', has_token: true, cred_status: { key: 'bad', text: 'synthetic' } }]) + +async function fixture(t, { nextPlan = false, notify = config } = {}) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => peer.close()) + await new SettingsRepo(db).set('routing_config', { revision: 1, config: { notify } }) + let execution = 0 + const tasks = [new SharedBackgroundTasks(db), new SharedBackgroundTasks(peer)].map((service) => + nextPlan + ? { + db: service.db, + now: () => service.now(), + run: (plan, callback) => + service.run({ taskKey: plan.taskKey, executionId: `synthetic-check-${++execution}` }, callback), + } + : service, + ) + const monitors = (send) => + tasks.map((sharedTasks) => createNotifyMonitor({ config, snapshot: async () => empty(), send, sharedTasks })) + return { db, peer, monitors } +} + +test('two independent PostgreSQL notification monitors execute one shared check and one send', async (t) => { + const { monitors } = await fixture(t) + let sends = 0 + const pair = monitors(async () => { + sends++ + await new Promise((resolve) => setTimeout(resolve, 30)) + return { ok: true } + }) + await Promise.all(pair.map((monitor) => monitor.runOnce())) + assert.equal(sends, 1) +}) + +test('another Pod reuses the persisted event baseline and cooldown on a later check', async (t) => { + const { monitors } = await fixture(t, { nextPlan: true }) + let sends = 0 + const [first, peer] = monitors(async () => { + sends++ + return { ok: true } + }) + await first.runOnce() + await peer.runOnce() + assert.equal(sends, 1) + assert.equal(peer.getSnapshot().available, 0) +}) + +test('a shared check obeys committed disabled configuration instead of its local enabled snapshot', async (t) => { + const { monitors } = await fixture(t, { notify: { ...config, enabled: false } }) + let sends = 0 + const [monitor] = monitors(async () => { + sends++ + return { ok: true } + }) + await monitor.runOnce() + assert.equal(sends, 0) + assert.equal(monitor.getConfig().enabled, false) +}) + +test('an uncertain notification attempt blocks another Pod and does not expire into a retry', async (t) => { + const { db, monitors } = await fixture(t, { nextPlan: true }) + let sends = 0 + const [first, peer] = monitors(async () => { + sends++ + throw new Error('synthetic transport lost after send') + }) + await first.runOnce() + await db.exec("UPDATE background_task_runs SET claimed_at=clock_timestamp()-interval '10 years'") + await peer.runOnce() + assert.equal(sends, 1) + const claims = (await db.query('SELECT finished_at, unknown_at FROM background_task_runs')).rows + assert.equal(claims.length, 1) + assert.equal(claims[0].finished_at, null) + assert.ok(claims[0].unknown_at) +}) + +test('PostgreSQL state reservation failure prevents sending and is returned to the caller', async (t) => { + const { db, monitors } = await fixture(t) + await db.exec(`CREATE FUNCTION refuse_notify_state() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.key='pool_notify_state' THEN RAISE EXCEPTION 'synthetic state failure' USING ERRCODE='P6483'; END IF; + RETURN NEW; END $$; + CREATE TRIGGER refuse_notify_state BEFORE INSERT OR UPDATE ON settings + FOR EACH ROW EXECUTE FUNCTION refuse_notify_state()`) + let sends = 0 + const [monitor] = monitors(async () => { + sends++ + return { ok: true } + }) + await assert.rejects(monitor.runOnce(), { code: 'P6483' }) + assert.equal(sends, 0) +}) From e726669eb1c9c1db8744356a23c28a4c0882d25f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:27:17 +0800 Subject: [PATCH 232/473] test(slotgate): require node-owned immutable Docker launch and crash retention --- .../internal/slotgate/docker_launch_test.go | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 worker/internal/slotgate/docker_launch_test.go diff --git a/worker/internal/slotgate/docker_launch_test.go b/worker/internal/slotgate/docker_launch_test.go new file mode 100644 index 000000000..32a540346 --- /dev/null +++ b/worker/internal/slotgate/docker_launch_test.go @@ -0,0 +1,315 @@ +package slotgate + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +type fakeLaunchEngine struct { + mu sync.Mutex + image string + container string + startedAt string + spec map[string]any + calls []string + failStart bool + badCreateID bool + createEntered chan struct{} + createProceed chan struct{} + onInspect func() +} + +func (f *fakeLaunchEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.calls = append(f.calls, r.Method+" "+r.URL.Path) + if r.URL.Path == "/info" { + f.mu.Unlock() + fmt.Fprint(w, `{"ID":"test-daemon"}`) + return + } + if strings.HasPrefix(r.URL.Path, "/images/") { + image := f.image + f.mu.Unlock() + json.NewEncoder(w).Encode(map[string]any{"Id": image, "Config": map[string]any{"Labels": map[string]string{"kin.slot.full": "1"}}}) + return + } + if r.URL.Path == "/containers/create" { + if err := json.NewDecoder(r.Body).Decode(&f.spec); err != nil { + f.mu.Unlock() + w.WriteHeader(400) + return + } + entered, proceed, bad, id := f.createEntered, f.createProceed, f.badCreateID, f.container + f.mu.Unlock() + if entered != nil { + close(entered) + <-proceed + } + w.WriteHeader(201) + if bad { + id = "not-an-immutable-id" + } + json.NewEncoder(w).Encode(map[string]any{"Id": id}) + return + } + if strings.HasSuffix(r.URL.Path, "/start") { + fail := f.failStart + f.mu.Unlock() + if fail { + w.WriteHeader(500) + } else { + w.WriteHeader(204) + } + return + } + if r.Method == "GET" && r.URL.Path == "/containers/"+f.container+"/json" { + id, image, started, spec, observe := f.container, f.image, f.startedAt, f.spec, f.onInspect + f.mu.Unlock() + if observe != nil { + observe() + } + json.NewEncoder(w).Encode(map[string]any{"Id": id, "Image": image, "Config": map[string]any{"Labels": spec["Labels"]}, "State": map[string]any{"Running": true, "Restarting": false, "StartedAt": started}}) + return + } + f.mu.Unlock() + w.WriteHeader(404) +} + +func launchFixture(t *testing.T) (*Authority, *dockerLauncher, *fakeLaunchEngine, AuthorityState, string) { + t.Helper() + root := shortRoot(t) + boot, err := nodeBootID() + if err != nil { + t.Fatal(err) + } + a := authorityForTest(t, filepath.Join(root, "authority"), terminatedExecutor) + state, err := a.CreateSlot("v") + if err != nil { + t.Fatal(err) + } + policy := DockerLaunchPolicy{ImageID: "sha256:" + strings.Repeat("b", 64), NetworkID: "none", UID: os.Getuid(), GID: os.Getgid()} + if policy.UID == 0 { + policy.UID = 10000 + policy.GID = 10000 + } + private := filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation) + for _, dir := range []string{"config", "run", "home", "claude"} { + if err = os.MkdirAll(filepath.Join(private, dir), 0o700); err != nil { + t.Fatal(err) + } + } + for _, file := range []string{"kernel.json", "internal.token"} { + if err = os.WriteFile(filepath.Join(private, "config", file), []byte("synthetic-only"), 0o600); err != nil { + t.Fatal(err) + } + } + fake := &fakeLaunchEngine{image: policy.ImageID, container: strings.Repeat("c", 64), startedAt: "2026-10-21T01:00:00Z"} + socket := filepath.Join(shortRoot(t), "docker.sock") + unixHTTP(t, socket, fake) + launcher, err := newDockerLauncher(socket, root, boot, a, policy) + if err != nil { + t.Fatal(err) + } + return a, launcher, fake, state, root +} + +func TestDockerLaunchBindsOnlyNodeCreatedInspectedIdentityWithRestrictedSpec(t *testing.T) { + a, launcher, engine, state, root := launchFixture(t) + active, err := launcher.Launch(context.Background(), "v", state.Fence, "launch-1") + if err != nil { + t.Fatal(err) + } + if active.Phase != PhaseActive || active.Fence.Executor.ContainerID != engine.container || active.Fence.Executor.StartedAt != engine.startedAt || active.Fence.Executor.DockerDaemonID != "test-daemon" { + t.Fatalf("bound executor: %+v", active) + } + engine.mu.Lock() + spec, calls := engine.spec, append([]string(nil), engine.calls...) + engine.mu.Unlock() + if spec["Image"] != engine.image || fmt.Sprint(spec["Entrypoint"]) != "[/opt/kin/kin-kernel]" || fmt.Sprint(spec["Cmd"]) != "[--gateway-worker --config /run/kin/kernel.json]" { + t.Fatalf("payload spec: %+v", spec) + } + host := spec["HostConfig"].(map[string]any) + if host["Privileged"] != false || host["ReadonlyRootfs"] != true || host["NetworkMode"] != "none" || fmt.Sprint(host["CapDrop"]) != "[ALL]" || fmt.Sprint(host["SecurityOpt"]) != "[no-new-privileges]" { + t.Fatalf("unsafe host config: %+v", host) + } + if fmt.Sprint(host["RestartPolicy"]) != "map[Name:no]" { + t.Fatalf("uncontrolled restart: %+v", host) + } + for _, bind := range host["Binds"].([]any) { + if !strings.HasPrefix(bind.(string), filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation)+"/") || strings.Contains(bind.(string), "docker.sock") || strings.Contains(bind.(string), "authority") { + t.Fatalf("escaped private mount: %s", bind) + } + } + if !strings.Contains(strings.Join(calls, "\n"), "POST /containers/"+engine.container+"/start") { + t.Fatalf("start did not pin immutable ID: %v", calls) + } + stored, _ := a.State("v") + if stored.Commands["launch-1"].LaunchTarget == nil || stored.Commands["launch-1"].LaunchTarget.ContainerID != engine.container || stored.Commands["launch-1"].Status != CommandCompleted { + t.Fatalf("launch target not retained: %+v", stored) + } +} + +func TestDockerLaunchNetworkIOLeavesOtherVMAdmissionAvailableAndPendingBlocksAdvance(t *testing.T) { + a, launcher, engine, state, _ := launchFixture(t) + other := launchForTest(t, a, "other", testExecutor("d")) + engine.createEntered, engine.createProceed = make(chan struct{}), make(chan struct{}) + finished := make(chan error, 1) + go func() { + _, err := launcher.Launch(context.Background(), "v", state.Fence, "slow-create") + finished <- err + }() + <-engine.createEntered + admitted := make(chan error, 1) + go func() { _, err := a.BeginCommand("other", other.Fence, "parallel", CommandForward); admitted <- err }() + select { + case err := <-admitted: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + close(engine.createProceed) + t.Fatal("Engine I/O held the Authority global lock") + } + if _, err := a.Revoke("v", state.Fence); err != nil { + t.Fatal(err) + } + _, err := a.Advance(context.Background(), "v", state.Fence) + requireAuthorityError(t, err, ErrPendingMutation) + close(engine.createProceed) + if err = <-finished; err == nil { + t.Fatal("late launch bound after revocation") + } + stored, _ := a.State("v") + if stored.Phase != PhaseRevoking || stored.Commands["slow-create"].LaunchTarget == nil { + t.Fatalf("late create identity was lost: %+v", stored) + } + engine.mu.Lock() + calls := append([]string(nil), engine.calls...) + engine.mu.Unlock() + for _, call := range calls { + if strings.HasSuffix(call, "/start") { + t.Fatalf("revoked launch started: %v", calls) + } + } +} + +func TestDockerLaunchStartFailureAndNodeRestartNeverResubmitCreate(t *testing.T) { + a, launcher, engine, state, root := launchFixture(t) + engine.failStart = true + if _, err := launcher.Launch(context.Background(), "v", state.Fence, "start-unknown"); err == nil { + t.Fatal("failed start succeeded") + } + stored, _ := a.State("v") + if stored.Commands["start-unknown"].LaunchTarget == nil || stored.Commands["start-unknown"].Status != CommandPending { + t.Fatalf("start error discarded identity: %+v", stored) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, filepath.Join(root, "authority"), terminatedExecutor) + launcher.authority = reopened + if _, err := launcher.Launch(context.Background(), "v", state.Fence, "retry-after-crash"); err == nil { + t.Fatal("unknown launch retried after restart") + } + engine.mu.Lock() + calls := append([]string(nil), engine.calls...) + engine.mu.Unlock() + creates := 0 + for _, call := range calls { + if call == "POST /containers/create" { + creates++ + } + } + if creates != 1 { + t.Fatalf("create count=%d, want 1", creates) + } + stored, _ = reopened.State("v") + if stored.Phase != PhaseRevoking || stored.Commands["start-unknown"].Status != CommandUnknown { + t.Fatalf("crash state: %+v", stored) + } +} + +func TestDockerLaunchUnknownCreateResponseAndBindPersistenceFailureRetainClaim(t *testing.T) { + t.Run("unknown-create-id", func(t *testing.T) { + a, launcher, engine, state, _ := launchFixture(t) + engine.badCreateID = true + if _, err := launcher.Launch(context.Background(), "v", state.Fence, "unknown-create"); err == nil { + t.Fatal("invalid immutable ID bound") + } + stored, _ := a.State("v") + if stored.Commands["unknown-create"].Status != CommandPending || !stored.Commands["unknown-create"].Dispatched { + t.Fatalf("unknown create released: %+v", stored) + } + }) + t.Run("bind-commit-failed", func(t *testing.T) { + a, launcher, engine, state, root := launchFixture(t) + journal := filepath.Join(root, "authority", "authority.json") + engine.onInspect = func() { + if err := os.Rename(journal, journal+".saved"); err != nil { + t.Error(err) + } + if err := os.Mkdir(journal, 0o700); err != nil { + t.Error(err) + } + } + if _, err := launcher.Launch(context.Background(), "v", state.Fence, "bind-uncertain"); err == nil { + t.Fatal("failed bind commit returned success") + } + if err := os.Remove(journal); err != nil { + t.Fatal(err) + } + if err := os.Rename(journal+".saved", journal); err != nil { + t.Fatal(err) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, filepath.Join(root, "authority"), terminatedExecutor) + stored, _ := reopened.State("v") + if stored.Phase != PhaseRevoking || stored.Commands["bind-uncertain"].LaunchTarget == nil || stored.Commands["bind-uncertain"].Status != CommandUnknown { + t.Fatalf("running executor forgotten: %+v", stored) + } + }) +} + +func TestDockerLaunchRejectsRootCapabilitiesAndUnsafePrivateFilesBeforeCreate(t *testing.T) { + for _, policy := range []DockerLaunchPolicy{ + {ImageID: "tag:mutable", NetworkID: "none", UID: 10000, GID: 10000}, + {ImageID: "sha256:" + strings.Repeat("b", 64), NetworkID: "host", UID: 10000, GID: 10000}, + {ImageID: "sha256:" + strings.Repeat("b", 64), NetworkID: "container:other", UID: 10000, GID: 10000}, + {ImageID: "sha256:" + strings.Repeat("b", 64), NetworkID: "none", UID: 0, GID: 0}, + } { + if _, err := newDockerLauncher("/tmp/synthetic-engine.sock", "/tmp/synthetic-node", "synthetic-boot", nil, policy); err == nil { + t.Fatalf("unsafe launch policy accepted: %+v", policy) + } + } + a, launcher, engine, state, root := launchFixture(t) + config := filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "config", "kernel.json") + if err := os.Remove(config); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "authority", "authority.json"), config); err != nil { + t.Fatal(err) + } + if _, err := launcher.Launch(context.Background(), "v", state.Fence, "unsafe-file"); err == nil { + t.Fatal("followed config symlink") + } + engine.mu.Lock() + calls := append([]string(nil), engine.calls...) + engine.mu.Unlock() + if len(calls) != 0 { + t.Fatalf("unsafe file reached Engine: %v", calls) + } + stored, _ := a.State("v") + if _, exists := stored.Commands["unsafe-file"]; exists { + t.Fatal("invalid preparation reserved a launch") + } +} From 20f52c2619fa3e45cf7735d3652b98e67cfd39db Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:27:42 +0800 Subject: [PATCH 233/473] test: retain unreceipted egress execution after PID identity changes --- test/unit/egress-stop-proof.test.mjs | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs index 193e9f7ac..bd73e3797 100644 --- a/test/unit/egress-stop-proof.test.mjs +++ b/test/unit/egress-stop-proof.test.mjs @@ -7,6 +7,7 @@ import vmContext from 'node:vm' import { spawn } from 'node:child_process' import { once } from 'node:events' import { egressRunDir, startEgressProcess, stopProxyEgress } from '../../src/lib/vm/egress.mjs' +import * as egress from '../../src/lib/vm/egress.mjs' function fixture(t) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'egress-stop-proof-')) @@ -23,7 +24,7 @@ function fixture(t) { t.after(() => { for (const pid of pids) { try { - process.kill(pid, 'SIGKILL') + if (fs.readFileSync(`/proc/${pid}/cmdline`, 'utf8').includes(root)) process.kill(pid, 'SIGKILL') } catch {} } fs.rmSync(root, { recursive: true, force: true }) @@ -114,9 +115,55 @@ test('Kubernetes legacy or foreign-namespace PID records cannot prove the old he assert.equal(result.ok, false) assert.equal(result.vmOperationUncertain, true) assert.equal(fs.existsSync(path.join(f.dir, 'egress.pid')), true) + await new Promise((resolve) => setTimeout(resolve, 150)) + assert.equal(child.exitCode, null) + assert.equal(child.signalCode, null) } }) +test('confirmed egress stop waits for its recorded execution and clears its receipt', { + skip: process.platform !== 'linux', +}, async (t) => { + const f = fixture(t) + assert.equal((await f.start()).ok, true) + const result = await egress.stopEgressProcessConfirmed(f.root, 'synthetic', { termTimeoutMs: 500, killGraceMs: 500 }) + assert.equal(result.ok, true) + assert.equal(result.ended, true) + assert.equal(fs.existsSync(path.join(f.dir, 'egress.owner.json')), false) + assert.equal(fs.existsSync(path.join(f.dir, 'egress.pid')), false) +}) + +test('an unreceipted helper using a recycled PID cannot be declared absent or signalled', { + skip: process.platform !== 'linux', +}, async (t) => { + const f = fixture(t) + const started = await f.start() + const receipt = JSON.parse(fs.readFileSync(path.join(f.dir, 'egress.owner.json'), 'utf8')) + receipt.start_ticks = String(BigInt(receipt.start_ticks) + 1n) + fs.writeFileSync(path.join(f.dir, 'egress.owner.json'), JSON.stringify(receipt)) + const result = await egress.stopEgressProcessConfirmed(f.root, 'synthetic') + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal(fs.existsSync(path.join(f.dir, 'egress.pid')), true) + assert.equal(process.kill(started.pid, 0), true) +}) + +test('process inspection permission failure retains its receipt and ownership', { + skip: process.platform !== 'linux', +}, async (t) => { + const f = fixture(t) + const started = await f.start() + const read = fs.readFileSync + t.mock.method(fs, 'readFileSync', (file, ...args) => { + if (file === `/proc/${started.pid}/stat`) throw Object.assign(new Error('inspection denied'), { code: 'EACCES' }) + return read(file, ...args) + }) + const result = await egress.stopEgressProcessConfirmed(f.root, 'synthetic') + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal(fs.existsSync(path.join(f.dir, 'egress.owner.json')), true) +}) + test('Docker network inspection failure cannot be treated as absent during proxy teardown', async (t) => { const f = fixture(t) const result = await stopProxyEgress(f.root, 'synthetic', { From 309d8bbca1cf3e1685d69ddd616e79442e00ce9e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:30:11 +0800 Subject: [PATCH 234/473] test: reject truncated notification provider responses --- test/unit/notify-shared.test.mjs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/unit/notify-shared.test.mjs b/test/unit/notify-shared.test.mjs index 624d2e303..55200eb7f 100644 --- a/test/unit/notify-shared.test.mjs +++ b/test/unit/notify-shared.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' -import { createNotifyMonitor, summarizePoolAvailability } from '../../src/lib/admin/notify.mjs' +import { createNotifyMonitor, dispatchNotify, summarizePoolAvailability } from '../../src/lib/admin/notify.mjs' const config = { enabled: true, @@ -105,3 +105,25 @@ test('PostgreSQL state reservation failure prevents sending and is returned to t await assert.rejects(monitor.runOnce(), { code: 'P6483' }) assert.equal(sends, 0) }) + +test('a truncated provider response cannot confirm the notification attempt ended', async () => { + for (const channel of ['telegram', 'wecom']) { + const cfg = { + [channel]: + channel === 'telegram' + ? { enabled: true, bot_token: 'synthetic-token', chat_id: 'synthetic-chat' } + : { enabled: true, webhook: 'synthetic-key-1234' }, + } + const result = await dispatchNotify(cfg, { text: 'synthetic notification' }, { + fetch: async () => ({ + ok: true, + status: 200, + text: async () => { + throw new Error('synthetic truncated provider body') + }, + }), + }) + assert.equal(result.ok, false, channel) + assert.equal(result.uncertain, true, channel) + } +}) From 304e3d176b1391df76118a2151fc9fa4df7f211e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:30:12 +0800 Subject: [PATCH 235/473] test: retain unknown Codex owner when changing egress policy --- test/unit/egress-stop-proof.test.mjs | 54 ++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs index bd73e3797..5087fde4d 100644 --- a/test/unit/egress-stop-proof.test.mjs +++ b/test/unit/egress-stop-proof.test.mjs @@ -164,6 +164,21 @@ test('process inspection permission failure retains its receipt and ownership', assert.equal(fs.existsSync(path.join(f.dir, 'egress.owner.json')), true) }) +test('a failed signal retains both PID and execution receipt', { skip: process.platform !== 'linux' }, async (t) => { + const f = fixture(t) + const started = await f.start() + const kill = process.kill + t.mock.method(process, 'kill', (pid, signal) => { + if (pid === started.pid && signal === 'SIGTERM') throw Object.assign(new Error('signal denied'), { code: 'EPERM' }) + return kill(pid, signal) + }) + const result = await egress.stopEgressProcessConfirmed(f.root, 'synthetic') + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal(fs.existsSync(path.join(f.dir, 'egress.pid')), true) + assert.equal(fs.existsSync(path.join(f.dir, 'egress.owner.json')), true) +}) + test('Docker network inspection failure cannot be treated as absent during proxy teardown', async (t) => { const f = fixture(t) const result = await stopProxyEgress(f.root, 'synthetic', { @@ -174,6 +189,31 @@ test('Docker network inspection failure cannot be treated as absent during proxy assert.equal(result.vmOperationUncertain, true) }) +for (const mode of ['normal', 'noop', 'replacement']) { + test(`proxy network teardown verifies immutable ID and observed absence (${mode})`, async (t) => { + const f = fixture(t) + let present = true + const removed = [] + const result = await stopProxyEgress(f.root, 'synthetic', { + ...noDocker, + runDocker: (args) => { + if (args[1] === 'rm') { + removed.push(args.at(-1)) + if (mode !== 'noop') present = false + return { ok: true, stdout: '' } + } + if (present) return { ok: true, stdout: '192.0.2.0/24|192.0.2.1|original-network-id' } + if (mode === 'replacement' && args.at(-1) !== 'original-network-id') + return { ok: true, stdout: '192.0.2.0/24|192.0.2.1|replacement-network-id' } + return { ok: false, stderr: 'Error: No such network: synthetic' } + }, + }) + assert.deepEqual(removed, ['original-network-id']) + assert.equal(result.ok, mode === 'normal') + if (mode !== 'normal') assert.equal(result.vmOperationUncertain, true) + }) +} + function remoteFixture({ noop = false, permission = false } = {}) { const source = fs.readFileSync(new URL('../../src/lib/cluster/remote-slot.mjs', import.meta.url), 'utf8') const body = source @@ -215,3 +255,17 @@ test('remote egress stop addresses immutable IDs and permission failures remain const unknown = await remoteFixture({ permission: true }).stop({ id: 'vm-synthetic', proxy: {} }, '/synthetic', false) assert.equal(unknown.vmOperationUncertain, true) }) + +test('a Kubernetes Codex exit-policy stop cannot infer another Pod kernel is absent', async () => { + const source = fs.readFileSync(new URL('../../src/lib/vm/slot-host.mjs', import.meta.url), 'utf8') + const body = source.slice(source.indexOf(' setProxyEgressEnabled: async'), source.indexOf(' localExitProxyUrl:')) + const stop = vmContext.runInNewContext(`({${body}}).setProxyEgressEnabled`, { + isCodexVm: () => true, + stopCodexKernel: () => null, + isKubernetesMode: () => true, + finishCodexRecycle: async () => ({ exited: true }), + }) + const result = await stop({ id: 'vm-synthetic' }, '/synthetic', false) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) +}) From 479c012d3ac09f95164ef66c05b4ccafca74d3c9 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:30:34 +0800 Subject: [PATCH 236/473] fix: verify egress execution identity and termination before release --- src/lib/cluster/remote-slot.mjs | 20 ++- src/lib/vm/egress.mjs | 264 +++++++++++++++++++++++++++++--- src/lib/vm/slot-host.mjs | 34 ++-- 3 files changed, 271 insertions(+), 47 deletions(-) diff --git a/src/lib/cluster/remote-slot.mjs b/src/lib/cluster/remote-slot.mjs index 79099a44f..013402589 100644 --- a/src/lib/cluster/remote-slot.mjs +++ b/src/lib/cluster/remote-slot.mjs @@ -532,11 +532,25 @@ export async function setRemoteProxyEgressEnabled(vm, projectRoot, enabled) { } for (const name of new Set([current, legacy])) { const existing = await inspectContainerOrNull(session.docker, name) - if (existing?.State?.Running) await containerAction(session.docker, name, 'stop') + if (!existing) continue + if (!existing.Id) return { ok: false, code: 'egress_container_unknown', vmOperationUncertain: true } + if (!remoteContainerStopped(existing)) { + try { + await containerAction(session.docker, existing.Id, 'stop') + } catch {} + } + const remaining = await inspectContainerOrNull(session.docker, existing.Id) + const named = await inspectContainerOrNull(session.docker, name) + if ( + (remaining && !remoteContainerStopped(remaining)) || + (named && (named.Id !== existing.Id || !remoteContainerStopped(named))) + ) { + return { ok: false, code: 'egress_stop_unconfirmed', vmOperationUncertain: true } + } } - return { ok: true } + return { ok: true, ended: true } } catch (err) { - return failure(err) + return { ...failure(err), vmOperationUncertain: true } } } diff --git a/src/lib/vm/egress.mjs b/src/lib/vm/egress.mjs index 546b15a31..f0473c5f4 100644 --- a/src/lib/vm/egress.mjs +++ b/src/lib/vm/egress.mjs @@ -12,6 +12,8 @@ import { getDb, isDbOpen } from '../db/database.mjs' import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { socksProxyUrl } from './socks-address.mjs' import { assertProxyAllowed, proxyBlockedReason } from './proxy-policy.mjs' +import { atomicWriteJson } from './vm-file.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' export const EGRESS_BIN = process.env.KIN_EGRESS_BIN || '/opt/kin-gateway/bin/kin-egress' export const LOCAL_EGRESS_ID = 'px-local' @@ -317,14 +319,27 @@ export function readEgressListenTcp(projectRoot, proxyId) { return '' } } -export function inspectEgressNetwork(proxyId, run = docker) { - const name = networkName(proxyId) +export function inspectEgressNetwork(proxyId, run = docker, { strict = false, target = null } = {}) { + const name = target || networkName(proxyId) if (!name) return null - const r = run(['network', 'inspect', '-f', '{{range .IPAM.Config}}{{.Subnet}}|{{.Gateway}}{{end}}', name]) - if (!r.ok) return null - const [subnet, gateway] = String(r.stdout || '').split('|') + const r = run(['network', 'inspect', '-f', '{{range .IPAM.Config}}{{.Subnet}}|{{.Gateway}}{{end}}|{{.Id}}', name]) + if (!r.ok) { + if ( + strict && + !/^(?:Error(?: response from daemon)?: )?(?:No such network: [^\r\n]+|network [^\r\n]+ not found)$/i.test( + r.stderr || '', + ) + ) { + throw Object.assign(new Error(r.stderr || 'Docker network inspection failed'), { code: 'egress_network_unknown' }) + } + return null + } + const [subnet, gateway, id] = String(r.stdout || '').split('|') + if (strict && !id?.trim()) + throw Object.assign(new Error('Docker network identity unavailable'), { code: 'egress_network_unknown' }) return { name, + id: id?.trim() || null, network: name, subnet: String(subnet || '').trim(), gateway: String(gateway || '').trim() || gatewayFromSubnet(String(subnet || '').trim()), @@ -387,9 +402,12 @@ function applyIptables(plan, runIptables = iptables) { return { ok: true } } -function removeIptables(plan, runIptables = iptables) { +function removeIptables(plan, runIptables = iptables, { strict = false } = {}) { for (const args of plan.del) { - runIptables(args) + const result = runIptables(args) + if (strict && !result.ok && !/Bad rule|No chain\/target\/match by that name/i.test(result.stderr || '')) { + return { ok: false, error: result.stderr || 'iptables removal unconfirmed' } + } } return { ok: true } } @@ -477,6 +495,12 @@ export async function startEgressProcess({ const listenTcp = `${listenHost}:${tcpPort}` const listenDns = `${listenHost}:${dnsPort}` const existing = readPid(pidFile) + let owner + try { + owner = isKubernetesMode() ? egressOwnerSnapshot(projectRoot, proxyId) : { legacy: true, pid: existing } + } catch (error) { + return egressOwnerUnknown(error, existing) + } const configMatches = ownsEgressProcess(existing, cfgPath) if (configMatches) { try { @@ -492,9 +516,17 @@ export async function startEgressProcess({ ) { return { ok: true, pid: existing, reused: true, configPath: cfgPath } } - } catch {} + } catch (error) { + if (isKubernetesMode()) return egressOwnerUnknown(error, existing) + } } - const superseded = [...new Set([...(configMatches ? [existing] : []), ...staleEgressPids(cfgPath)])] + if (!owner.legacy && (owner.receipt || isKubernetesMode())) { + const stopped = await stopEgressProcessConfirmed(projectRoot, proxyId) + if (!stopped.ok) return stopped + } + const superseded = owner.legacy + ? [...new Set([...(configMatches ? [existing] : []), ...staleEgressPids(cfgPath)])] + : [] for (const pid of superseded) { const stopped = await stopOwnedEgress(pid) if (!stopped.ok) return { ok: false, error: stopped.error, pid } @@ -522,7 +554,26 @@ export async function startEgressProcess({ fs.writeFileSync(cfgPath, JSON.stringify(cfg, null, 2) + '\n', { mode: 0o600 }) const spawned = await spawnDetachedEgress(bin, cfgPath) if (!spawned.ok) return spawned - fs.writeFileSync(pidFile, String(spawned.pid) + '\n', { mode: 0o600 }) + try { + fs.writeFileSync(pidFile, String(spawned.pid) + '\n', { mode: 0o600 }) + if (process.platform === 'linux') { + const identity = readEgressIdentity(spawned.pid, cfgPath) + if (!identity || identity.state === 'Z' || identity.other) + throw new Error('Egress execution identity was not recorded') + atomicWriteJson( + path.join(dir, 'egress.owner.json'), + { + ...egressExecutionDomain(), + proxy_id: proxyId, + pid: spawned.pid, + start_ticks: identity.start_ticks, + }, + { mode: 0o600 }, + ) + } else if (isKubernetesMode()) throw new Error('Egress execution identity is unavailable on this platform') + } catch (error) { + return { ...egressOwnerUnknown(error, spawned.pid), code: 'egress_receipt_persist_failed' } + } return { ok: true, pid: spawned.pid, reused: false, configPath: cfgPath, listen_tcp: listenTcp } } @@ -550,6 +601,161 @@ export function stopEgressProcess(projectRoot, proxyId) { return { ok: true, pending } } +function egressOwnerUnknown(error, pid = null) { + return { + ok: false, + code: 'egress_owner_unreachable', + error: String(error?.message || error), + pid, + vmOperationUncertain: true, + } +} + +function egressExecutionDomain() { + if (process.platform !== 'linux') throw new Error('Egress execution domain is not observable') + return { + boot_id: fs.readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim(), + pid_namespace: fs.readlinkSync('/proc/self/ns/pid'), + } +} + +function readEgressIdentity(pid, configPath) { + let stat + try { + stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8') + } catch (error) { + if (error.code === 'ENOENT' || error.code === 'ESRCH') return null + throw error + } + const fields = stat + .slice(stat.lastIndexOf(')') + 1) + .trim() + .split(/\s+/) + if (!/^\d+$/.test(fields[19] || '')) throw new Error('Egress process start identity is unavailable') + const identity = { pid, state: fields[0], start_ticks: fields[19] } + if (identity.state === 'Z') return identity + try { + const args = fs.readFileSync(`/proc/${pid}/cmdline`, 'utf8').split('\0') + const index = args.indexOf('-config') + const argument = args[index + 1] + if ( + index <= 0 || + !argument || + !args.some((arg) => path.basename(arg.replace(/ \(deleted\)$/, '')).startsWith('kin-egress')) + ) { + return { ...identity, other: true } + } + const actual = path.isAbsolute(argument) + ? path.resolve(argument) + : path.resolve(fs.readlinkSync(`/proc/${pid}/cwd`), argument) + if (actual !== path.resolve(configPath)) return { ...identity, other: true } + processExe(pid) // A permission error is unknown, never absence. + return identity + } catch (error) { + if (error.code === 'ENOENT' || error.code === 'ESRCH') { + try { + fs.statSync(`/proc/${pid}`) + } catch (absent) { + if (absent.code === 'ENOENT' || absent.code === 'ESRCH') return null + } + } + throw error + } +} + +function egressOwnerSnapshot(projectRoot, proxyId) { + const dir = egressRunDir(projectRoot, proxyId) + const config = path.join(dir, 'egress.json') + let pid = null + try { + pid = Number(fs.readFileSync(path.join(dir, 'egress.pid'), 'utf8').trim()) + if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error('Invalid egress PID record') + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + let receipt = null + try { + receipt = JSON.parse(fs.readFileSync(path.join(dir, 'egress.owner.json'), 'utf8')) + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + if (!receipt) { + if (!isKubernetesMode()) return { legacy: true, pid } + if (pid || egressPidsForProxyConfig(projectRoot, proxyId).length) + throw new Error('Egress PID has no execution identity receipt') + return { receipt: null, pid: null } + } + const domain = egressExecutionDomain() + if ( + receipt.proxy_id !== proxyId || + !Number.isSafeInteger(receipt.pid) || + receipt.pid <= 0 || + typeof receipt.start_ticks !== 'string' || + !/^\d+$/.test(receipt.start_ticks) || + receipt.boot_id !== domain.boot_id || + receipt.pid_namespace !== domain.pid_namespace || + (pid && pid !== receipt.pid) + ) + throw new Error('Egress execution owner is not observable in this PID namespace') + const identity = readEgressIdentity(receipt.pid, config) + if (identity && identity.start_ticks !== receipt.start_ticks && !identity.other && identity.state !== 'Z') { + throw new Error('An unreceipted egress helper reused the recorded PID') + } + if (identity?.start_ticks === receipt.start_ticks && identity.other) + throw new Error('Egress process execution changed') + for (const other of egressPidsForProxyConfig(projectRoot, proxyId)) { + if (other !== receipt.pid) throw new Error('An additional egress process has no execution identity receipt') + } + return { receipt, identity, pid: receipt.pid } +} + +/** Confirm the recorded execution ended; never signal a reused PID or another Pod's namespace. */ +export async function stopEgressProcessConfirmed( + projectRoot, + proxyId, + { termTimeoutMs = EGRESS_STOP_TIMEOUT_MS, killGraceMs = EGRESS_KILL_GRACE_MS } = {}, +) { + try { + const owner = egressOwnerSnapshot(projectRoot, proxyId) + if (owner.legacy) { + const drained = await drainEgressForProxyConfig(projectRoot, proxyId) + if (!drained.ok) return { ...drained, vmOperationUncertain: true } + return { ...stopEgressProcess(projectRoot, proxyId), ended: false, legacy: true } + } + const config = path.join(egressRunDir(projectRoot, proxyId), 'egress.json') + const ended = () => { + if (!owner.receipt) return true + const current = readEgressIdentity(owner.pid, config) + if (!current || current.state === 'Z') return true + if (current.start_ticks !== owner.receipt.start_ticks) { + if (!current.other) throw new Error('An unreceipted egress helper reused the recorded PID') + return true + } + if (current.other) throw new Error('Egress process execution changed') + return false + } + for (const [signal, timeout] of [ + ['SIGTERM', termTimeoutMs], + ['SIGKILL', killGraceMs], + ]) { + if (ended()) break + try { + process.kill(owner.pid, signal) + } catch (error) { + if (error.code !== 'ESRCH') throw error + } + const deadline = Date.now() + timeout + while (!ended() && Date.now() < deadline) await delay(EGRESS_POLL_MS) + } + if (!ended()) throw new Error('Egress process termination was not confirmed') + for (const file of ['egress.pid', 'egress.owner.json']) + fs.rmSync(path.join(egressRunDir(projectRoot, proxyId), file), { force: true }) + return { ok: true, ended: true } + } catch (error) { + return egressOwnerUnknown(error) + } +} + async function stopOwnedEgress(pid) { try { process.kill(pid, 'SIGTERM') @@ -856,18 +1062,34 @@ export async function startProxyEgress( export function stopProxyEgress(projectRoot, proxyId, { subnet, runDocker = docker, runIptables = iptables } = {}) { // A node's local row owns nothing here; its bridge name is the control plane's px-local one. if (localEgressNodeId(proxyId)) return { ok: true, skipped: 'node_local' } - const net = inspectEgressNetwork(proxyId, runDocker) - const plan = iptablesPlan({ - chain: chainName(proxyId), - bridge: bridgeName(proxyId), - subnet: subnet || net?.subnet || '0.0.0.0/0', - tcpPort: portsForProxy(proxyId).tcp, - dnsPort: portsForProxy(proxyId).dns, + return runExclusiveProxyEgress(projectRoot, proxyId, async () => { + try { + const net = inspectEgressNetwork(proxyId, runDocker, { strict: true }) + const stopped = await stopEgressProcessConfirmed(projectRoot, proxyId) + if (!stopped.ok) return stopped + const plan = iptablesPlan({ + chain: chainName(proxyId), + bridge: bridgeName(proxyId), + subnet: subnet || net?.subnet || '0.0.0.0/0', + tcpPort: portsForProxy(proxyId).tcp, + dnsPort: portsForProxy(proxyId).dns, + }) + const rules = removeIptables(plan, runIptables, { strict: true }) + if (!rules.ok) return { ...rules, vmOperationUncertain: true } + if (net) { + runDocker(['network', 'rm', net.id]) + if ( + inspectEgressNetwork(proxyId, runDocker, { strict: true, target: net.id }) || + inspectEgressNetwork(proxyId, runDocker, { strict: true }) + ) { + throw new Error('Egress network removal was not confirmed') + } + } + return { ok: true, ended: stopped.ended, ...(stopped.legacy ? { legacy: true } : {}) } + } catch (error) { + return egressOwnerUnknown(error) + } }) - removeIptables(plan, runIptables) - stopEgressProcess(projectRoot, proxyId) - if (net?.name) runDocker(['network', 'rm', net.name]) - return { ok: true } } export function slotNetworkForVm(vm, _env = process.env) { diff --git a/src/lib/vm/slot-host.mjs b/src/lib/vm/slot-host.mjs index c49f8328a..fdb9d9f27 100644 --- a/src/lib/vm/slot-host.mjs +++ b/src/lib/vm/slot-host.mjs @@ -85,15 +85,15 @@ import { destroyVmRuntime, officialCcUidGid, reloadSlotWorker, startVmRuntime, s import { boundProxyUrl, ensureProxyEgress, - inspectEgressProcess, isLocalEgressProxy, localEgressProxyUrl, - stopEgressProcess, + stopEgressProcessConfirmed, } from './egress.mjs' import { isCodexVm } from './vm-kind.mjs' import { nodeEgressProxyUrl } from '../cluster/node-egress-socks.mjs' import { isKvmRuntime } from './runtime-kind.mjs' -import { stopCodexKernel } from '../transport/codex-kernel-supervisor.mjs' +import { stopCodexKernel, finishCodexRecycle } from '../transport/codex-kernel-supervisor.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' const noop = async () => ({ skipped: true }) @@ -118,28 +118,16 @@ const LOCAL_HOST = Object.freeze({ if (isCodexVm(vm)) { if (enabled) return { ok: true } const child = stopCodexKernel(vm.id) - if (!child || child.exitCode !== null || child.signalCode !== null) return { ok: true } - return new Promise((resolve) => { - const exited = () => { - clearTimeout(timeout) - resolve({ ok: true }) - } - const timeout = setTimeout(() => { - child.off('exit', exited) - resolve({ ok: false, error: 'codex_stop_timeout' }) - }, 2000) - child.once('exit', exited) - }) + if (!child) + return isKubernetesMode() + ? { ok: false, code: 'codex_kernel_owner_unknown', vmOperationUncertain: true } + : { ok: true, notStarted: true } + if (!(await finishCodexRecycle(child)).exited) + return { ok: false, code: 'codex_kernel_exit_unconfirmed', vmOperationUncertain: true } + return { ok: true, ended: true } } if (enabled) return await ensureProxyEgress(projectRoot, vm.proxy) - const stopped = stopEgressProcess(projectRoot, vm.proxy?.id) - if (!stopped.ok) return stopped - const deadline = Date.now() + 2000 - while (inspectEgressProcess(projectRoot, vm.proxy?.id).ok) { - if (Date.now() >= deadline) return { ok: false, error: 'egress_stop_timeout' } - await new Promise((resolve) => setTimeout(resolve, 20)) - } - return stopEgressProcess(projectRoot, vm.proxy?.id) + return await stopEgressProcessConfirmed(projectRoot, vm.proxy?.id) }, localExitProxyUrl: (vm) => (isCodexVm(vm) ? localEgressProxyUrl() : ''), // The container bind-mounts vms// directly: every local write is already published. From e3a0c235a2d9c73e5d1d6f55cd41cc77e8fa964c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:33:54 +0800 Subject: [PATCH 237/473] test: reproduce per-controller watchdog budget and fault divergence --- test/unit/shared-watchdog-state.test.mjs | 78 ++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 test/unit/shared-watchdog-state.test.mjs diff --git a/test/unit/shared-watchdog-state.test.mjs b/test/unit/shared-watchdog-state.test.mjs new file mode 100644 index 000000000..368cbe0b3 --- /dev/null +++ b/test/unit/shared-watchdog-state.test.mjs @@ -0,0 +1,78 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { createKernelWatchdog } from '../../src/lib/transport/kernel-watchdog.mjs' +import { kernelFaults } from '../../src/lib/transport/rust-kernel-client.mjs' + +const vmId = 'vm-shared-budget' +const key = 'kernel_watchdog_state:kin-shared-budget' +function state(patch = {}) { + return { version: 1, attempts: [], nextAt: null, telemetryDue: 0, fault: null, ...patch } +} +function reserve(id = vmId) { + return new ExecutionAdmissionsRepo(getDb()).tryReserve({ + accountId: 'budget-account', + vmId: id, + accountLimit: 2, + vmLimit: 2, + fableLimit: 0, + rpmLimit: 0, + }) +} + +test('a new watchdog controller uses the persisted restart budget', async (t) => { + const db = getDb() + const repo = new SettingsRepo(db) + const now = (await db.prepare('SELECT extract(epoch FROM clock_timestamp()) * 1000 AS now').get()).now + await repo.set(key, state({ attempts: [now - 1000], nextAt: now - 1 })) + t.after(() => kernelFaults.delete(vmId)) + let restarted = 0 + const watchdog = createKernelWatchdog({ + listTargets: () => [{ id: vmId, status: 'running' }], + config: { restart_backoff_sec: [10], restart_window_sec: 3600 }, + health: async () => ({ ok: true, healthy: false, unhealthy_reason: 'synthetic failure' }), + observeRuntime: async () => ({ state: 'running' }), + ensure: async () => { + restarted++ + return { ok: true } + }, + sharedTasks: new SharedBackgroundTasks(db), + operations: new SharedVmOperations(db), + }) + await watchdog.tick() + assert.equal(restarted, 0) + assert.match((await repo.get(key)).fault, /after 1 container restarts/) + assert.equal((await reserve()).code, 'kernel_faulted') +}) + +test('durable watchdog fault rejects an old candidate with an empty local fault cache', async () => { + await new SettingsRepo(getDb()).set(key, state({ fault: 'synthetic fault' })) + kernelFaults.delete(vmId) + assert.equal((await reserve()).code, 'kernel_faulted') + assert.equal((await reserve('shared-budget')).code, 'kernel_faulted') +}) + +test('corrupt durable watchdog state fails closed before inference admission', async () => { + await new SettingsRepo(getDb()).set(key, { version: 1, attempts: 'invalid' }) + await assert.rejects(reserve(), { code: 'watchdog_state_invalid' }) +}) + +test('operator watchdog reset persists before another controller admits a request', async () => { + const db = getDb() + await new SettingsRepo(db).set(key, state({ fault: 'synthetic fault', attempts: [Date.now()] })) + const watchdog = createKernelWatchdog({ + sharedTasks: new SharedBackgroundTasks(db), + operations: new SharedVmOperations(db), + }) + const result = await watchdog.reset(vmId) + assert.equal(result.faulted, true) + assert.equal((await new SettingsRepo(db).get(key)).fault, null) + assert.deepEqual((await new SettingsRepo(db).get(key)).attempts, []) + const next = await reserve() + assert.equal(next.ok, true) + await next.release() +}) From 988f1dab7aa7a54878c0e346d68931df8c1a58c8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:34:09 +0800 Subject: [PATCH 238/473] feat: share notification state and durable send attempts across pods --- src/lib/admin/notify.mjs | 193 ++++++++++++++++++++++--------- test/unit/notify-shared.test.mjs | 96 +++++++++++++-- 2 files changed, 223 insertions(+), 66 deletions(-) diff --git a/src/lib/admin/notify.mjs b/src/lib/admin/notify.mjs index 6acccbab1..73440efee 100644 --- a/src/lib/admin/notify.mjs +++ b/src/lib/admin/notify.mjs @@ -6,9 +6,14 @@ */ import net from 'node:net' import tls from 'node:tls' +import { randomUUID } from 'node:crypto' import { CREDENTIAL_REFRESH_FAIL } from '../pool/availability.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { RoutingConfigRepo } from '../db/repos/routing-config-repo.mjs' export const SECRET_KEEP = '__KEEP__' +const NOTIFY_STATE_KEY = 'pool_notify_state' export const DEFAULT_NOTIFY = Object.freeze({ enabled: false, interval_sec: 60, @@ -533,7 +538,7 @@ function smtpExpectOk(reply, allowed) { .split(/\r?\n/) .filter(Boolean) .pop() || `smtp_${reply.code}` - throw new Error(hint.slice(0, 180)) + throw Object.assign(new Error(hint.slice(0, 180)), { notificationAttemptEnded: true }) } } @@ -705,7 +710,7 @@ export async function sendTelegramNotify(cfg, message, opts = {}) { }), signal: opts.signal, }) - const raw = await res.text().catch(() => '') + const raw = await res.text() let data = {} try { data = raw ? JSON.parse(raw) : {} @@ -713,7 +718,12 @@ export async function sendTelegramNotify(cfg, message, opts = {}) { data = { description: raw } } if (!res.ok || data.ok === false) { - throw new Error(redactTelegramError(data.description || `telegram_http_${res.status}`, tg.bot_token)) + throw Object.assign( + new Error(redactTelegramError(data.description || `telegram_http_${res.status}`, tg.bot_token)), + { + notificationAttemptEnded: true, + }, + ) } return { ok: true, channel: 'telegram', chat_id: tg.chat_id } } @@ -771,7 +781,7 @@ export async function sendWecomNotify(cfg, message, opts = {}) { } catch (err) { throw new Error(redactWecomError(err?.message || err, cfg)) } - const raw = await res.text().catch(() => '') + const raw = await res.text() let data = {} try { data = raw ? JSON.parse(raw) : {} @@ -781,7 +791,7 @@ export async function sendWecomNotify(cfg, message, opts = {}) { if (!res.ok || (data.errcode != null && Number(data.errcode) !== 0)) { const reason = data.errcode != null ? `wecom_${data.errcode} ${data.errmsg || ''}`.trim() : `wecom_http_${res.status}` - throw new Error(redactWecomError(reason, cfg)) + throw Object.assign(new Error(redactWecomError(reason, cfg)), { notificationAttemptEnded: true }) } return { ok: true, channel: 'wecom' } } @@ -794,25 +804,41 @@ export async function dispatchNotify(cfg, message, opts = {}) { try { results.push(await sendEmailNotify(cfg, message, opts)) } catch (err) { - results.push({ ok: false, channel: 'email', error: String(err?.message || err).slice(0, 180) }) + results.push({ + ok: false, + uncertain: err?.notificationAttemptEnded !== true, + channel: 'email', + error: String(err?.message || err).slice(0, 180), + }) } } if (ready.telegram) { try { results.push(await sendTelegramNotify(cfg, message, opts)) } catch (err) { - results.push({ ok: false, channel: 'telegram', error: String(err?.message || err).slice(0, 180) }) + results.push({ + ok: false, + uncertain: err?.notificationAttemptEnded !== true, + channel: 'telegram', + error: String(err?.message || err).slice(0, 180), + }) } } if (ready.wecom) { try { results.push(await sendWecomNotify(cfg, message, opts)) } catch (err) { - results.push({ ok: false, channel: 'wecom', error: String(err?.message || err).slice(0, 180) }) + results.push({ + ok: false, + uncertain: err?.notificationAttemptEnded !== true, + channel: 'wecom', + error: String(err?.message || err).slice(0, 180), + }) } } return { ok: results.some((row) => row.ok), + uncertain: results.some((row) => row.uncertain), results, } } @@ -841,6 +867,9 @@ export async function sendNotifyTest(cfg, channel, extra = {}) { } export function createNotifyMonitor(opts = {}) { + const tasks = backgroundTasksFor(opts.sharedTasks) + const settings = tasks ? new SettingsRepo(tasks.db) : null + const routing = tasks ? new RoutingConfigRepo(tasks.db) : null let config = normalizeNotifyConfig(opts.config) let snapshot = null let lastEvents = [] @@ -851,6 +880,22 @@ export function createNotifyMonitor(opts = {}) { const lastSent = new Map() const nowFn = opts.now || (() => Date.now()) + const restoreState = (state = {}) => { + snapshot = state.snapshot || null + lastEvents = state.last_events || [] + lastError = state.last_error || null + lastRunAt = state.last_run_at || 0 + lastSent.clear() + for (const [key, value] of Object.entries(state.last_sent || {})) lastSent.set(key, value) + } + const persistedState = () => ({ + snapshot, + last_events: lastEvents, + last_error: lastError, + last_run_at: lastRunAt, + last_sent: Object.fromEntries(lastSent), + }) + const getConfig = () => config const getSnapshot = () => snapshot const getStatus = () => ({ @@ -868,55 +913,91 @@ export function createNotifyMonitor(opts = {}) { return now - prev >= wait * 1000 } - const runOnce = async ({ send = true, force = false, digest = false } = {}) => { - if (inflight) return inflight - inflight = (async () => { - const started = nowFn() - lastRunAt = started - let deliveryError = null - let delivered = false - const next = typeof opts.snapshot === 'function' ? await opts.snapshot() : null - const prev = snapshot - snapshot = next - const events = detectPoolNotifyEvents(prev, next, config) - if (digest || (config.events.digest && (!prev || allowed('digest', started)))) { - events.push({ type: 'digest', title: EVENT_TITLES.digest }) - } - lastEvents = events - const dispatched = [] - const canSend = send && config.enabled && (typeof opts.send === 'function' || notifyChannelsReady(config).any) - if (canSend) { - for (const event of events) { - if (!force && !digest && !allowed(event.type, started)) continue - const message = formatNotifyMessage(event, next, config, opts.baseUrl) - const sendFn = opts.send || ((msg) => dispatchNotify(config, msg)) - try { - const result = await sendFn(message, event) - lastSent.set(event.type, started) - const failedChannels = Array.isArray(result?.results) - ? result.results.filter((row) => row?.ok === false) - : [] - if (result?.ok === false || failedChannels.length) { - deliveryError = String( - failedChannels.map((row) => row.error || 'notify_delivery_failed').join('; ') || - result?.error || - 'notify_delivery_failed', - ).slice(0, 180) - } else { - delivered = true - } - dispatched.push({ type: event.type, ok: result?.ok !== false, result }) - } catch (err) { - deliveryError = String(err?.message || err).slice(0, 180) - dispatched.push({ type: event.type, ok: false, error: deliveryError }) + const check = async ({ send = true, force = false, digest = false } = {}, { started = nowFn(), persist } = {}) => { + const checkConfig = config + lastRunAt = started + let deliveryError = null + let delivered = false + let uncertain = false + const next = typeof opts.snapshot === 'function' ? await opts.snapshot() : null + const prev = snapshot + snapshot = next + const events = detectPoolNotifyEvents(prev, next, checkConfig) + if (digest || (checkConfig.events.digest && (!prev || allowed('digest', started)))) { + events.push({ type: 'digest', title: EVENT_TITLES.digest }) + } + lastEvents = events + // Commit the observation before contacting a provider. The durable task + // claim keeps a crash or unknown send from being replayed by another Pod. + if (persist) await persist() + const dispatched = [] + const canSend = + send && checkConfig.enabled && (typeof opts.send === 'function' || notifyChannelsReady(checkConfig).any) + if (canSend) { + for (const event of events) { + if (!force && !digest && !allowed(event.type, started)) continue + const message = formatNotifyMessage(event, next, checkConfig, opts.baseUrl) + const sendFn = opts.send || ((msg) => dispatchNotify(checkConfig, msg)) + try { + const result = await sendFn(message, event) + uncertain ||= result?.uncertain === true + lastSent.set(event.type, started) + const failedChannels = Array.isArray(result?.results) ? result.results.filter((row) => row?.ok === false) : [] + if (result?.ok === false || failedChannels.length) { + deliveryError = String( + failedChannels.map((row) => row.error || 'notify_delivery_failed').join('; ') || + result?.error || + 'notify_delivery_failed', + ).slice(0, 180) + } else { + delivered = true } + dispatched.push({ type: event.type, ok: result?.ok !== false, result }) + } catch (err) { + uncertain = true + deliveryError = String(err?.message || err).slice(0, 180) + dispatched.push({ type: event.type, ok: false, error: deliveryError }) } + if (tasks && uncertain) break + } + } + // A quiet/cooldown-suppressed check is not proof of delivery recovery. + // Any failure in this batch takes precedence over successful events. + if (deliveryError !== null) lastError = deliveryError + else if (delivered) lastError = null + if (persist) await persist() + return { snapshot: next, events, dispatched, prev, ...(tasks ? { uncertain } : {}) } + } + + const sharedCheck = async (options, scheduled) => { + const plan = scheduled + ? { taskKey: 'notify-monitor', intervalMs: config.interval_sec * 1000 } + : { taskKey: 'notify-monitor', executionId: randomUUID() } + const result = await tasks.run(plan, async () => { + const current = await routing.get() + if (!current) + throw Object.assign(new Error('shared routing configuration missing'), { code: 'routing_config_unavailable' }) + config = normalizeNotifyConfig(current.config.notify) + const previous = await settings.get(NOTIFY_STATE_KEY, {}) + restoreState(previous) + try { + const value = await check(options, { + started: await tasks.now(), + persist: () => settings.set(NOTIFY_STATE_KEY, persistedState()), + }) + return { terminal: !value.uncertain, value } + } catch (error) { + restoreState(previous) + throw error } - // A quiet/cooldown-suppressed check is not proof of delivery recovery. - // Any failure in this batch takes precedence over successful events. - if (deliveryError !== null) lastError = deliveryError - else if (delivered) lastError = null - return { snapshot: next, events, dispatched, prev } + }) + return result.ran ? result.value : { snapshot, events: [], dispatched: [], prev: snapshot, code: result.code } + } + + const runOnce = async (options = {}, scheduled = false) => { + if (inflight) return inflight + inflight = (async () => { + return tasks ? await sharedCheck(options, scheduled) : await check(options) })().finally(() => { inflight = null }) @@ -934,12 +1015,12 @@ export function createNotifyMonitor(opts = {}) { stop() if (!config.enabled) return { started: false, reason: 'disabled' } timer = setInterval(() => { - runOnce().catch(() => {}) + runOnce({}, true).catch(() => {}) }, config.interval_sec * 1000) if (typeof timer.unref === 'function') timer.unref() if (immediate && config.run_on_start) { queueMicrotask(() => { - runOnce().catch(() => {}) + runOnce({}, true).catch(() => {}) }) } return { started: true, interval_sec: config.interval_sec, run_on_start: config.run_on_start } diff --git a/test/unit/notify-shared.test.mjs b/test/unit/notify-shared.test.mjs index 55200eb7f..7d3b839c5 100644 --- a/test/unit/notify-shared.test.mjs +++ b/test/unit/notify-shared.test.mjs @@ -1,9 +1,11 @@ import test from 'node:test' import assert from 'node:assert/strict' -import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import pg from 'pg' +import { createDatabase, getDb, getActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' import { createNotifyMonitor, dispatchNotify, summarizePoolAvailability } from '../../src/lib/admin/notify.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' const config = { enabled: true, @@ -114,16 +116,90 @@ test('a truncated provider response cannot confirm the notification attempt ende ? { enabled: true, bot_token: 'synthetic-token', chat_id: 'synthetic-chat' } : { enabled: true, webhook: 'synthetic-key-1234' }, } - const result = await dispatchNotify(cfg, { text: 'synthetic notification' }, { - fetch: async () => ({ - ok: true, - status: 200, - text: async () => { - throw new Error('synthetic truncated provider body') - }, - }), - }) + const result = await dispatchNotify( + cfg, + { text: 'synthetic notification' }, + { + fetch: async () => ({ + ok: true, + status: 200, + text: async () => { + throw new Error('synthetic truncated provider body') + }, + }), + }, + ) assert.equal(result.ok, false, channel) assert.equal(result.uncertain, true, channel) } }) + +test('a complete provider rejection ends its attempt and permits a later explicit check', async (t) => { + const { db, monitors } = await fixture(t) + const [first, peer] = monitors(() => + dispatchNotify( + { telegram: { enabled: true, bot_token: 'synthetic', chat_id: 'synthetic' } }, + { text: 'synthetic' }, + { fetch: async () => ({ ok: false, status: 400, text: async () => '{"ok":false}' }) }, + ), + ) + await first.runOnce({ digest: true }) + await peer.runOnce({ digest: true }) + const claims = (await db.query('SELECT finished_at, unknown_at FROM background_task_runs')).rows + assert.equal(claims.length, 2) + assert.ok(claims.every((claim) => claim.finished_at && !claim.unknown_at)) + assert.ok(peer.getStatus().last_error) +}) + +test('a failure committing completion after a send retains the claim and prevents another send', async (t) => { + const { db, monitors } = await fixture(t, { nextPlan: true }) + await db.exec(`CREATE FUNCTION refuse_notify_completion() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.finished_at IS NOT NULL THEN RAISE EXCEPTION 'synthetic completion failure' USING ERRCODE='P6484'; END IF; + RETURN NEW; END $$; + CREATE TRIGGER refuse_notify_completion BEFORE UPDATE ON background_task_runs + FOR EACH ROW EXECUTE FUNCTION refuse_notify_completion()`) + let sends = 0 + const [first, peer] = monitors(async () => { + sends++ + return { ok: true } + }) + await assert.rejects(first.runOnce(), { code: 'P6484' }) + await peer.runOnce() + assert.equal(sends, 1) + const claim = (await db.query('SELECT finished_at FROM background_task_runs')).rows[0] + assert.equal(claim.finished_at, null) +}) + +test('shared checks run inside fresh durable activities and finish before global maintenance', async (t) => { + const { db, peer, monitors } = await fixture(t) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + let schema + t.after(async () => { + await getActivityCoordinator(db)?.stop() + await getActivityCoordinator(peer)?.stop() + await cleanup.connect() + try { + if (schema) await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + let sends = 0 + const [first, second] = monitors(async () => { + assert.equal(getActivityCoordinator(db).assertActivity().kind, 'background') + sends++ + return { ok: true } + }) + await withDatabaseActivities([db, peer], async () => { + schema = getActivityCoordinator(db).schema + await first.runOnce() + await second.runOnce() + }) + assert.equal(sends, 1) + const operation = await getActivityCoordinator(db).beginMaintenance({ kind: 'backup' }) + try { + await operation.resume() + } finally { + await operation.close() + } +}) From 2ebc3f42bf449da00f56d8de0a8aebd736687cd0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:35:39 +0800 Subject: [PATCH 239/473] test: reproduce panel proxy operations bypassing VM claims --- test/unit/panel-proxy-vm-mutations.test.mjs | 189 ++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 test/unit/panel-proxy-vm-mutations.test.mjs diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs new file mode 100644 index 000000000..003f6396c --- /dev/null +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -0,0 +1,189 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' +import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' + +async function fixture(t) { + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-proxy-claims-')) + fs.mkdirSync(path.join(project, 'vms')) + const bin = path.join(project, 'bin') + fs.mkdirSync(bin) + fs.writeFileSync(path.join(bin, 'docker'), '#!/bin/sh\necho "Error: No such network: synthetic" >&2\nexit 1\n', { + mode: 0o700, + }) + fs.writeFileSync(path.join(bin, 'iptables'), '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + const saved = { mode: process.env.VM2API_CONTROL_PLANE_MODE, mock: process.env.KIN_CRS_MOCK, path: process.env.PATH } + process.env.VM2API_CONTROL_PLANE_MODE = 'standalone' + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const pool = await new ProxyPool({ db, geoLookup: async () => ({ ok: true, geo: { timezone: 'UTC' } }) }).init() + const imported = await pool.importLines('192.0.2.1:1080') + const proxyId = imported.items[0].id + await pool.bind(proxyId, 'vm-a') + pool.stopScheduler() + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + process.env.KIN_CRS_MOCK = '1' + process.env.PATH = bin + path.delimiter + process.env.PATH + t.after(async () => { + pool.stopScheduler() + await peer.close() + for (const [key, value] of [ + ['VM2API_CONTROL_PLANE_MODE', saved.mode], + ['KIN_CRS_MOCK', saved.mock], + ['PATH', saved.path], + ]) { + if (value == null) delete process.env[key] + else process.env[key] = value + } + fs.rmSync(project, { recursive: true, force: true }) + }) + const operations = new SharedVmOperations(db) + const otherOperations = new SharedVmOperations(peer) + function seed(id, extra = {}) { + fs.writeFileSync( + path.join(project, 'vms', id + '.json'), + JSON.stringify({ + id, + name: id, + status: 'stopped', + schedulable: false, + timezone: 'UTC', + timezone_source: 'manual', + proxy: pool.getProxyByIdWithAuth(proxyId), + ...extra, + }), + ) + } + seed('vm-a') + function handler(body, extra = {}) { + const response = {} + const handle = createPanelHandler({ + cfg: { paths: { project } }, + routingConfig: {}, + proxyPool: pool, + vmOperations: operations, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + readBody: async () => body, + json(_res, status, payload) { + response.status = status + response.body = payload + return true + }, + ...extra, + }) + return { response, request: (method, route) => handle({ method }, {}, new URL('http://localhost' + route)) } + } + return { db, peer, pool, project, proxyId, operations, otherOperations, handler, seed } +} + +for (const [method, route, body] of [ + ['PUT', '', { label: 'changed' }], + ['POST', '/bind', { vm_id: 'vm-a' }], + ['POST', '/unbind', {}], + ['DELETE', '', {}], + ['POST', '/enable', {}], + ['POST', '/disable', {}], + ['POST', '/geo', {}], +]) { + test(`panel proxy ${method} ${route || '/'} locks its bound VMs before mutation`, async (t) => { + const f = await fixture(t) + await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + vmId: 'vm-a', + accountId: 'synthetic', + accountLimit: 2, + vmLimit: 2, + rpmLimit: 0, + fableLimit: 0, + }) + const before = fs.readFileSync(path.join(f.project, 'vms', 'vm-a.json'), 'utf8') + const h = f.handler(body) + await h.request(method, '/api/panel/proxies/' + f.proxyId + route).catch((error) => { + h.response.status = 500 + h.response.body = { error: { code: error.code } } + }) + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(h.response.body.error.code, 'vm_execution_active') + assert.equal(fs.readFileSync(path.join(f.project, 'vms', 'vm-a.json'), 'utf8'), before) + }) +} + +test('proxy global configuration and auto allocation reject globally busy targets', async (t) => { + const f = await fixture(t) + await f.otherOperations.tryAcquire({ vmId: 'vm-a' }) + for (const [method, route, body] of [ + ['PUT', '/api/panel/proxies/config', { dns_primary: '1.1.1.1:53' }], + ['POST', '/api/panel/vms/vm-a/allocate-proxy', {}], + ['POST', '/api/panel/proxies/import', { text: '192.0.2.2:1080', bind_vm_id: 'vm-a' }], + ]) { + const h = f.handler(body) + await h.request(method, route).catch((error) => { + h.response.status = 500 + h.response.body = { error: { code: error.code } } + }) + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(h.response.body.error.code, 'vm_operation_active') + } +}) + +test('a failed worker reload after proxy editing keeps the target claim unknown', async (t) => { + const f = await fixture(t) + process.env.KIN_CRS_MOCK = '0' + f.seed('vm-a', { status: 'running' }) + const h = f.handler( + { port: 2080 }, + { + vmMutationOps: { + ensureProxyEgress: async () => ({ ok: true }), + reloadSlotReady: async () => ({ ok: false, error: 'worker ACK missing', vmOperationUncertain: true }), + }, + }, + ) + await h.request('PUT', '/api/panel/proxies/' + f.proxyId) + assert.equal(h.response.status, 200) + assert.equal(h.response.body.data.workers[0].ok, false) + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-a' })).ok, false) +}) + +test('proxy unbind awaits confirmed helper stop before releasing its VM claim', async (t) => { + const f = await fixture(t) + let finish + let entered + const pending = new Promise((resolve) => { + finish = resolve + }) + const ready = new Promise((resolve) => { + entered = resolve + }) + const h = f.handler( + {}, + { + vmMutationOps: { + stopProxyEgress: async () => { + entered() + await pending + return { ok: true, ended: true } + }, + }, + }, + ) + const request = h.request('POST', '/api/panel/proxies/' + f.proxyId + '/unbind') + await ready + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-a' })).ok, false) + finish() + await request + assert.equal(h.response.status, 200) + const next = await f.otherOperations.tryAcquire({ vmId: 'vm-a' }) + assert.equal(next.ok, true) + await next.release() +}) From d37192155422fd5ac3f4f6aca5287ae8eb8c2873 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:35:53 +0800 Subject: [PATCH 240/473] test: require durable proxy pool operation claims before control mutations --- test/unit/shared-proxy-operations.test.mjs | 135 +++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 test/unit/shared-proxy-operations.test.mjs diff --git a/test/unit/shared-proxy-operations.test.mjs b/test/unit/shared-proxy-operations.test.mjs new file mode 100644 index 000000000..3857e5ae4 --- /dev/null +++ b/test/unit/shared-proxy-operations.test.mjs @@ -0,0 +1,135 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { BackgroundTaskRunsRepo } from '../../src/lib/db/repos/background-task-runs-repo.mjs' + +const TASK_KEY = 'proxy_operation:pool' + +async function fixture(t) { + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => peer.close()) + const firstPool = await new ProxyPool({ db }).init() + const secondPool = await new ProxyPool({ db: peer }).init() + t.after(() => { + firstPool.stopScheduler() + secondPool.stopScheduler() + }) + const imported = await firstPool.importLines('192.0.2.1:1080') + firstPool.stopScheduler() + return { db, peer, firstPool, secondPool, id: imported.items[0].id } +} + +async function services(t) { + const f = await fixture(t) + const { SharedProxyOperations } = await import('../../src/lib/vm/shared-proxy-operations.mjs') + return { + ...f, + first: new SharedProxyOperations(f.db, { ownerId: 'same-owner' }), + second: new SharedProxyOperations(f.peer, { ownerId: 'same-owner' }), + } +} + +test('even an empty proxy cannot be edited while a foreign persistent proxy operation is active', async (t) => { + const f = await fixture(t) + const mode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + }) + await new BackgroundTaskRunsRepo(f.peer).tryClaim({ + taskKey: TASK_KEY, + executionId: 'foreign', + ownerId: 'same-owner', + }) + await assert.rejects(f.firstPool.update(f.id, { port: 2080 }), { code: 'proxy_operation_active' }) + assert.equal((await f.firstPool.repo.getById(f.id)).port, 1080) +}) + +test('proxy operation claims commit before callback and have exactly one winner across PG connections', async (t) => { + const f = await services(t) + const result = await Promise.all([f.first.tryAcquire(), f.second.tryAcquire()]) + assert.equal(result.filter((claim) => claim.ok).length, 1) + const winner = result.find((claim) => claim.ok) + assert.equal( + ( + await f.peer + .prepare('SELECT count(*) AS n FROM background_task_runs WHERE task_key = $1 AND finished_at IS NULL') + .get(TASK_KEY) + ).n, + 1, + ) + assert.equal((await winner.release()).released, true) + const next = await f.second.tryAcquire() + assert.equal(next.ok, true) + await next.release() +}) + +test('only the current exact task/execution/owner identity can edit or bind under its pool operation', async (t) => { + const f = await services(t) + const mode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + }) + await f.first.run(async () => { + assert.equal((await f.firstPool.update(f.id, { port: 2080 })).ok, true) + // VM claim and pool operation are independent, nested permissions. + await new SharedVmOperations(f.db).run('vm-new', () => f.firstPool.bind(f.id, 'vm-new')) + return { terminal: true } + }) + assert.equal((await f.firstPool.repo.getById(f.id)).port, 2080) +}) + +test('uncertain proxy operations survive DB reconnection and do not expire or authorize another same-owner request', async (t) => { + const f = await services(t) + const claim = await f.first.tryAcquire() + await claim.markUncertain() + assert.equal((await claim.release()).released, false) + await f.db.prepare("UPDATE background_task_runs SET claimed_at = '2000-01-01' WHERE task_key = $1").run(TASK_KEY) + assert.equal((await f.second.tryAcquire()).ok, false) +}) + +test('a normal callback without explicit terminal proof retains its persistent proxy operation', async (t) => { + const f = await services(t) + await f.first.run(async () => ({ value: 'HTTP response sent' })) + assert.equal((await f.second.tryAcquire()).ok, false) +}) + +test('an exception retains ownership while an explicitly not-started failure can release', async (t) => { + const f = await services(t) + await assert.rejects( + f.first.run(async () => { + throw Object.assign(new Error('synthetic unsubmitted'), { proxyOperationNotStarted: true }) + }), + ) + assert.equal((await f.second.tryAcquire()).ok, true) +}) + +test('expired callback contexts cannot authorize detached proxy mutations', async (t) => { + const f = await services(t) + let start + const pending = new Promise((resolve) => { + start = resolve + }) + let descendant + await f.first.run(async () => { + descendant = pending.then(() => f.firstPool.update(f.id, { port: 2080 })) + return { terminal: true } + }) + const foreign = await f.second.tryAcquire() + const mode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + }) + start() + await assert.rejects(descendant, { code: 'proxy_operation_active' }) + await foreign.release() +}) From 9ec6b69fad192a01026218b49309feaf673123bd Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:36:28 +0800 Subject: [PATCH 241/473] test: reproduce stale catalog policy and duplicate scheduled syncs --- test/unit/catalog-shared.test.mjs | 130 ++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 test/unit/catalog-shared.test.mjs diff --git a/test/unit/catalog-shared.test.mjs b/test/unit/catalog-shared.test.mjs new file mode 100644 index 000000000..0a19637fc --- /dev/null +++ b/test/unit/catalog-shared.test.mjs @@ -0,0 +1,130 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { loadModelPolicy, getModelPolicy, syncWorkerModelsIntoPolicy } from '../../src/lib/protocol/model-policy.mjs' +import { + loadGptModelPolicy, + getGptModelPolicy, + syncGptIdsIntoPolicy, +} from '../../src/lib/protocol/gpt-model-policy.mjs' +import { + syncClaudeCatalog, + syncGptCatalog, + readCatalogSync, + createCatalogMonitor, +} from '../../src/lib/admin/catalog-monitor.mjs' +import { syncModelsDev } from '../../src/lib/admin/price-catalog.mjs' + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-catalog-')) + fs.mkdirSync(path.join(projectRoot, 'vms')) + t.after(async () => { + await peer.close() + fs.rmSync(projectRoot, { recursive: true, force: true }) + }) + await loadModelPolicy({ force: true }) + await loadGptModelPolicy({ force: true }) + return { db, peer, projectRoot, repo: new SettingsRepo(peer) } +} + +function seedVm(projectRoot) { + fs.writeFileSync( + path.join(projectRoot, 'vms', 'catalog-vm.json'), + JSON.stringify({ + id: 'catalog-vm', + status: 'running', + claude: { access_token: 'synthetic', refresh_token: 'synthetic', account_uuid: 'synthetic' }, + proxy: { url: 'socks5h://127.0.0.1:19099' }, + }), + ) +} + +test("Claude catalog merging preserves another connection's committed model edits", async (t) => { + const { repo } = await fixture(t) + const current = await repo.get('model_policy') + current.models['claude-opus-5'].enabled = false + await repo.set('model_policy', current) + await syncWorkerModelsIntoPolicy(['claude-opus-6-synthetic']) + const stored = await repo.get('model_policy') + assert.equal(stored.models['claude-opus-5'].enabled, false) + assert.ok(stored.models['claude-opus-6-synthetic']) +}) + +test("GPT catalog merging preserves another connection's committed model edits", async (t) => { + const { repo } = await fixture(t) + const current = await repo.get('gpt_model_policy') + current.models['gpt-5.6'].enabled = false + await repo.set('gpt_model_policy', current) + await syncGptIdsIntoPolicy(['gpt-5.6', 'gpt-7-synthetic']) + assert.equal((await repo.get('gpt_model_policy')).models['gpt-5.6'].enabled, false) +}) + +test('failed catalog policy commits reject without publishing uncommitted models', async (t) => { + const { db } = await fixture(t) + await db.exec(`CREATE FUNCTION refuse_catalog_policy() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.key IN ('model_policy','gpt_model_policy') THEN + RAISE EXCEPTION 'synthetic catalog commit failure' USING ERRCODE='P6485'; END IF; RETURN NEW; END $$; + CREATE TRIGGER refuse_catalog_policy BEFORE INSERT OR UPDATE ON settings + FOR EACH ROW EXECUTE FUNCTION refuse_catalog_policy()`) + await assert.rejects(syncWorkerModelsIntoPolicy(['claude-opus-6-synthetic']), { code: 'P6485' }) + assert.equal(getModelPolicy().models['claude-opus-6-synthetic'], undefined) + await assert.rejects(syncGptIdsIntoPolicy(['gpt-7-synthetic']), { code: 'P6485' }) + assert.equal(getGptModelPolicy().models['gpt-7-synthetic'], undefined) +}) + +test('concurrent Claude and GPT status writes preserve both platform fields', async (t) => { + const { projectRoot } = await fixture(t) + seedVm(projectRoot) + await Promise.all([ + syncClaudeCatalog({ projectRoot, runOauth: async () => ({ ok: true, body: { data: [{ id: 'claude-opus-5' }] } }) }), + syncGptCatalog({ projectRoot }), + ]) + assert.equal(readCatalogSync().claude?.status, 'ok') + assert.equal(readCatalogSync().codex?.status, 'skipped') +}) + +test('two catalog monitors share the scheduled models.dev request', async (t) => { + const { db, peer, projectRoot } = await fixture(t) + let requests = 0 + t.mock.method(globalThis, 'fetch', async () => { + requests++ + return new Response(JSON.stringify({ anthropic: { models: {} }, openai: { models: {} } })) + }) + const monitors = [db, peer].map((connection) => + createCatalogMonitor({ + projectRoot, + sharedTasks: new SharedBackgroundTasks(connection), + }), + ) + t.mock.timers.enable({ apis: ['setTimeout', 'setInterval'] }) + t.after(() => monitors.forEach((monitor) => monitor.stop())) + for (const monitor of monitors) monitor.start() + t.mock.timers.tick(5000) + await Promise.all(monitors.map((monitor) => monitor.flush())) + assert.equal(requests, 1) +}) + +test('a failed price fetch preserves the latest committed snapshot from another connection', async (t) => { + const { repo } = await fixture(t) + await repo.set('price_catalog_modelsdev', { + status: 'ok', + fetched_at: '2026-10-21T00:00:00.000Z', + anthropic: { 'claude-opus-6-synthetic': { name: 'synthetic', cost: { input: 1 } } }, + openai: {}, + }) + await syncModelsDev({ + fetchImpl: async () => { + throw new Error('synthetic network failure') + }, + }) + const stored = await repo.get('price_catalog_modelsdev') + assert.ok(stored.anthropic['claude-opus-6-synthetic']) + assert.equal(stored.fetched_at, '2026-10-21T00:00:00.000Z') +}) From 1f59187ea5aa6ce563e5fe707253e0e118076c80 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:37:04 +0800 Subject: [PATCH 242/473] test: keep local model policies unchanged when PostgreSQL rejects writes --- test/unit/catalog-shared.test.mjs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/test/unit/catalog-shared.test.mjs b/test/unit/catalog-shared.test.mjs index 0a19637fc..5e78d20f1 100644 --- a/test/unit/catalog-shared.test.mjs +++ b/test/unit/catalog-shared.test.mjs @@ -6,11 +6,19 @@ import path from 'node:path' import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' -import { loadModelPolicy, getModelPolicy, syncWorkerModelsIntoPolicy } from '../../src/lib/protocol/model-policy.mjs' +import { + loadModelPolicy, + getModelPolicy, + syncWorkerModelsIntoPolicy, + saveModelPolicy, + resetModelPolicy, +} from '../../src/lib/protocol/model-policy.mjs' import { loadGptModelPolicy, getGptModelPolicy, syncGptIdsIntoPolicy, + saveGptModelPolicy, + resetGptModelPolicy, } from '../../src/lib/protocol/gpt-model-policy.mjs' import { syncClaudeCatalog, @@ -79,6 +87,24 @@ test('failed catalog policy commits reject without publishing uncommitted models assert.equal(getGptModelPolicy().models['gpt-7-synthetic'], undefined) }) +for (const [name, write, read] of [ + ['Claude save', saveModelPolicy, getModelPolicy], + ['Claude reset', resetModelPolicy, getModelPolicy], + ['GPT save', saveGptModelPolicy, getGptModelPolicy], + ['GPT reset', resetGptModelPolicy, getGptModelPolicy], +]) { + test(`${name} rejects a PostgreSQL failure before publishing local policy`, async (t) => { + const { db } = await fixture(t) + const before = structuredClone(read()) + await db.exec(`CREATE FUNCTION refuse_policy_write() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic policy write failure' USING ERRCODE='P6486'; END $$; + CREATE TRIGGER refuse_policy_write BEFORE INSERT OR UPDATE ON settings + FOR EACH ROW EXECUTE FUNCTION refuse_policy_write()`) + await assert.rejects(write({ ...before, source: 'synthetic-not-committed' }), { code: 'P6486' }) + assert.deepEqual(read(), before) + }) +} + test('concurrent Claude and GPT status writes preserve both platform fields', async (t) => { const { projectRoot } = await fixture(t) seedVm(projectRoot) From d2a211951965dd39d31e11477966e0cdc79439d2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:38:28 +0800 Subject: [PATCH 243/473] test: close completed proxy contexts and preserve nested operation values --- test/unit/shared-proxy-operations.test.mjs | 77 +++++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/test/unit/shared-proxy-operations.test.mjs b/test/unit/shared-proxy-operations.test.mjs index 3857e5ae4..175932feb 100644 --- a/test/unit/shared-proxy-operations.test.mjs +++ b/test/unit/shared-proxy-operations.test.mjs @@ -5,6 +5,9 @@ import { createRepositoryTestDatabase } from '../repository-database.mjs' import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { BackgroundTaskRunsRepo } from '../../src/lib/db/repos/background-task-runs-repo.mjs' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { createInterface } from 'node:readline' const TASK_KEY = 'proxy_operation:pool' @@ -86,13 +89,14 @@ test('only the current exact task/execution/owner identity can edit or bind unde assert.equal((await f.firstPool.repo.getById(f.id)).port, 2080) }) -test('uncertain proxy operations survive DB reconnection and do not expire or authorize another same-owner request', async (t) => { +test('uncertain proxy operations survive owner DB disconnection and do not expire', async (t) => { const f = await services(t) - const claim = await f.first.tryAcquire() + const claim = await f.second.tryAcquire() await claim.markUncertain() assert.equal((await claim.release()).released, false) + await f.peer.close() await f.db.prepare("UPDATE background_task_runs SET claimed_at = '2000-01-01' WHERE task_key = $1").run(TASK_KEY) - assert.equal((await f.second.tryAcquire()).ok, false) + assert.equal((await f.first.tryAcquire()).ok, false) }) test('a normal callback without explicit terminal proof retains its persistent proxy operation', async (t) => { @@ -133,3 +137,70 @@ test('expired callback contexts cannot authorize detached proxy mutations', asyn await assert.rejects(descendant, { code: 'proxy_operation_active' }) await foreign.release() }) + +test('completed callback contexts cannot fall back to unclaimed mutations when no new owner exists', async (t) => { + const f = await services(t) + let start + const pending = new Promise((resolve) => { + start = resolve + }) + let descendant + await f.first.run(async () => { + descendant = pending.then(() => f.firstPool.update(f.id, { port: 2080 })) + return { terminal: true } + }) + const mode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + }) + start() + await assert.rejects(descendant, { code: 'proxy_operation_inactive' }) +}) + +test('nested pool operations reuse one claim and retain the value contract', async (t) => { + const f = await services(t) + await f.first.run(async () => { + const value = await f.second.run(async () => ({ terminal: true, value: 'synthetic' })) + assert.equal(value, 'synthetic') + assert.equal( + ( + await f.peer + .prepare('SELECT count(*) AS n FROM background_task_runs WHERE task_key = $1 AND finished_at IS NULL') + .get(TASK_KEY) + ).n, + 1, + ) + return { terminal: true } + }) +}) + +test('SIGKILL does not resell a proxy pool operation to another process', async (t) => { + const f = await services(t) + const repositoryUrl = new URL('../repository-database.mjs', import.meta.url).href + const serviceUrl = new URL('../../src/lib/vm/shared-proxy-operations.mjs', import.meta.url).href + const script = `import { createRepositoryTestDatabase } from ${JSON.stringify(repositoryUrl)}; + import { SharedProxyOperations } from ${JSON.stringify(serviceUrl)}; + const db = await createRepositoryTestDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: process.env.KIN_DATABASE_SCHEMA }); + const claim = await new SharedProxyOperations(db).tryAcquire(); + process.stdout.write(JSON.stringify({ ok: claim.ok }) + '\\n'); + setInterval(() => {}, 1000);` + const child = spawn(process.execPath, ['--input-type=module', '-e', script], { + env: { ...process.env, KIN_DATABASE_SCHEMA: f.db.schema }, + stdio: ['ignore', 'pipe', 'ignore'], + }) + t.after(() => child.kill('SIGKILL')) + const lines = createInterface({ input: child.stdout }) + const message = await Promise.race([ + once(lines, 'line').then(([line]) => JSON.parse(line)), + once(child, 'exit').then(() => { + throw new Error('synthetic owner exited before reporting') + }), + ]) + assert.equal(message.ok, true) + const exited = once(child, 'exit') + child.kill('SIGKILL') + await exited + assert.equal((await f.second.tryAcquire()).ok, false) +}) From f4a685b98f31d467b6a2c8c5a8b7cf7749bf15c5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:40:27 +0800 Subject: [PATCH 244/473] feat: persist watchdog recovery budgets and enforce global faults --- .../db/repos/execution-admissions-repo.mjs | 6 +- src/lib/pool/pool-scheduler.mjs | 4 +- src/lib/transport/kernel-watchdog-state.mjs | 36 ++++++++++++ src/lib/transport/kernel-watchdog.mjs | 56 +++++++++++++++++-- src/lib/vm/slot-recovery.mjs | 2 +- test/unit/shared-watchdog-state.test.mjs | 33 +++++++++++ 6 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 src/lib/transport/kernel-watchdog-state.mjs diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 19efc4ace..4e8a16f65 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -1,6 +1,7 @@ import crypto from 'node:crypto' import { withTransaction } from '../database.mjs' import { physicalContainerName, physicalVmAliases } from '../../vm/vm-physical-target.mjs' +import { readWatchdogState } from '../../transport/kernel-watchdog-state.mjs' function invalid(field) { return Object.assign(new Error(`Invalid execution admission ${field}`), { code: 'invalid_execution_admission' }) @@ -88,7 +89,9 @@ export class ExecutionAdmissionsRepo { if (vmId != null) identifier(vmId, 'vmId') if (accountId == null && vmId == null) throw invalid('resource') const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() - return this._counts(accountId ?? null, vmId ?? null, now) + const counts = await this._counts(accountId ?? null, vmId ?? null, now) + const fault = vmId == null ? null : (await readWatchdogState(this.db, vmId)).fault + return fault ? { ...counts, kernelFault: fault } : counts } async tryReserve(input = {}) { @@ -114,6 +117,7 @@ export class ExecutionAdmissionsRepo { .prepare('SELECT id FROM vms WHERE id = ANY($1::text[]) AND schedulable IS DISTINCT FROM 1 LIMIT 1') .get(physicalVmAliases(vmId)) if (held) return denied('vm_not_schedulable', 0, vmLimit) + if ((await readWatchdogState(this.db, vmId)).fault) return denied('kernel_faulted', 0, vmLimit) const alias = await this.db .prepare(`SELECT id FROM execution_admissions WHERE vm_id = ANY($1::text[]) AND vm_id <> $2 AND released_at IS NULL LIMIT 1`) diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 663078499..02b370f22 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -1315,7 +1315,9 @@ export class PoolScheduler extends EventEmitter { } if (this.shared && gateZone === GATE_DRAINING && inflight > 0) markWait('quota_band_inflight') // A sticky preference must not bypass exhausted kernel recovery. - if (kernelFaults.has(vm.id)) return { ok: false, reason: 'kernel_faulted' } + if (this.shared ? sharedCounts.kernelFault : kernelFaults.has(vm.id)) { + return { ok: false, reason: 'kernel_faulted' } + } // Node offline or asleep: the relay has no link behind it, a pick would only fail. Back on reconnect. if (!vmNodeReady(vm)) return { ok: false, reason: 'node_offline' } if (rustKernelBusy(workerStatus)) { diff --git a/src/lib/transport/kernel-watchdog-state.mjs b/src/lib/transport/kernel-watchdog-state.mjs new file mode 100644 index 000000000..1a3e00a76 --- /dev/null +++ b/src/lib/transport/kernel-watchdog-state.mjs @@ -0,0 +1,36 @@ +import { SettingsRepo } from '../db/repos/settings-repo.mjs' +import { physicalContainerName } from '../vm/vm-physical-target.mjs' + +export function watchdogStateKey(vmId) { + return `kernel_watchdog_state:${physicalContainerName(vmId)}` +} + +function validate(value) { + if ( + value?.version !== 1 || + !Array.isArray(value.attempts) || + value.attempts.some((at) => !Number.isFinite(at) || at < 0) || + !(value.nextAt === null || (Number.isFinite(value.nextAt) && value.nextAt >= 0)) || + !Number.isFinite(value.telemetryDue) || + value.telemetryDue < 0 || + !(value.fault === null || typeof value.fault === 'string') + ) + throw Object.assign(new Error('Invalid persistent watchdog state'), { code: 'watchdog_state_invalid' }) + return value +} + +export async function readWatchdogState(db, vmId) { + const row = await db.prepare('SELECT value FROM settings WHERE key = $1').get(watchdogStateKey(vmId)) + if (!row) return { version: 1, attempts: [], nextAt: null, telemetryDue: 0, fault: null } + let value + try { + value = JSON.parse(row.value) + } catch { + throw Object.assign(new Error('Invalid persistent watchdog state'), { code: 'watchdog_state_invalid' }) + } + return validate(value) +} + +export function writeWatchdogState(db, vmId, value) { + return new SettingsRepo(db).set(watchdogStateKey(vmId), validate(value)) +} diff --git a/src/lib/transport/kernel-watchdog.mjs b/src/lib/transport/kernel-watchdog.mjs index 0ac27fe2e..b86c8470b 100644 --- a/src/lib/transport/kernel-watchdog.mjs +++ b/src/lib/transport/kernel-watchdog.mjs @@ -30,6 +30,7 @@ import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' import { SharedVmOperations } from '../vm/shared-vm-operations.mjs' import { physicalContainerName } from '../vm/vm-physical-target.mjs' import { flushVmDbSync } from '../vm/vm-db-sync.mjs' +import { readWatchdogState, writeWatchdogState } from './kernel-watchdog-state.mjs' /** A failed observation is not evidence of a dead container. Use the slot's own host, including KVM runners. */ export async function observeWatchdogContainer( @@ -185,6 +186,27 @@ export function createKernelWatchdog({ const tasks = backgroundTasksFor(sharedTasks) const mutations = operations || (tasks ? new SharedVmOperations(tasks.db) : null) const sideEffects = new Set() + const databaseTimes = new Map() + const clock = (vm) => (tasks ? databaseTimes.get(vm.id) : now()) + + function adoptState(vmId, state) { + if (state.nextAt == null) restarts.delete(vmId) + else restarts.set(vmId, { attempts: [...state.attempts], nextAt: state.nextAt }) + telemetryDue.set(vmId, state.telemetryDue) + if (state.fault) kernelFaults.set(vmId, state.fault) + else kernelFaults.delete(vmId) + } + + function currentState(vmId) { + const budget = restarts.get(vmId) + return { + version: 1, + attempts: budget?.attempts || [], + nextAt: budget?.nextAt ?? null, + telemetryDue: telemetryDue.get(vmId) || 0, + fault: kernelFaults.get(vmId) || null, + } + } function trackSideEffect(operation) { sideEffects.add(operation) @@ -258,7 +280,7 @@ export function createKernelWatchdog({ /** One VM's check. `held` is re-read after every await: an operator cycle may start mid-check. */ async function reviveDownContainer(vm, state, markUnknown = () => {}) { if (vm.status !== 'running' || typeof revive !== 'function') return - if (!(await admitRestart(vm, `container ${state}`, now()))) return + if (!(await admitRestart(vm, `container ${state}`, clock(vm)))) return const revived = await revive(vm) if (tasks && revived?.ok !== true) markUnknown() if (!revived?.ok) console.warn(`[kernel-watchdog] ${vm.id} boot failed: ${revived?.error || revived?.code}`) @@ -301,7 +323,7 @@ export function createKernelWatchdog({ // Permission/transport failures and deliberately paused containers never authorize recovery. if (!['running', 'missing', 'stopped', 'restarting'].includes(observed?.state)) return if (observed.state !== 'running') return reviveDownContainer(vm, observed.state, markUnknown) - if (!(await admitRestart(vm, reason, now()))) return + if (!(await admitRestart(vm, reason, clock(vm)))) return console.warn(`[kernel-watchdog] ${vm.id} restarting container: ${reason}`) const result = await ensure(exec, { timeoutMs: config.timeout_ms }) if (tasks && result?.ok !== true) markUnknown() @@ -344,13 +366,23 @@ export function createKernelWatchdog({ await mutations.run( vm.id, async () => { + adoptState(vm.id, await readWatchdogState(tasks.db, vm.id)) + const at = ( + await tasks.db.prepare('SELECT extract(epoch FROM clock_timestamp()) * 1000 AS now').get() + ).now + databaseTimes.set(vm.id, at) const markUnknown = () => { unknown = true } - if (isTelemetryHealTarget(vm)) await healTelemetry(vm, now(), markUnknown) - if (isKernelWatchdogTarget(vm, routing)) await checkVm(vm, routing, markUnknown) - await flushVmDbSync() - return { vmOperationUncertain: unknown } + try { + if (isTelemetryHealTarget(vm)) await healTelemetry(vm, at, markUnknown) + if (isKernelWatchdogTarget(vm, routing)) await checkVm(vm, routing, markUnknown) + await writeWatchdogState(tasks.db, vm.id, currentState(vm.id)) + await flushVmDbSync() + return { vmOperationUncertain: unknown } + } finally { + databaseTimes.delete(vm.id) + } }, { kind: 'kernel_watchdog' }, ) @@ -415,6 +447,18 @@ export function createKernelWatchdog({ }, /** Operator recovery: drop the L5 fault and spent budget so the next failure starts fresh. */ reset(vmId) { + if (tasks) + return mutations.run( + vmId, + async () => { + const previous = await readWatchdogState(tasks.db, vmId) + const state = { ...previous, attempts: [], nextAt: null, fault: null } + await writeWatchdogState(tasks.db, vmId, state) + adoptState(vmId, state) + return { faulted: !!previous.fault } + }, + { kind: 'watchdog_reset' }, + ) const faulted = kernelFaults.delete(vmId) restarts.delete(vmId) return { faulted } diff --git a/src/lib/vm/slot-recovery.mjs b/src/lib/vm/slot-recovery.mjs index 096d1e288..5fa0832cb 100644 --- a/src/lib/vm/slot-recovery.mjs +++ b/src/lib/vm/slot-recovery.mjs @@ -527,7 +527,7 @@ export async function recoverSlot({ // Only an attempted boot earns a fresh budget; a cycle that never reached start fixed nothing. if (watchdog?.reset && proceed) { await step('watchdog_reset', async () => { - const { faulted } = watchdog.reset(id) + const { faulted } = await watchdog.reset(id) return { ok: true, message: faulted ? '已清除内核故障标记与重启计数' : '重启计数已清零' } }) } diff --git a/test/unit/shared-watchdog-state.test.mjs b/test/unit/shared-watchdog-state.test.mjs index 368cbe0b3..35f2da10b 100644 --- a/test/unit/shared-watchdog-state.test.mjs +++ b/test/unit/shared-watchdog-state.test.mjs @@ -76,3 +76,36 @@ test('operator watchdog reset persists before another controller admits a reques assert.equal(next.ok, true) await next.release() }) + +test('a state persistence failure after watchdog recovery retains both durable claims', async () => { + const db = getDb() + const at = (await db.prepare('SELECT extract(epoch FROM clock_timestamp()) * 1000 AS now').get()).now + await new SettingsRepo(db).set(key, state({ nextAt: at - 1 })) + await db.exec(`CREATE FUNCTION reject_watchdog_state() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.key LIKE 'kernel_watchdog_state:%' THEN + RAISE EXCEPTION 'synthetic watchdog state failure' USING ERRCODE = 'P6498'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER reject_watchdog_state BEFORE INSERT OR UPDATE ON settings + FOR EACH ROW EXECUTE FUNCTION reject_watchdog_state()`) + let recovered = 0 + const watchdog = createKernelWatchdog({ + listTargets: () => [{ id: vmId, status: 'running' }], + health: async () => ({ ok: true, healthy: false }), + observeRuntime: async () => ({ state: 'running' }), + ensure: async () => { + recovered++ + return { ok: true } + }, + sharedTasks: new SharedBackgroundTasks(db), + operations: new SharedVmOperations(db), + }) + await watchdog.tick() + assert.equal(recovered, 1) + assert.equal((await reserve()).code, 'vm_operation_active') + const row = await db.prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.ok(row.unknown_at) + assert.equal(row.finished_at, null) +}) From 43d8ba8abe4b6efa56bfa8fd6162569a5bb0b35d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:40:52 +0800 Subject: [PATCH 245/473] test: fence catalog credential work with shared VM operations --- test/unit/catalog-shared.test.mjs | 123 ++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/test/unit/catalog-shared.test.mjs b/test/unit/catalog-shared.test.mjs index 5e78d20f1..6dabc16ce 100644 --- a/test/unit/catalog-shared.test.mjs +++ b/test/unit/catalog-shared.test.mjs @@ -27,6 +27,8 @@ import { createCatalogMonitor, } from '../../src/lib/admin/catalog-monitor.mjs' import { syncModelsDev } from '../../src/lib/admin/price-catalog.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { syncCodexCatalog } from '../../src/lib/admin/vm-test-chat.mjs' async function fixture(t) { const db = getDb() @@ -39,6 +41,10 @@ async function fixture(t) { }) await loadModelPolicy({ force: true }) await loadGptModelPolicy({ force: true }) + await new SettingsRepo(db).set('routing_config', { + revision: 1, + config: { codex: { catalog_client_version: '0.130.0' } }, + }) return { db, peer, projectRoot, repo: new SettingsRepo(peer) } } @@ -154,3 +160,120 @@ test('a failed price fetch preserves the latest committed snapshot from another assert.ok(stored.anthropic['claude-opus-6-synthetic']) assert.equal(stored.fetched_at, '2026-10-21T00:00:00.000Z') }) + +test('Claude catalog work cannot execute while another Pod owns its VM mutation claim', async (t) => { + const { db, peer, projectRoot } = await fixture(t) + seedVm(projectRoot) + const occupied = await new SharedVmOperations(peer).tryAcquire({ vmId: 'catalog-vm', kind: 'synthetic' }) + let calls = 0 + try { + const result = await syncClaudeCatalog({ + projectRoot, + sharedTasks: new SharedBackgroundTasks(db), + runOauth: async () => { + calls++ + return { ok: true, body: { data: [] } } + }, + }) + assert.equal(calls, 0) + assert.equal(result.error, 'vm_operation_active') + } finally { + await occupied.release() + } +}) + +test('an unknown Claude worker result retains both its background and VM mutation claims', async (t) => { + const { db, projectRoot } = await fixture(t) + seedVm(projectRoot) + await syncClaudeCatalog({ + projectRoot, + sharedTasks: new SharedBackgroundTasks(db), + runOauth: async () => ({ ok: false, transportError: true, body: { error: { code: 'synthetic_timeout' } } }), + }) + const operation = (await db.query('SELECT released_at,uncertain_at FROM vm_operations')).rows[0] + assert.ok(operation?.uncertain_at) + assert.equal(operation.released_at, null) + const task = (await db.query('SELECT finished_at,unknown_at FROM background_task_runs')).rows[0] + assert.ok(task.unknown_at) + assert.equal(task.finished_at, null) +}) + +function seedCodex(projectRoot) { + fs.mkdirSync(path.join(projectRoot, 'vms', 'catalog-codex')) + fs.writeFileSync( + path.join(projectRoot, 'vms', 'catalog-codex.json'), + JSON.stringify({ + id: 'catalog-codex', + status: 'running', + platform: 'openai', + family: 'codex', + codex: { has_access: true }, + proxy: { url: 'socks5h://127.0.0.1:19099' }, + }), + ) + fs.writeFileSync( + path.join(projectRoot, 'vms', 'catalog-codex', 'codex-credentials.json'), + JSON.stringify({ + accounts: [{ access_token: 'synthetic-access', refresh_token: 'synthetic-refresh' }], + }), + ) +} + +test('Codex catalog rotation cannot fetch credentials while another Pod owns that VM', async (t) => { + const { db, peer, projectRoot } = await fixture(t) + seedCodex(projectRoot) + const occupied = await new SharedVmOperations(peer).tryAcquire({ vmId: 'catalog-codex', kind: 'synthetic' }) + let calls = 0 + try { + const result = await syncCodexCatalog({ + projectRoot, + rotate: true, + catalogClientVersion: '0.130.0', + operations: new SharedVmOperations(db), + fetchImpl: async () => { + calls++ + return { status: 200, json: async () => ({ models: [{ slug: 'gpt-5.6' }] }) } + }, + }) + assert.equal(calls, 0) + assert.equal(result.error, 'vm_operation_active') + } finally { + await occupied.release() + } +}) + +test('a lost Codex token rotation response leaves its VM operation uncertain', async (t) => { + const { db, projectRoot } = await fixture(t) + seedCodex(projectRoot) + const result = await syncCodexCatalog({ + projectRoot, + rotate: true, + catalogClientVersion: '0.130.0', + operations: new SharedVmOperations(db), + fetchImpl: async (url) => { + if (String(url).includes('/oauth/token')) throw new Error('synthetic lost rotation response') + return { status: 401, json: async () => ({}) } + }, + }) + assert.equal(result.executionUncertain, true) + const operation = (await db.query('SELECT released_at,uncertain_at FROM vm_operations')).rows[0] + assert.ok(operation?.uncertain_at) + assert.equal(operation.released_at, null) +}) + +test('a read-only Codex catalog failure does not occupy a VM mutation resource', async (t) => { + const { db, projectRoot } = await fixture(t) + seedCodex(projectRoot) + const result = await syncCodexCatalog({ + projectRoot, + rotate: false, + catalogClientVersion: '0.130.0', + operations: new SharedVmOperations(db), + fetchImpl: async () => { + throw new Error('synthetic read timeout') + }, + }) + assert.equal(result.ok, false) + assert.notEqual(result.executionUncertain, true) + assert.equal((await db.query('SELECT count(*) AS n FROM vm_operations')).rows[0].n, 0) +}) From d6aabe89063864391eee852721dc844b45e133cf Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:41:50 +0800 Subject: [PATCH 246/473] test: retain proxy claims after caught nested failures --- test/unit/shared-proxy-operations.test.mjs | 38 +++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/test/unit/shared-proxy-operations.test.mjs b/test/unit/shared-proxy-operations.test.mjs index 175932feb..bf114e1e8 100644 --- a/test/unit/shared-proxy-operations.test.mjs +++ b/test/unit/shared-proxy-operations.test.mjs @@ -105,7 +105,7 @@ test('a normal callback without explicit terminal proof retains its persistent p assert.equal((await f.second.tryAcquire()).ok, false) }) -test('an exception retains ownership while an explicitly not-started failure can release', async (t) => { +test('an explicitly not-started failure can release its proxy operation', async (t) => { const f = await services(t) await assert.rejects( f.first.run(async () => { @@ -115,6 +115,42 @@ test('an exception retains ownership while an explicitly not-started failure can assert.equal((await f.second.tryAcquire()).ok, true) }) +test('a caught nested exception still retains the outer proxy operation', async (t) => { + const f = await services(t) + await f.first.run(async () => { + await assert.rejects( + f.second.run(async () => { + throw new Error('synthetic unknown effect') + }), + ) + return { terminal: true } + }) + assert.equal((await f.second.tryAcquire()).ok, false) + assert.notEqual( + (await f.db.prepare('SELECT unknown_at FROM background_task_runs WHERE task_key = $1').get(TASK_KEY)).unknown_at, + null, + ) +}) + +test('failed or unconfirmed release can retry without reselling an active claim', async (t) => { + const f = await services(t) + const claim = await f.first.tryAcquire() + const complete = f.first.runs.complete.bind(f.first.runs) + let calls = 0 + f.first.runs.complete = async (...args) => { + calls += 1 + if (calls === 1) throw new Error('synthetic DB failure') + if (calls === 2) return { completed: false } + return complete(...args) + } + await assert.rejects(claim.release(), /synthetic DB failure/) + assert.equal((await f.second.tryAcquire()).ok, false) + assert.deepEqual(await claim.release(), { released: false }) + assert.equal((await f.second.tryAcquire()).ok, false) + assert.deepEqual(await claim.release(), { released: true }) + assert.equal((await f.second.tryAcquire()).ok, true) +}) + test('expired callback contexts cannot authorize detached proxy mutations', async (t) => { const f = await services(t) let start From c2d7133c84a6b653b28a4ca31667ce8c901f41c0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:41:55 +0800 Subject: [PATCH 247/473] feat: coordinate catalog syncs and commit policy merges atomically --- src/lib/admin/catalog-monitor.mjs | 116 +++++++++++++++++--- src/lib/admin/price-catalog.mjs | 84 ++++++++++----- src/lib/admin/vm-test-chat.mjs | 146 +++++++++++++++----------- src/lib/db/repos/settings-repo.mjs | 16 ++- src/lib/protocol/gpt-model-policy.mjs | 71 +++++++------ src/lib/protocol/model-policy.mjs | 122 +++++++++++---------- 6 files changed, 362 insertions(+), 193 deletions(-) diff --git a/src/lib/admin/catalog-monitor.mjs b/src/lib/admin/catalog-monitor.mjs index 62ee4b892..452021c1e 100644 --- a/src/lib/admin/catalog-monitor.mjs +++ b/src/lib/admin/catalog-monitor.mjs @@ -6,7 +6,12 @@ * Results land in settings `model_catalog_sync` so the models page can show them. */ import path from 'node:path' +import { randomUUID } from 'node:crypto' import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' +import { SettingsRepo as PersistentSettingsRepo } from '../db/repos/settings-repo.mjs' +import { RoutingConfigRepo } from '../db/repos/routing-config-repo.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { SharedVmOperations } from '../vm/shared-vm-operations.mjs' import { runSlotOauth } from '../transport/slot-oauth.mjs' import { getVm, listVms, vmHasClaudeCredential } from '../vm/vm-registry.mjs' import { isCodexVm } from '../vm/vm-kind.mjs' @@ -33,12 +38,27 @@ export function readCatalogSync() { } async function writeCatalogSync(platform, status) { - const repo = new SettingsRepo() - const cur = repo.get(CATALOG_SYNC_KEY, null) || { claude: null, codex: null } - await repo.set(CATALOG_SYNC_KEY, { ...cur, [platform]: status }) + await new PersistentSettingsRepo().update(CATALOG_SYNC_KEY, (current) => ({ + ...(current || { claude: null, codex: null }), + [platform]: status, + })) return status } +async function runCatalogTask(tasks, platform, scheduled, callback) { + if (!tasks) return callback() + const plan = scheduled + ? { taskKey: `catalog:${platform}`, intervalMs: UPSTREAM_EVERY_MS } + : { taskKey: `catalog:${platform}`, executionId: randomUUID() } + const result = await tasks.run(plan, async () => { + const value = await callback() + return { terminal: value?.executionUncertain !== true, value, sharedResult: value } + }) + return result.ran + ? result.value + : result.sharedResult || { ok: false, status: 'skipped', error: result.code, added: [] } +} + function syncStatus(status, { added = [], error = null } = {}) { return { at: new Date().toISOString(), status, added, error } } @@ -52,7 +72,14 @@ function claudeCatalogTargets(projectRoot) { } /** Live Claude catalog → model_policy. */ -export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth } = {}) { +export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth, sharedTasks, scheduled = false } = {}) { + const tasks = backgroundTasksFor(sharedTasks) + return runCatalogTask(tasks, 'claude', scheduled, () => + syncClaudeCatalogOnce({ projectRoot, runOauth, mutations: tasks ? new SharedVmOperations(tasks.db) : null }), + ) +} + +async function syncClaudeCatalogOnce({ projectRoot, runOauth, mutations }) { const targets = claudeCatalogTargets(projectRoot) if (!targets.length) return await writeCatalogSync('claude', syncStatus('skipped', { error: 'no_claude_slot' })) const ids = new Set() @@ -60,8 +87,36 @@ export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth } let lastError = null for (const vm of targets) { if (successes >= CLAUDE_ACCOUNTS_PER_SYNC) break - const exec = { vmId: vm.id, homeDir: path.join(projectRoot, 'vms', vm.id, 'cli-home'), oauth: vm.claude, vm } - const res = await runOauth(exec, 'models', { timeoutMs: 30_000 }) + let res + const read = async () => { + const current = getVm(projectRoot, vm.id) || vm + const exec = { + vmId: current.id, + homeDir: path.join(projectRoot, 'vms', current.id, 'cli-home'), + oauth: current.claude, + vm: current, + } + const result = await runOauth(exec, 'models', { timeoutMs: 30_000 }) + return result && { ...result, vmOperationUncertain: result.transportError === true } + } + try { + res = mutations ? await mutations.run(vm.id, read, { kind: 'claude_catalog' }) : await read() + } catch (error) { + if (!['vm_operation_active', 'vm_execution_active'].includes(error.code)) throw error + lastError = error.code + continue + } + if (res?.transportError || (!res && process.env.KIN_CRS_MOCK !== '1')) { + return { + ...(await writeCatalogSync( + 'claude', + syncStatus('error', { + error: res?.body?.error?.code || 'models_execution_unknown', + }), + )), + executionUncertain: true, + } + } const list = Array.isArray(res?.body?.data) ? res.body.data : null if (!res?.ok || !list) { lastError = res?.body?.error?.code || `http_${res?.status || 0}` @@ -79,8 +134,34 @@ export async function syncClaudeCatalog({ projectRoot, runOauth = runSlotOauth } } /** Live ChatGPT Codex catalog → gpt_model_policy. Timers never rotate OAuth; the panel button may. */ -export async function syncGptCatalog({ projectRoot, vmId = '', catalogClientVersion = 'auto', rotate = false } = {}) { - const result = await syncCodexCatalog({ projectRoot, vmId, rotate, catalogClientVersion }) +export async function syncGptCatalog({ + projectRoot, + vmId = '', + catalogClientVersion = 'auto', + rotate = false, + sharedTasks, + scheduled = false, +} = {}) { + const tasks = backgroundTasksFor(sharedTasks) + return runCatalogTask(tasks, 'codex', scheduled, async () => { + if (tasks && scheduled) { + const routing = await new RoutingConfigRepo(tasks.db).get() + if (!routing) + throw Object.assign(new Error('shared routing configuration missing'), { code: 'routing_config_unavailable' }) + catalogClientVersion = routing.config.codex?.catalog_client_version || 'auto' + } + return syncGptCatalogOnce({ + projectRoot, + vmId, + catalogClientVersion, + rotate, + mutations: tasks ? new SharedVmOperations(tasks.db) : undefined, + }) + }) +} + +async function syncGptCatalogOnce({ projectRoot, vmId, catalogClientVersion, rotate, mutations }) { + const result = await syncCodexCatalog({ projectRoot, vmId, rotate, catalogClientVersion, operations: mutations }) if (!result.ok) { const skipped = result.error === 'no_codex_slot' || result.error === 'proxy_required' await writeCatalogSync('codex', syncStatus(skipped ? 'skipped' : 'error', { error: result.error })) @@ -93,7 +174,13 @@ export async function syncGptCatalog({ projectRoot, vmId = '', catalogClientVers /** * @param {{ projectRoot: string, codexCatalogVersion?: () => string, log?: (event: object) => void }} opts */ -export function createCatalogMonitor({ projectRoot, codexCatalogVersion = () => 'auto', log = () => {} } = {}) { +export function createCatalogMonitor({ + projectRoot, + codexCatalogVersion = () => 'auto', + log = () => {}, + sharedTasks, +} = {}) { + const tasks = backgroundTasksFor(sharedTasks) const timers = [] let running = false const pending = new Set() @@ -110,10 +197,15 @@ export function createCatalogMonitor({ projectRoot, codexCatalogVersion = () => pending.delete(operation) } } - const modelsDev = guard('models.dev', async () => await syncModelsDev()) + const modelsDev = guard('models.dev', async () => await syncModelsDev({ sharedTasks: tasks, scheduled: true })) const upstream = guard('upstream', async () => { - const claude = await syncClaudeCatalog({ projectRoot }) - await syncGptCatalog({ projectRoot, catalogClientVersion: codexCatalogVersion() }) + const claude = await syncClaudeCatalog({ projectRoot, sharedTasks: tasks, scheduled: true }) + await syncGptCatalog({ + projectRoot, + catalogClientVersion: codexCatalogVersion(), + sharedTasks: tasks, + scheduled: true, + }) return claude }) diff --git a/src/lib/admin/price-catalog.mjs b/src/lib/admin/price-catalog.mjs index 282fe4e84..0590cab06 100644 --- a/src/lib/admin/price-catalog.mjs +++ b/src/lib/admin/price-catalog.mjs @@ -3,6 +3,9 @@ * ID matching is exact / alias / Claude date-strip — no fuzzy family regex. */ import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' +import { SettingsRepo as PersistentSettingsRepo } from '../db/repos/settings-repo.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' +import { randomUUID } from 'node:crypto' import { BUILTIN_ALIASES, BUILTIN_RATES, BUILTIN_SOURCE } from './builtin-rates.mjs' export const PRICE_FIELDS = ['input', 'output', 'cache_read', 'cache_5m', 'cache_1h'] @@ -301,8 +304,13 @@ async function persistOverrides(next) { await settingsRepo().set(SETTINGS_OVERRIDES, next) } -async function persistSnapshot(next) { - await settingsRepo().set(SETTINGS_SNAPSHOT, next) +async function persistSnapshot(change) { + if (injectedSettings) { + const value = change(readSnapshot()) + await settingsRepo().set(SETTINGS_SNAPSHOT, value) + return value + } + return new PersistentSettingsRepo().update(SETTINGS_SNAPSHOT, change) } function statusOf(snap) { @@ -345,9 +353,29 @@ async function readCappedBody(res, maxBytes) { return text } -export async function syncModelsDev({ fetchImpl } = {}) { - const prev = readSnapshot() || {} +export async function syncModelsDev({ fetchImpl, sharedTasks, scheduled = false } = {}) { + const tasks = backgroundTasksFor(sharedTasks) + if (!tasks) return syncModelsDevOnce({ fetchImpl }) + const plan = scheduled + ? { taskKey: 'catalog:models.dev', intervalMs: MODELS_DEV_STALE_MS } + : { taskKey: 'catalog:models.dev', executionId: randomUUID() } + const result = await tasks.run(plan, async () => { + if (scheduled) { + const current = await new PersistentSettingsRepo(tasks.db).get(SETTINGS_SNAPSHOT) + if (current?.fetched_at && (await tasks.now()) - Date.parse(current.fetched_at) < MODELS_DEV_STALE_MS) { + return { terminal: true, value: statusOf(current), sharedResult: statusOf(current) } + } + } + const value = await syncModelsDevOnce({ fetchImpl }) + return { terminal: true, value, sharedResult: value } + }) + return result.ran ? result.value : result.sharedResult || { status: 'skipped', error: result.code } +} + +async function syncModelsDevOnce({ fetchImpl }) { const fetchFn = fetchImpl || injectedFetch || globalThis.fetch + let parsed + let error try { if (typeof fetchFn !== 'function') throw new Error('fetch is not available') const res = await fetchFn(MODELS_DEV_URL, { signal: AbortSignal.timeout(MODELS_DEV_TIMEOUT_MS) }) @@ -359,29 +387,29 @@ export async function syncModelsDev({ fetchImpl } = {}) { } catch { throw new Error('models.dev invalid JSON') } - const parsed = parseModelsDev(json) - const snap = { - fetched_at: new Date().toISOString(), - status: 'ok', - error: null, - anthropic: parsed.anthropic, - openai: parsed.openai, - } - await persistSnapshot(snap) - invalidatePriceCatalog() - return statusOf(snap) + parsed = parseModelsDev(json) } catch (e) { - const snap = { - fetched_at: prev.fetched_at || null, - status: 'error', - error: String(e?.message || e), - anthropic: prev.anthropic || {}, - openai: prev.openai || {}, - } - await persistSnapshot(snap) - invalidatePriceCatalog() - return statusOf(snap) - } + error = String(e?.message || e) + } + const snap = await persistSnapshot((prev) => + error + ? { + fetched_at: prev?.fetched_at || null, + status: 'error', + error, + anthropic: prev?.anthropic || {}, + openai: prev?.openai || {}, + } + : { + fetched_at: new Date().toISOString(), + status: 'ok', + error: null, + anthropic: parsed.anthropic, + openai: parsed.openai, + }, + ) + invalidatePriceCatalog() + return statusOf(snap) } function canPersist() { @@ -403,8 +431,8 @@ export function maybeRefreshModelsDev({ now } = {}) { const t = now ?? Date.now() if (fetchedAt && Number.isFinite(fetchedAt) && t - fetchedAt < MODELS_DEV_STALE_MS) return refreshInflight = Promise.resolve() - .then(async () => await syncModelsDev()) - .catch(() => {}) + .then(async () => await syncModelsDev({ scheduled: true })) + .catch((error) => console.warn('[models.dev] background refresh failed:', error.code || error.message)) .finally(() => { refreshInflight = null }) diff --git a/src/lib/admin/vm-test-chat.mjs b/src/lib/admin/vm-test-chat.mjs index 402965398..7001e711f 100644 --- a/src/lib/admin/vm-test-chat.mjs +++ b/src/lib/admin/vm-test-chat.mjs @@ -14,6 +14,9 @@ import fs from 'node:fs' import { Readable } from 'node:stream' import path from 'node:path' import { readRoutingConfigFile } from '../core/config.mjs' +import { getDb } from '../db/database.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { SharedVmOperations } from '../vm/shared-vm-operations.mjs' import { getVm, vmHasClaudeCredential } from '../vm/vm-registry.mjs' import { isCodexVm } from '../vm/vm-kind.mjs' import { summarizeCodexSlot, readCodexAccounts, writeCodexAccounts } from '../vm/codex-slot.mjs' @@ -157,6 +160,7 @@ export async function syncCodexCatalog({ fetchImpl, rotate = true, catalogClientVersion = 'auto', + operations, } = {}) { if (!projectRoot) { return { ok: false, error: 'invalid_request', message: 'projectRoot required', ids: [], synced: 0 } @@ -187,81 +191,99 @@ export async function syncCodexCatalog({ const merged = new Map() let usedVm = null let lastError = null - for (const target of targets) { - const direct = isLocalEgressProxy(target.proxy) - const proxyUrl = hostProxyUrlForVm(target) - if (!proxyUrl && !fetchImpl && !direct) { - lastError = { - ok: false, - error: 'proxy_required', - message: 'GPT 槽未绑定 SOCKS5', - vm_id: target.id, + const mutations = operations === undefined && isKubernetesMode() ? new SharedVmOperations(getDb()) : operations + let executionUncertain = false + for (const listed of targets) { + const readTarget = async () => { + const target = getVm(projectRoot, listed.id) || listed + const direct = isLocalEgressProxy(target.proxy) + const proxyUrl = hostProxyUrlForVm(target) + if (!proxyUrl && !fetchImpl && !direct) { + lastError = { + ok: false, + error: 'proxy_required', + message: 'GPT 槽未绑定 SOCKS5', + vm_id: target.id, + } + return } - continue - } - const accounts = readCodexAccounts(projectRoot, target.id) - const rows = accounts.length ? accounts : [target.codex || {}] - for (const account of rows) { - let access = account.access_token || target.codex?.access_token - const accountId = account.chatgpt_account_id || target.codex?.chatgpt_account_id - const refreshToken = account.refresh_token || target.codex?.refresh_token - const fetchOnce = async () => - await fetchChatgptModelCatalog({ - accessToken: access, - accountId, - proxyUrl, - fetchImpl, - direct, - catalogVersion: catalogVersion.version, - }) - let live = await fetchOnce() - if (live.error === 'upstream_auth' && rotate && refreshToken) { - const tok = await refreshCodexAccessToken({ refreshToken, proxyUrl, fetchImpl }) - if (!tok.ok) { + const accounts = readCodexAccounts(projectRoot, target.id) + const rows = accounts.length ? accounts : [target.codex || {}] + for (const account of rows) { + let access = account.access_token || target.codex?.access_token + const accountId = account.chatgpt_account_id || target.codex?.chatgpt_account_id + const refreshToken = account.refresh_token || target.codex?.refresh_token + const fetchOnce = async () => + await fetchChatgptModelCatalog({ + accessToken: access, + accountId, + proxyUrl, + fetchImpl, + direct, + catalogVersion: catalogVersion.version, + }) + let live = await fetchOnce() + if (live.error === 'upstream_auth' && rotate && refreshToken) { + const tok = await refreshCodexAccessToken({ refreshToken, proxyUrl, fetchImpl }) + if (!tok.ok) { + lastError = { + ok: false, + error: tok.error || 'refresh_failed', + message: 'GPT OAuth 刷新失败，请重新登录', + vm_id: target.id, + } + if (['timeout', 'fetch_failed', 'refresh_empty'].includes(tok.error)) { + executionUncertain = true + return { vmOperationUncertain: true } + } + continue + } + persistRefreshedCodexAccount( + projectRoot, + target.id, + { + access_token: tok.access_token, + refresh_token: tok.refresh_token, + id_token: tok.id_token || account.id_token, + expires_at: tok.expires_at || account.expires_at, + }, + account, + ) + access = tok.access_token + live = await fetchOnce() + } + if (!live.ok || !live.ids?.length) { + const auth = live.error === 'upstream_auth' lastError = { ok: false, - error: tok.error || 'refresh_failed', - message: 'GPT OAuth 刷新失败，请重新登录', + error: live.error || 'empty_catalog', + message: auth ? 'GPT OAuth 已过期，请重新登录' : '同步 GPT 目录失败', vm_id: target.id, } continue } - persistRefreshedCodexAccount( - projectRoot, - target.id, - { - access_token: tok.access_token, - refresh_token: tok.refresh_token, - id_token: tok.id_token || account.id_token, - expires_at: tok.expires_at || account.expires_at, - }, - account, - ) - access = tok.access_token - live = await fetchOnce() - } - if (!live.ok || !live.ids?.length) { - const auth = live.error === 'upstream_auth' - lastError = { - ok: false, - error: live.error || 'empty_catalog', - message: auth ? 'GPT OAuth 已过期，请重新登录' : '同步 GPT 目录失败', - vm_id: target.id, + usedVm = target.id + const rowsLive = live.models?.length ? live.models : live.ids.map((id) => ({ id })) + for (const row of rowsLive) { + const id = String(row?.id || row || '').trim() + if (!id) continue + merged.set(id.toLowerCase(), typeof row === 'object' ? row : { id }) } - continue - } - usedVm = target.id - const rowsLive = live.models?.length ? live.models : live.ids.map((id) => ({ id })) - for (const row of rowsLive) { - const id = String(row?.id || row || '').trim() - if (!id) continue - merged.set(id.toLowerCase(), typeof row === 'object' ? row : { id }) } } + try { + if (rotate && mutations) await mutations.run(listed.id, readTarget, { kind: 'codex_catalog_refresh' }) + else await readTarget() + } catch (error) { + if (!['vm_operation_active', 'vm_execution_active'].includes(error.code)) throw error + lastError = { ok: false, error: error.code, vm_id: listed.id } + } + if (executionUncertain) break } - if (!merged.size) { + if (executionUncertain || !merged.size) { return { ok: false, + ...(executionUncertain ? { executionUncertain: true } : {}), error: lastError?.error || 'empty_catalog', message: lastError?.message || '同步 GPT 目录失败', vm_id: lastError?.vm_id || targets[0].id, diff --git a/src/lib/db/repos/settings-repo.mjs b/src/lib/db/repos/settings-repo.mjs index e5fba389b..42f52c7d9 100644 --- a/src/lib/db/repos/settings-repo.mjs +++ b/src/lib/db/repos/settings-repo.mjs @@ -3,7 +3,7 @@ * Used for: legacy_import_done, backup_schedule, proxy_pool_config, active_vm … */ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' const cacheRevisions = new WeakMap() @@ -47,6 +47,20 @@ export class SettingsRepo { return value } + /** Merge one settings document against its committed value, including first creation. */ + async update(key, change, fallback = null) { + return withTransaction(this.db, async () => { + await this.db + .prepare(`INSERT INTO settings(key,value,updated_at) VALUES ($1,$2,$3) + ON CONFLICT(key) DO NOTHING`) + .run(key, JSON.stringify(fallback), new Date().toISOString()) + const row = await this.db.prepare('SELECT value FROM settings WHERE key=$1 FOR UPDATE').get(key) + const value = change(JSON.parse(row.value)) + await this.set(key, value) + return value + }) + } + async has(key) { return !!(await this._get.get(key)) } diff --git a/src/lib/protocol/gpt-model-policy.mjs b/src/lib/protocol/gpt-model-policy.mjs index 97024cb05..7b3def99e 100644 --- a/src/lib/protocol/gpt-model-policy.mjs +++ b/src/lib/protocol/gpt-model-policy.mjs @@ -3,6 +3,8 @@ * Live ids from ChatGPT sync; seed is empty-catalog fallback. */ import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' +import { getDb } from '../db/database.mjs' +import { SettingsRepo as PersistentSettingsRepo } from '../db/repos/settings-repo.mjs' import { isGptSeriesId, isSyncableGptCatalogId } from './gpt-ids.mjs' const SETTINGS_KEY = 'gpt_model_policy' @@ -56,12 +58,13 @@ export function seedGptPolicy() { let policy = seedGptPolicy() let loaded = false -async function persist() { - try { - await new SettingsRepo().set(SETTINGS_KEY, policy) - } catch (e) { - console.warn('[gpt-model-policy] persist failed', e?.message || e) - } +async function persist(next = policy) { + const db = getDb() + await new PersistentSettingsRepo(db).set(SETTINGS_KEY, next) + db.afterCommit(() => { + policy = next + loaded = true + }) } function normalizeGptPolicy(stored) { @@ -111,7 +114,7 @@ export async function saveGptModelPolicy(next) { models[id] = { ...models[id], ...row, id, family: 'codex' } } } - policy = { + const saved = { ...seed, ...next, models, @@ -119,8 +122,8 @@ export async function saveGptModelPolicy(next) { updated_at: new Date().toISOString(), source: 'panel', } - await persist() - return policy + await persist(saved) + return saved } export function listGptPolicyModels() { return Object.entries(policy.models || {}) @@ -144,10 +147,9 @@ export function isGptModelEnabled(id) { } export async function resetGptModelPolicy() { - policy = seedGptPolicy() - loaded = true - await persist() - return policy + const saved = seedGptPolicy() + await persist(saved) + return saved } /** @@ -170,30 +172,33 @@ export async function syncGptIdsIntoPolicy(ids = [], { displayNameOf = null } = if (!live.length) { return { ok: true, synced: 0, added: [], ids: listGptPolicyModels().map((m) => m.id) } } - const next = {} + const db = getDb() const added = [] - for (const row of live) { - const prev = policy.models[row.id] - if (!prev) added.push(row.id) - next[row.id] = entry(row.id, { - ...prev, - display_name: row.display_name || prev?.display_name || displayNameOf?.(row.id) || row.id, - enabled: prev ? prev.enabled !== false : false, - source: 'codex', - }) - } - policy = { - ...policy, - models: next, - platform: 'openai', - updated_at: new Date().toISOString(), - source: 'codex', - } - await persist() + const saved = await new PersistentSettingsRepo(db).update(SETTINGS_KEY, (current) => { + const previous = normalizeGptPolicy(current) || seedGptPolicy() + const models = {} + for (const row of live) { + const prev = previous.models[row.id] + if (!prev) added.push(row.id) + models[row.id] = entry(row.id, { + ...prev, + display_name: row.display_name || prev?.display_name || displayNameOf?.(row.id) || row.id, + enabled: prev ? prev.enabled !== false : false, + source: 'codex', + }) + } + return { ...previous, models, platform: 'openai', updated_at: new Date().toISOString(), source: 'codex' } + }) + db.afterCommit(() => { + policy = saved + loaded = true + }) return { ok: true, synced: live.length, added, - ids: listGptPolicyModels().map((m) => m.id), + ids: Object.values(saved.models) + .filter((row) => row.enabled !== false) + .map((row) => row.id), } } diff --git a/src/lib/protocol/model-policy.mjs b/src/lib/protocol/model-policy.mjs index eb5561427..d9e8ae6cb 100644 --- a/src/lib/protocol/model-policy.mjs +++ b/src/lib/protocol/model-policy.mjs @@ -9,6 +9,8 @@ */ import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' +import { getDb } from '../db/database.mjs' +import { SettingsRepo as PersistentSettingsRepo } from '../db/repos/settings-repo.mjs' import { CONTEXT_1M_BETA, DEFAULT_CONTEXT_1M_WHITELIST, @@ -636,27 +638,27 @@ export function getModelPolicy() { } export async function saveModelPolicy(next) { - policy = normalizePolicy(next) - policy.updated_at = new Date().toISOString() - policy.source = 'panel' - try { - const repo = new SettingsRepo() - await repo.set(SETTINGS_KEY, policy) - } catch (e) { - console.warn('[model-policy] persist failed', e?.message || e) - } - loaded = true - return policy + const saved = normalizePolicy(next) + saved.updated_at = new Date().toISOString() + saved.source = 'panel' + const db = getDb() + await new PersistentSettingsRepo(db).set(SETTINGS_KEY, saved) + db.afterCommit(() => { + policy = saved + loaded = true + }) + return saved } export async function resetModelPolicy() { - policy = seedDefaultPolicy() - try { - const repo = new SettingsRepo() - await repo.set(SETTINGS_KEY, policy) - } catch {} - loaded = true - return policy + const saved = seedDefaultPolicy() + const db = getDb() + await new PersistentSettingsRepo(db).set(SETTINGS_KEY, saved) + db.afterCommit(() => { + policy = saved + loaded = true + }) + return saved } export function resolvePolicyModelId(raw = '') { @@ -981,46 +983,52 @@ export function listPolicyModels() { * `displayNameOf(id)` may supply a display name (models.dev) for new entries. */ export async function syncWorkerModelsIntoPolicy(workerIds = [], { displayNameOf = null } = {}) { + const db = getDb() const added = [] - const enabled = policy.catalog_mode === 'worker_only' - for (const workerId of workerIds || []) { - const id = workerId === FABLE_51_LEGACY_ID ? FABLE_51_ID : workerId === OPUS_55_LEGACY_ID ? OPUS_55_ID : workerId - if (!id || !isCatalogModelId(id)) continue - if (/^gpt/i.test(id)) continue - if (policy.models[id]) continue - const family = familyOfModelId(id) - const caps = heuristicCapabilities(id) - policy.models[id] = entry({ - id, - display_name: displayNameOf?.(id) || id, - family, - capabilities: caps, - betas: family === 'codex' ? { required: [], drop: [CONTEXT_1M], pass_context_1m: false } : undefined, - params: - family === 'codex' - ? { - max_tokens_default: 16384, - max_tokens_cap: 128000, - on_adaptive: 'passthrough', - on_enabled: 'passthrough', - } - : { - on_adaptive: caps.supports_adaptive ? 'passthrough' : 'convert_to_enabled', - on_enabled: 'passthrough', - }, - }) - policy.models[id].enabled = enabled - // pass_context_1m left unset → inherit defaults.context_1m_whitelist - added.push(id) - } - if (added.length) { - policy.updated_at = new Date().toISOString() - policy.source = 'worker_merge' - try { - await new SettingsRepo().set(SETTINGS_KEY, policy) - } catch {} - } - return { policy, added } + const next = await new PersistentSettingsRepo(db).update(SETTINGS_KEY, (current) => { + const next = normalizePolicy(current || seedDefaultPolicy()) + const enabled = next.catalog_mode === 'worker_only' + for (const workerId of workerIds || []) { + const id = workerId === FABLE_51_LEGACY_ID ? FABLE_51_ID : workerId === OPUS_55_LEGACY_ID ? OPUS_55_ID : workerId + if (!id || !isCatalogModelId(id)) continue + if (/^gpt/i.test(id)) continue + if (next.models[id]) continue + const family = familyOfModelId(id) + const caps = heuristicCapabilities(id) + next.models[id] = entry({ + id, + display_name: displayNameOf?.(id) || id, + family, + capabilities: caps, + betas: family === 'codex' ? { required: [], drop: [CONTEXT_1M], pass_context_1m: false } : undefined, + params: + family === 'codex' + ? { + max_tokens_default: 16384, + max_tokens_cap: 128000, + on_adaptive: 'passthrough', + on_enabled: 'passthrough', + } + : { + on_adaptive: caps.supports_adaptive ? 'passthrough' : 'convert_to_enabled', + on_enabled: 'passthrough', + }, + }) + next.models[id].enabled = enabled + // pass_context_1m left unset → inherit defaults.context_1m_whitelist + added.push(id) + } + if (added.length) { + next.updated_at = new Date().toISOString() + next.source = 'worker_merge' + } + return next + }) + db.afterCommit(() => { + policy = next + loaded = true + }) + return { policy: next, added } } /** Last-resort fill when the model has no policy default. Probe / short tests use this. */ From 02803edae1227adf6c569535b1fda673919dc34b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:41:59 +0800 Subject: [PATCH 248/473] feat: persist proxy pool management operation claims --- src/lib/vm/proxy-pool.mjs | 2 + src/lib/vm/shared-proxy-operations.mjs | 190 +++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 src/lib/vm/shared-proxy-operations.mjs diff --git a/src/lib/vm/proxy-pool.mjs b/src/lib/vm/proxy-pool.mjs index f9ea75eec..0e6d24aae 100644 --- a/src/lib/vm/proxy-pool.mjs +++ b/src/lib/vm/proxy-pool.mjs @@ -18,6 +18,7 @@ import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.m import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { currentVmOperationIds } from './shared-vm-operations.mjs' import { physicalVmAliases } from './vm-physical-target.mjs' +import { assertCurrentProxyOperation } from './shared-proxy-operations.mjs' import { canBindProxyToVm, normalizeOwnerId, proxyOwnerId, vmOwnerId } from '../admin/resource-owner.mjs' import { validTimezone } from '../core/timezone.mjs' import { lookupProxyGeo, lookupProxyGeoV6 } from './proxy-geo.mjs' @@ -432,6 +433,7 @@ export class ProxyPool extends EventEmitter { _mutate(change, { observation = false } = {}) { const operation = withTransaction(this.db, async () => { await this.repo.lockMutations() + if (!observation && isKubernetesMode()) await assertCurrentProxyOperation(this.db) const snapshot = await this.repo.snapshot() const working = { config: { ...DEFAULT_CONFIG, ...snapshot.config }, proxies: snapshot.proxies.map(hydrateProxy) } const before = structuredClone(working) diff --git a/src/lib/vm/shared-proxy-operations.mjs b/src/lib/vm/shared-proxy-operations.mjs new file mode 100644 index 000000000..131b414e0 --- /dev/null +++ b/src/lib/vm/shared-proxy-operations.mjs @@ -0,0 +1,190 @@ +import { randomUUID } from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' +import { getDb, withTransaction } from '../db/database.mjs' +import { ProxiesRepo } from '../db/repos/proxies-repo.mjs' +import { BackgroundTaskRunsRepo } from '../db/repos/background-task-runs-repo.mjs' +import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' +import { currentVmOperationIds } from './shared-vm-operations.mjs' +import { physicalVmAliases } from './vm-physical-target.mjs' +import { parseBoundVmIds } from './proxy-pool.mjs' + +const TASK_KEY = 'proxy_operation:pool' +const callbacks = new AsyncLocalStorage() +const services = new WeakMap() + +function error(code, status = 409) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code, status }) +} + +function contextFor(db) { + return (callbacks.getStore() || []).findLast( + (context) => + context.active && + !context.claim.uncertain && + context.db.schema === db.schema && + context.db.connectionString === db.connectionString, + ) +} + +/** Typed control operations use one pool claim, including when no VM is bound yet. */ +export class SharedProxyOperations { + constructor(db, { ownerId = randomUUID() } = {}) { + if (typeof ownerId !== 'string' || !ownerId.trim() || ownerId.length > 200) + throw error('invalid_proxy_operation', 400) + this.db = db + this.ownerId = ownerId + this.runs = new BackgroundTaskRunsRepo(db) + } + + async tryAcquire() { + if (this.db.isTransaction) throw error('proxy_operation_transaction_active', 500) + return withTransaction(this.db, async () => { + await new ProxiesRepo(this.db).lockMutations() + const active = await this.db + .prepare('SELECT execution_id FROM background_task_runs WHERE task_key = $1 AND finished_at IS NULL') + .get(TASK_KEY) + if (active) return { ok: false, code: 'proxy_operation_active' } + // Older callers can already hold a VM claim before reaching their proxy work. + const rows = await this.db.prepare('SELECT bound_vm_id FROM proxies WHERE bound_vm_id IS NOT NULL').all() + const ids = [...new Set(rows.flatMap((row) => parseBoundVmIds(row.bound_vm_id)))] + if (ids.length) { + await new ExecutionAdmissionsRepo(this.db).lockResources({ vmIds: ids }) + const current = currentVmOperationIds(this.db) + const operations = await this.db + .prepare('SELECT id, uncertain_at FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL') + .all([...new Set(ids.flatMap(physicalVmAliases))]) + if (operations.some((operation) => !current.has(operation.id) || operation.uncertain_at)) + return { ok: false, code: 'vm_operation_active' } + } + const identity = { taskKey: TASK_KEY, executionId: randomUUID(), ownerId: this.ownerId } + await this.db + .prepare('INSERT INTO background_task_runs (task_key, execution_id, owner_id) VALUES ($1, $2, $3)') + .run(identity.taskKey, identity.executionId, identity.ownerId) + return this._handle(identity) + }) + } + + _handle(identity) { + let uncertain = false + let closed = false + let marking = null + let releasing = null + const claim = { + ok: true, + ...identity, + get uncertain() { + return uncertain + }, + run: async (callback) => { + if (closed || uncertain) throw error('proxy_operation_inactive') + const parents = callbacks.getStore() || [] + const context = { db: this.db, identity, claim, active: true } + try { + return await callbacks.run([...parents, context], () => callback(claim)) + } finally { + context.active = false + } + }, + markUncertain: () => { + uncertain = true + if (!marking) { + const pending = this.runs.markUnknown(identity) + marking = pending + pending.catch(() => { + if (marking === pending) marking = null + }) + } + return marking + }, + release: async () => { + if (uncertain) { + if (marking) await marking + return { released: false } + } + if (!releasing) { + const pending = this.runs.complete(identity).then((result) => { + if (result.completed) closed = true + return { released: result.completed } + }) + releasing = pending + pending.then( + (result) => { + if (!result.released && releasing === pending) releasing = null + }, + () => { + if (releasing === pending) releasing = null + }, + ) + } + return releasing + }, + } + return claim + } + + async run(callback) { + const parent = contextFor(this.db) + if (parent) { + let nested + try { + nested = await callback(parent.claim) + } catch (failure) { + if (failure?.proxyOperationNotStarted !== true) await parent.claim.markUncertain() + throw failure + } + if (nested?.terminal !== true && nested?.notStarted !== true) await parent.claim.markUncertain() + return nested?.value + } + const claim = await this.tryAcquire() + if (!claim.ok) throw error(claim.code) + let result + try { + result = await claim.run(callback) + } catch (failure) { + if (failure?.proxyOperationNotStarted === true && !claim.uncertain) { + if (!(await claim.release()).released) throw error('proxy_operation_release_unconfirmed') + } else await claim.markUncertain() + throw failure + } + if (claim.uncertain || (result?.terminal !== true && result?.notStarted !== true)) await claim.markUncertain() + else if (!(await claim.release()).released) throw error('proxy_operation_release_unconfirmed') + return result?.value + } +} + +/** Called under ProxiesRepo.lockMutations, so a claim cannot race a binding transaction. */ +export async function assertCurrentProxyOperation(db) { + if (!db.isTransaction) throw error('proxy_operation_transaction_required', 500) + const row = await db + .prepare( + 'SELECT task_key, execution_id, owner_id, unknown_at FROM background_task_runs WHERE task_key = $1 AND finished_at IS NULL', + ) + .get(TASK_KEY) + if (!row) { + if ( + (callbacks.getStore() || []).some( + (context) => context.db.schema === db.schema && context.db.connectionString === db.connectionString, + ) + ) + throw error('proxy_operation_inactive') + return + } + const current = contextFor(db)?.identity + if ( + row.unknown_at || + !current || + current.taskKey !== row.task_key || + current.executionId !== row.execution_id || + current.ownerId !== row.owner_id + ) + throw error('proxy_operation_active') +} + +export function markCurrentProxyOperationUncertain(db) { + return contextFor(db)?.claim.markUncertain() +} + +export function getSharedProxyOperations(db = getDb()) { + if (!services.has(db)) services.set(db, new SharedProxyOperations(db)) + return services.get(db) +} From 77477cb6d658f7a4fb88ad59e6b634363f7c6210 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:43:46 +0800 Subject: [PATCH 249/473] test: require pool ownership for idle proxy binding and cleanup --- test/unit/panel-proxy-vm-mutations.test.mjs | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 003f6396c..6d8c13191 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -9,6 +9,7 @@ import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedProxyOperations } from '../../src/lib/vm/shared-proxy-operations.mjs' async function fixture(t) { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-proxy-claims-')) @@ -153,6 +154,37 @@ test('a failed worker reload after proxy editing keeps the target claim unknown' assert.equal(h.response.status, 200) assert.equal(h.response.body.data.workers[0].ok, false) assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-a' })).ok, false) + assert.equal((await new SharedProxyOperations(f.peer).tryAcquire()).ok, false) +}) + +test('an idle proxy binding owns the new target and releases both persistent claims after completion', async (t) => { + const f = await fixture(t) + f.seed('vm-b') + const h = f.handler({ vm_id: 'vm-b' }, { vmMutationOps: { ensureProxyEgress: async () => ({ ok: true }) } }) + await h.request('POST', '/api/panel/proxies/' + f.proxyId + '/bind') + assert.equal(h.response.status, 200, JSON.stringify(h.response.body)) + assert.deepEqual(f.pool.getProxyByIdWithAuth(f.proxyId).bound_vm_ids, ['vm-a', 'vm-b']) + const vmClaim = await f.otherOperations.tryAcquire({ vmId: 'vm-b' }) + assert.equal(vmClaim.ok, true) + await vmClaim.release() + const proxyClaim = await new SharedProxyOperations(f.peer).tryAcquire() + assert.equal(proxyClaim.ok, true) + await proxyClaim.release() +}) + +test('unknown cleanup of an empty proxy keeps its pool claim without inventing a VM target', async (t) => { + const f = await fixture(t) + process.env.VM2API_CONTROL_PLANE_MODE = 'standalone' + await f.pool.unbind(f.proxyId) + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + const h = f.handler( + {}, + { vmMutationOps: { stopProxyEgress: async () => ({ ok: false, error: 'synthetic unknown' }) } }, + ) + await h.request('DELETE', '/api/panel/proxies/' + f.proxyId) + assert.equal(h.response.status, 502) + assert.equal((await new SharedProxyOperations(f.peer).tryAcquire()).ok, false) + assert.equal((await f.db.prepare('SELECT count(*) AS n FROM vm_operations').get()).n, 0) }) test('proxy unbind awaits confirmed helper stop before releasing its VM claim', async (t) => { From 04a016b236de63e132511108efa6083779b86eeb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:43:54 +0800 Subject: [PATCH 250/473] test: reproduce monitor startup queries after bootstrap activity closes --- .../unit/maintenance-monitor-startup.test.mjs | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 test/unit/maintenance-monitor-startup.test.mjs diff --git a/test/unit/maintenance-monitor-startup.test.mjs b/test/unit/maintenance-monitor-startup.test.mjs new file mode 100644 index 000000000..3ad6d7352 --- /dev/null +++ b/test/unit/maintenance-monitor-startup.test.mjs @@ -0,0 +1,183 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import pg from 'pg' +import { getDb, registerActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { SharedRoutingConfig } from '../../src/lib/admin/shared-routing-config.mjs' +import { RuntimeSettings, loadSettingsState } from '../../src/lib/db/settings-state.mjs' +import { createCredentialRefreshMonitor } from '../../src/lib/oauth/credential-refresh-monitor.mjs' +import { createHealthProbeMonitor } from '../../src/lib/admin/health-probe.mjs' +import { createUsageProbeMonitor } from '../../src/lib/oauth/usage-probe-monitor.mjs' +import { VmTestScheduleRunner } from '../../src/lib/admin/vm-test-schedule.mjs' + +const vm = { + id: 'scope-vm', + status: 'running', + schedulable: true, + has_token: true, + has_refresh: true, + expires_at: 1, + proxy: { url: 'socks5h://127.0.0.1:19099' }, +} +const plan = { + enabled: true, + frequency: 'daily', + time: '09:00', + timezone: 'UTC', + cron: '0 9 * * *', + revision: 'synthetic-revision', + model: 'claude-haiku-4-5', + prompt: 'synthetic', + max_tokens: 8, +} + +async function fixture(t) { + const db = getDb() + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-monitor-startup-')) + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + registerActivityCoordinator(db, coordinator) + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(async () => { + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + await coordinator.stop() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + fs.rmSync(root, { recursive: true, force: true }) + } + }) + const file = path.join(root, 'routing.json') + fs.writeFileSync( + file, + JSON.stringify({ + credential_refresh: { enabled: true }, + health_probe: { enabled: true }, + usage_probe: { enabled: true }, + }), + ) + const routing = new SharedRoutingConfig({ + db, + file, + normalize: (x) => x, + merge: (a, b) => ({ ...a, ...b }), + apply: () => ({ status: 'applied' }), + }) + const bootstrap = await coordinator.admitBootstrap() + await bootstrap.run(async () => { + await routing.init() + await loadSettingsState(db) + await new RuntimeSettings(db).set('vm_test_schedule:scope-vm', plan) + }) + await coordinator.acknowledgeEpoch(bootstrap.epoch) + await bootstrap.complete({ ended: true }) + await assert.rejects(db.query('SELECT 1'), { code: 'maintenance_activity_closed' }) + return { db, root, coordinator, tasks: new SharedBackgroundTasks(db) } +} + +test('credential refresh obtains a fresh activity after initialization has ended', async (t) => { + const { db, tasks } = await fixture(t) + let calls = 0 + const monitor = createCredentialRefreshMonitor({ + sharedTasks: tasks, + config: { enabled: true }, + listTargets: () => [vm], + refreshOne: async () => { + await db.query('SELECT 1') + calls++ + return { ok: true } + }, + }) + t.after(() => monitor.stop()) + await monitor.runOnce() + assert.equal(calls, 1) +}) + +test('health probing obtains a fresh activity after initialization has ended', async (t) => { + const { db, tasks } = await fixture(t) + let calls = 0 + const monitor = createHealthProbeMonitor({ + sharedTasks: tasks, + config: { enabled: true }, + listTargets: () => [vm], + runChat: async () => { + await db.query('SELECT 1') + calls++ + return { + ok: true, + text: 'synthetic', + body: { id: 'msg_synthetic', content: [{ type: 'text', text: 'synthetic' }] }, + } + }, + }) + t.after(() => monitor.stop()) + await monitor.runRealProbe() + assert.equal(calls, 1) +}) + +test('usage probing obtains a fresh activity before configuration refresh and preparation', async (t) => { + const { db, tasks } = await fixture(t) + let calls = 0 + const monitor = createUsageProbeMonitor({ + sharedTasks: tasks, + config: { enabled: true }, + listTargets: () => [vm], + accountForVm: () => ({}), + reconcile: async () => { + await db.query('SELECT 1') + }, + probeOne: async () => { + await db.query('SELECT 1') + calls++ + return { ok: true } + }, + }) + t.after(() => monitor.stop()) + await monitor.runOnce() + assert.equal(calls, 1) +}) + +test('VM schedule initial and subsequent discovery use new activities after initialization', async (t) => { + const { db, tasks, root } = await fixture(t) + const runner = new VmTestScheduleRunner({ + projectRoot: root, + sharedTasks: tasks, + settings: () => new RuntimeSettings(db), + list: () => [vm], + runTest: async () => ({ ok: true }), + }) + t.after(() => runner.stop()) + await runner.start() + assert.equal(runner.jobs.has(vm.id), true) + await runner.reload() + assert.equal(runner.jobs.has(vm.id), true) +}) + +test('VM schedule cron preparation enters a fresh activity before refreshing settings', async (t) => { + const { db, tasks, root } = await fixture(t) + let calls = 0 + const runner = new VmTestScheduleRunner({ + projectRoot: root, + sharedTasks: tasks, + settings: () => new RuntimeSettings(db), + list: () => [vm], + runTest: async () => { + await db.query('SELECT 1') + calls++ + return { ok: true } + }, + }) + t.after(() => runner.stop()) + runner.started = true + await runner._run(vm.id, plan.revision) + assert.equal(calls, 1) +}) From b9fc7462e4f282a05afada43636c7e56e94b3e54 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:45:10 +0800 Subject: [PATCH 251/473] fix: hold persistent claims across panel proxy effects --- src/lib/admin/panel-routes.mjs | 729 ++++++++++++-------- test/unit/panel-proxy-vm-mutations.test.mjs | 2 +- 2 files changed, 442 insertions(+), 289 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 6a2b0cabb..56f9ebf1c 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -174,6 +174,8 @@ import * as panel from './panel-api.mjs' import { GATEWAY_CAPABILITIES } from '../vm/execution-context.mjs' import { withVmLock, atomicWriteJson, isValidVmId, invalidateVmSlotReadCache } from '../vm/vm-file.mjs' import { withVmMutations } from '../vm/vm-mutations.mjs' +import { getSharedProxyOperations, markCurrentProxyOperationUncertain } from '../vm/shared-proxy-operations.mjs' +import { parseBoundVmIds } from '../vm/proxy-pool.mjs' import { physicalVmAliases } from '../vm/vm-physical-target.mjs' import { snapshotDatabaseMetrics } from '../db/database-metrics.mjs' import { getUsageCache } from '../oauth/usage-cache.mjs' @@ -640,6 +642,7 @@ export function createPanelHandler(ctx) { result = await callback() } catch (error) { await operation?.markUncertain?.() + await markCurrentProxyOperationUncertain(getDb()) throw error } if ( @@ -649,9 +652,39 @@ export function createPanelHandler(ctx) { (result?.ok !== true && result?.notStarted !== true && result?.vmOperationNotStarted !== true) ) { await operation?.markUncertain?.() + await markCurrentProxyOperationUncertain(getDb()) } return result } + async function mutateProxyVms(ids, callback) { + if (!isKubernetesMode() && !ctx.proxyOperations) return mutateVms(ids, callback) + return (ctx.proxyOperations || getSharedProxyOperations()).run(async () => { + await ctx.proxyPool.refresh?.() + const bindings = await getDb().prepare('SELECT bound_vm_id FROM proxies WHERE bound_vm_id IS NOT NULL').all() + const targets = [...new Set([...ids, ...bindings.flatMap((proxy) => parseBoundVmIds(proxy.bound_vm_id))])] + let entered = false + try { + const value = await mutateVms(targets, async (handles) => { + entered = true + return callback(handles) + }) + return { terminal: true, value } + } catch (error) { + if (!entered) error.proxyOperationNotStarted = true + throw error + } + }) + } + function proxyEffect(handles, callback) { + return vmEffect( + { + markUncertain: async () => { + for (const operation of handles.values()) await operation.markUncertain() + }, + }, + callback, + ) + } const hostUnsupported = (res, vm, cap) => { if (slotHost(vm).supports(cap)) return false json(res, 409, { @@ -3404,7 +3437,7 @@ export function createPanelHandler(ctx) { // DELETE /api/panel/vms/:id — remove VM record, cli-home, unbind proxy if (req.method === 'DELETE' && /^\/api\/panel\/vms\/[^/]+$/.test(p)) { const id = p.split('/')[4] - return mutateVms([id], async (handles) => { + return mutateProxyVms([id], async (handles) => { if (!isValidVmId(id)) { return json(res, 400, { ok: false, error: { message: 'invalid vm id' } }) } @@ -4079,7 +4112,7 @@ export function createPanelHandler(ctx) { const rawId = body.id || 'vm-' + padVm(idx) const id = String(rawId).replace(/[^a-zA-Z0-9_-]/g, '') if (!isValidVmId(id)) return json(res, 400, { ok: false, error: { message: 'invalid id' } }) - return mutateVms([id], async (handles) => { + return mutateProxyVms([id], async (handles) => { const existing = listVms(cfg.paths.project) const aliases = physicalVmAliases(id) const committed = @@ -4448,7 +4481,7 @@ export function createPanelHandler(ctx) { // POST /api/panel/vms/:id/start if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/start$/.test(p)) { const id = p.split('/')[4] - return mutateVms([id], async (handles) => { + return mutateProxyVms([id], async (handles) => { if (!getVm(cfg.paths.project, id)) return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'vm not found' } }) const started = await vmEffect(handles.get(id), () => @@ -4496,7 +4529,7 @@ export function createPanelHandler(ctx) { // POST /api/panel/vms/:id/recover — power cycle (stop → start) plus a fresh diagnosis if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/recover$/.test(p)) { const id = p.split('/')[4] - return mutateVms([id], async (handles) => { + return mutateProxyVms([id], async (handles) => { if (!getVm(cfg.paths.project, id)) return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'vm not found' } }) const recovered = await vmEffect(handles.get(id), () => @@ -5684,51 +5717,73 @@ export function createPanelHandler(ctx) { return json(res, 200, panel.ok(snap)) } if (req.method === 'POST' && p === '/api/panel/proxies/local') { - const result = await proxyPool.ensureLocal() - if (result.created) await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(result.proxy.id)) - // Enabling local egress on a cluster that already has nodes: each VPS gets its row now. - await proxyPool.syncLocalNodes(clusterNodesOf(ctx).map((node) => node.id)) - return json(res, 200, panel.ok(result)) + return mutateProxyVms([], async (handles) => { + const result = await proxyPool.ensureLocal() + if (result.created) { + const egress = await proxyEffect(handles, () => + (ctx.vmMutationOps?.ensureProxyEgress || ensureProxyEgress)( + cfg.paths.project, + proxyPool.getProxyByIdWithAuth(result.proxy.id), + ), + ) + if (!egress.ok) + return json(res, 502, { + ok: false, + error: { code: egress.code || 'egress_start_unknown', message: egress.error }, + }) + } + // Enabling local egress on a cluster that already has nodes: each VPS gets its row now. + await proxyPool.syncLocalNodes(clusterNodesOf(ctx).map((node) => node.id)) + return json(res, 200, panel.ok(result)) + }) } if (req.method === 'POST' && p === '/api/panel/proxies/import') { const body = await readBody(req, 2 * 1024 * 1024) - const text = body.text || body.lines || (Array.isArray(body) ? body.join('\n') : '') - const ident = panelIdentity(req) - const bindVmId = String(body.bind_vm_id || body.vm_id || '').trim() - if (bindVmId && userMissesVmId(req, cfg.paths.project, bindVmId)) { - return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - const result = await proxyPool.importLines(text, { - fields: body.proxies || body.entries || null, - host: body.host, - port: body.port, - username: body.username ?? body.user, - password: body.password ?? body.pass, - ownerUserId: ident.role === 'user' ? req.panelUserId : null, - }) - let bound = null - let worker = null - let timezone = null - if (bindVmId && result.items?.[0]?.id) { - const bind = await proxyPool.bind(result.items[0].id, bindVmId) - if (bind.ok) { - bindVmProxy(cfg.paths.project, bindVmId, proxyPool.getProxyForVm(bindVmId)) - bound = bind.proxy - // Before the reload: the worker reads vm.timezone when it starts. - timezone = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, bindVmId, { - proxyId: result.items[0].id, - }) - const vm = getVm(cfg.paths.project, bindVmId) - if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { - setVmSchedulable(cfg.paths.project, bindVmId, false, 'proxy_rebind_worker_reload') - worker = await reloadSlotReady(getVm(cfg.paths.project, bindVmId), cfg.paths.project, { - routing: ctx.routingConfig, + return mutateProxyVms([String(body.bind_vm_id || body.vm_id || '').trim()].filter(Boolean), async (handles) => { + const text = body.text || body.lines || (Array.isArray(body) ? body.join('\n') : '') + const ident = panelIdentity(req) + const bindVmId = String(body.bind_vm_id || body.vm_id || '').trim() + if (bindVmId && userMissesVmId(req, cfg.paths.project, bindVmId)) { + return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + const result = await proxyPool.importLines(text, { + fields: body.proxies || body.entries || null, + host: body.host, + port: body.port, + username: body.username ?? body.user, + password: body.password ?? body.pass, + ownerUserId: ident.role === 'user' ? req.panelUserId : null, + }) + let bound = null + let worker = null + let timezone = null + if (bindVmId && result.items?.[0]?.id) { + const bind = await proxyPool.bind(result.items[0].id, bindVmId) + if (bind.ok) { + bindVmProxy(cfg.paths.project, bindVmId, proxyPool.getProxyForVm(bindVmId)) + bound = bind.proxy + // Before the reload: the worker reads vm.timezone when it starts. + timezone = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, bindVmId, { + proxyId: result.items[0].id, }) - if (worker.ok) restoreSchedulableIfReady(bindVmId) + const vm = getVm(cfg.paths.project, bindVmId) + if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { + setVmSchedulable(cfg.paths.project, bindVmId, false, 'proxy_rebind_worker_reload') + worker = await vmEffect(handles.get(bindVmId), () => + (ctx.vmMutationOps?.reloadSlotReady || reloadSlotReady)( + getVm(cfg.paths.project, bindVmId), + cfg.paths.project, + { + routing: ctx.routingConfig, + }, + ), + ) + if (worker.ok) restoreSchedulableIfReady(bindVmId) + } } } - } - return json(res, 200, panel.ok({ ...result, bound, worker, timezone })) + return json(res, 200, panel.ok({ ...result, bound, worker, timezone })) + }) } if (req.method === 'POST' && p === '/api/panel/proxies/geo') { const body = await readBody(req, 8192).catch(() => ({})) @@ -5749,95 +5804,121 @@ export function createPanelHandler(ctx) { } if (req.method === 'PUT' && p === '/api/panel/proxies/config') { const body = await readBody(req, 64 * 1024) - const previousConfig = proxyPool.snapshot().config - const result = await proxyPool.updateConfig(body) - if (!result.ok) - return json(res, 400, { - ok: false, - error: { type: 'invalid_request_error', code: result.error, message: result.error, details: result }, - }) - // DNS changes must reach running egress helpers; slots stay intact. - const egress = [] - if ( - ((body.dns_primary != null && body.dns_primary !== previousConfig.dns_primary) || - (body.dns_disable_svcb_https != null && - body.dns_disable_svcb_https !== previousConfig.dns_disable_svcb_https)) && - egressEnabled() && - process.env.KIN_CRS_MOCK !== '1' - ) { - const dnsUpstream = dnsUpstreamChain(result.config.dns_primary) - const dnsEmptyTypes = result.config.dns_disable_svcb_https ? [64, 65] : [] - for (const proxy of proxyPool.snapshot().proxies) { - if (isLocalEgressProxy(proxy) || !proxy.bound_vm_ids?.length) continue - const r = await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(proxy.id), { - dnsUpstream, - dnsEmptyTypes, - }) - egress.push({ proxy_id: proxy.id, ok: r.ok, error: r.ok ? null : r.error }) - } - } - if (body.ipv6_enabled != null && process.env.KIN_CRS_MOCK !== '1') { - egress.push(...(await syncIpv6ProxyEgress(cfg.paths.project, proxyPool))) - } - return json(res, 200, panel.ok({ ...result.config, egress })) + return mutateProxyVms( + body.ipv6_enabled != null ? listVms(cfg.paths.project).map((vm) => vm.id) : [], + async (handles) => { + const previousConfig = proxyPool.snapshot().config + const result = await proxyPool.updateConfig(body) + if (!result.ok) + return json(res, 400, { + ok: false, + error: { type: 'invalid_request_error', code: result.error, message: result.error, details: result }, + }) + // DNS changes must reach running egress helpers; slots stay intact. + const egress = [] + if ( + ((body.dns_primary != null && body.dns_primary !== previousConfig.dns_primary) || + (body.dns_disable_svcb_https != null && + body.dns_disable_svcb_https !== previousConfig.dns_disable_svcb_https)) && + egressEnabled() && + process.env.KIN_CRS_MOCK !== '1' + ) { + const dnsUpstream = dnsUpstreamChain(result.config.dns_primary) + const dnsEmptyTypes = result.config.dns_disable_svcb_https ? [64, 65] : [] + for (const proxy of proxyPool.snapshot().proxies) { + if (isLocalEgressProxy(proxy) || !proxy.bound_vm_ids?.length) continue + const r = await proxyEffect(handles, () => + (ctx.vmMutationOps?.ensureProxyEgress || ensureProxyEgress)( + cfg.paths.project, + proxyPool.getProxyByIdWithAuth(proxy.id), + { + dnsUpstream, + dnsEmptyTypes, + }, + ), + ) + egress.push({ proxy_id: proxy.id, ok: r.ok, error: r.ok ? null : r.error }) + } + } + if (body.ipv6_enabled != null && process.env.KIN_CRS_MOCK !== '1') { + const results = await syncIpv6ProxyEgress(cfg.paths.project, proxyPool) + await proxyEffect(handles, async () => ({ ok: results.every((result) => result.ok) })) + egress.push(...results) + } + return json(res, 200, panel.ok({ ...result.config, egress })) + }, + ) } // Must stay BELOW /proxies/config: `[^/]+` matches "config" too, and this // route shares its method, so ordering alone decides the winner. The // negative lookahead makes that dependency explicit rather than positional. if (req.method === 'PUT' && /^\/api\/panel\/proxies\/(?!config$)[^/]+$/.test(p)) { - const id = p.split('/')[4] - const body = await readBody(req, 64 * 1024) - // Forward only the keys the caller actually sent — update() reads - // presence, not value, to tell "leave alone" from "clear". - const patch = {} - for (const key of ['host', 'port', 'username', 'password', 'label', 'domain_forward']) { - if (Object.prototype.hasOwnProperty.call(body, key)) patch[key] = body[key] - } - const result = await proxyPool.update(id, patch) - if (!result.ok) { - const status = result.error === 'proxy_not_found' ? 404 : 400 - const type = status === 404 ? 'not_found_error' : 'invalid_request_error' - return json(res, status, { ok: false, error: { type, code: result.error, message: result.error } }) - } - // Hostname mode and SOCKS upstream both live inside kin-egress. - // Domain-forward-only: restart the helper; slots keep the same bridge. - // Same-id connection change: network name is unchanged so reload would - // only docker restart — the helper must pick up the new proxy_url or - // transparent traffic keeps the old exit. - let egress = null - if ( - (result.domain_forward_changed || result.connection_changed) && - egressEnabled() && - process.env.KIN_CRS_MOCK !== '1' && - (result.proxy.bound_vm_ids || []).length - ) { - const restarted = await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(id)) - egress = { ok: !!restarted.ok, error: restarted.ok ? null : restarted.error || 'egress_restart_failed' } - } - // A label is display-only; reloading every bound worker for it would - // pull live slots out of scheduling for nothing. - if (!result.connection_changed) return json(res, 200, panel.ok({ proxy: result.proxy, workers: [], egress })) - // The pool store is only one of three places the credentials live - // (pool -> vms/.json -> worker.json). Without this the edit looks - // like it worked while every bound slot keeps dialing the old proxy. - const workers = [] - for (const vmId of result.proxy.bound_vm_ids || []) { - bindVmProxy(cfg.paths.project, vmId, proxyPool.getProxyByIdWithAuth(id)) - const vm = getVm(cfg.paths.project, vmId) - if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { - setVmSchedulable(cfg.paths.project, vmId, false, 'proxy_edit_worker_reload') - const reloaded = await reloadSlotReady(getVm(cfg.paths.project, vmId), cfg.paths.project, { - routing: ctx.routingConfig, - }) - if (reloaded.ok) restoreSchedulableIfReady(vmId) - // A slot that fails to reload does not fail the request: the store - // is already updated, so report per-slot and let ops retry. - workers.push({ vm_id: vmId, ok: !!reloaded.ok, error: reloaded.ok ? null : reloaded.error || null }) - } else { - workers.push({ vm_id: vmId, ok: true, error: null }) + return mutateProxyVms([], async (handles) => { + const id = p.split('/')[4] + const body = await readBody(req, 64 * 1024) + // Forward only the keys the caller actually sent — update() reads + // presence, not value, to tell "leave alone" from "clear". + const patch = {} + for (const key of ['host', 'port', 'username', 'password', 'label', 'domain_forward']) { + if (Object.prototype.hasOwnProperty.call(body, key)) patch[key] = body[key] } - } - return json(res, 200, panel.ok({ proxy: result.proxy, workers, egress })) + const result = await proxyPool.update(id, patch) + if (!result.ok) { + const status = result.error === 'proxy_not_found' ? 404 : 400 + const type = status === 404 ? 'not_found_error' : 'invalid_request_error' + return json(res, status, { ok: false, error: { type, code: result.error, message: result.error } }) + } + // Hostname mode and SOCKS upstream both live inside kin-egress. + // Domain-forward-only: restart the helper; slots keep the same bridge. + // Same-id connection change: network name is unchanged so reload would + // only docker restart — the helper must pick up the new proxy_url or + // transparent traffic keeps the old exit. + let egress = null + if ( + (result.domain_forward_changed || result.connection_changed) && + egressEnabled() && + process.env.KIN_CRS_MOCK !== '1' && + (result.proxy.bound_vm_ids || []).length + ) { + const restarted = await proxyEffect(handles, () => + (ctx.vmMutationOps?.ensureProxyEgress || ensureProxyEgress)( + cfg.paths.project, + proxyPool.getProxyByIdWithAuth(id), + ), + ) + egress = { ok: !!restarted.ok, error: restarted.ok ? null : restarted.error || 'egress_restart_failed' } + } + // A label is display-only; reloading every bound worker for it would + // pull live slots out of scheduling for nothing. + if (!result.connection_changed) return json(res, 200, panel.ok({ proxy: result.proxy, workers: [], egress })) + // The pool store is only one of three places the credentials live + // (pool -> vms/.json -> worker.json). Without this the edit looks + // like it worked while every bound slot keeps dialing the old proxy. + const workers = [] + for (const vmId of result.proxy.bound_vm_ids || []) { + bindVmProxy(cfg.paths.project, vmId, proxyPool.getProxyByIdWithAuth(id)) + const vm = getVm(cfg.paths.project, vmId) + if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { + setVmSchedulable(cfg.paths.project, vmId, false, 'proxy_edit_worker_reload') + const reloaded = await vmEffect(handles.get(vmId), () => + (ctx.vmMutationOps?.reloadSlotReady || reloadSlotReady)( + getVm(cfg.paths.project, vmId), + cfg.paths.project, + { + routing: ctx.routingConfig, + }, + ), + ) + if (reloaded.ok) restoreSchedulableIfReady(vmId) + // A slot that fails to reload does not fail the request: the store + // is already updated, so report per-slot and let ops retry. + workers.push({ vm_id: vmId, ok: !!reloaded.ok, error: reloaded.ok ? null : reloaded.error || null }) + } else { + workers.push({ vm_id: vmId, ok: true, error: null }) + } + } + return json(res, 200, panel.ok({ proxy: result.proxy, workers, egress })) + }) } // The one endpoint allowed to return proxy credentials. Mirrors // POST /api/panel/api-keys/:id/reveal: POST so it never lands in browser @@ -5876,185 +5957,251 @@ export function createPanelHandler(ctx) { return json(res, 200, panel.ok(result)) } if (req.method === 'POST' && /^\/api\/panel\/proxies\/[^/]+\/geo$/.test(p)) { - const id = p.split('/')[4] - const body = await readBody(req, 8192).catch(() => ({})) - const result = await proxyPool.detectGeo(id, { force: body?.force !== false }) - if (result.error === 'ipv6_disabled') { - return json(res, 409, { - ok: false, - error: { - code: result.error, - message: 'IPv6 已关闭，请在设置 → SOCKS5 开启 IPv6 代理出口', - }, - }) - } - if (!result.ok && result.error === 'proxy_not_found') { - return json(res, 404, { - ok: false, - error: { type: 'not_found_error', code: result.error, message: result.error }, - }) - } - if (!result.ok) { - return json(res, 502, { - ok: false, - error: { - type: 'upstream_error', - code: 'geo_lookup_failed', - message: String(result.error || 'geo lookup failed').slice(0, 200), - details: { proxy: result.proxy || null }, - }, - }) - } - // A slot already bound to this proxy adopts the freshly learned zone. - const timezones = [] - for (const vmId of result.proxy?.bound_vm_ids || []) { - const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, vmId, { - proxyId: id, - detect: false, - }) - timezones.push({ vm_id: vmId, applied: synced.applied, timezone: synced.timezone, reason: synced.reason }) - } - return json(res, 200, panel.ok({ ...result, timezones })) + return mutateProxyVms([], async (handles) => { + const id = p.split('/')[4] + const body = await readBody(req, 8192).catch(() => ({})) + const result = await proxyPool.detectGeo(id, { force: body?.force !== false }) + if (result.error === 'ipv6_disabled') { + return json(res, 409, { + ok: false, + error: { + code: result.error, + message: 'IPv6 已关闭，请在设置 → SOCKS5 开启 IPv6 代理出口', + }, + }) + } + if (!result.ok && result.error === 'proxy_not_found') { + return json(res, 404, { + ok: false, + error: { type: 'not_found_error', code: result.error, message: result.error }, + }) + } + if (!result.ok) { + return json(res, 502, { + ok: false, + error: { + type: 'upstream_error', + code: 'geo_lookup_failed', + message: String(result.error || 'geo lookup failed').slice(0, 200), + details: { proxy: result.proxy || null }, + }, + }) + } + // A slot already bound to this proxy adopts the freshly learned zone. + const timezones = [] + for (const vmId of result.proxy?.bound_vm_ids || []) { + const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, vmId, { + proxyId: id, + detect: false, + }) + timezones.push({ vm_id: vmId, applied: synced.applied, timezone: synced.timezone, reason: synced.reason }) + } + return json(res, 200, panel.ok({ ...result, timezones })) + }) } if (req.method === 'POST' && /^\/api\/panel\/proxies\/[^/]+\/enable$/.test(p)) { - const id = p.split('/')[4] - const result = await proxyPool.setEnabled(id, true) - if (!result.ok) - return json(res, 404, { - ok: false, - error: { type: 'not_found_error', code: result.error, message: result.error }, - }) - // re-enable bound VM scheduling if any - for (const vmId of result.proxy?.bound_vm_ids || - (result.proxy?.bound_vm_id ? [result.proxy.bound_vm_id] : [])) { - const bound = getVm(cfg.paths.project, vmId) - const reason = String(bound?.schedule_disabled_reason || '') - if (vmHasSchedulableCredential(bound, cfg.paths.project) && reason !== 'disabled' && reason !== 'stopped') { - setVmSchedulable(cfg.paths.project, vmId, true, null) + return mutateProxyVms([], async (handles) => { + const id = p.split('/')[4] + const result = await proxyPool.setEnabled(id, true) + if (!result.ok) + return json(res, 404, { + ok: false, + error: { type: 'not_found_error', code: result.error, message: result.error }, + }) + // re-enable bound VM scheduling if any + for (const vmId of result.proxy?.bound_vm_ids || + (result.proxy?.bound_vm_id ? [result.proxy.bound_vm_id] : [])) { + const bound = getVm(cfg.paths.project, vmId) + const reason = String(bound?.schedule_disabled_reason || '') + if (vmHasSchedulableCredential(bound, cfg.paths.project) && reason !== 'disabled' && reason !== 'stopped') { + setVmSchedulable(cfg.paths.project, vmId, true, null) + } } - } - return json(res, 200, panel.ok(result.proxy)) + return json(res, 200, panel.ok(result.proxy)) + }) } if (req.method === 'POST' && /^\/api\/panel\/proxies\/[^/]+\/disable$/.test(p)) { - const id = p.split('/')[4] - const result = await proxyPool.setEnabled(id, false) - if (!result.ok) - return json(res, 404, { - ok: false, - error: { type: 'not_found_error', code: result.error, message: result.error }, - }) - return json(res, 200, panel.ok(result.proxy)) + return mutateProxyVms([], async (handles) => { + const id = p.split('/')[4] + const result = await proxyPool.setEnabled(id, false) + if (!result.ok) + return json(res, 404, { + ok: false, + error: { type: 'not_found_error', code: result.error, message: result.error }, + }) + return json(res, 200, panel.ok(result.proxy)) + }) } if (req.method === 'POST' && /^\/api\/panel\/proxies\/[^/]+\/bind$/.test(p)) { const id = p.split('/')[4] const body = await readBody(req, 64 * 1024) - const vmId = body.vm_id - if (!vmId) - return json(res, 400, { - ok: false, - error: { type: 'invalid_request_error', code: 'missing_field', message: 'vm_id required', param: 'vm_id' }, - }) - const result = await proxyPool.bind(id, vmId) - if (!result.ok) { - const message = - result.error === 'proxy_bind_limit' - ? `SOCKS5 最多绑定 ${result.max || 5} 台虚拟机` - : result.error === 'local_proxy_node_mismatch' - ? LOCAL_PROXY_MISMATCH_MESSAGE - : result.error - return json(res, 400, { - ok: false, - error: { type: 'invalid_request_error', code: result.error, message, details: result }, - }) - } - bindVmProxy(cfg.paths.project, vmId, proxyPool.getProxyForVm(vmId)) - if (egressEnabled()) { - await ensureProxyEgress( - cfg.paths.project, - proxyPool.getProxyForVm(vmId) || proxyPool.getProxyByIdWithAuth(id), - ) - } - let worker = null - // Runs before the reload so the worker starts with the exit node's zone. - const timezone = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, vmId, { proxyId: id }) - const vm = getVm(cfg.paths.project, vmId) - if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { - setVmSchedulable(cfg.paths.project, vmId, false, 'proxy_rebind_worker_reload') - worker = await reloadSlotReady(getVm(cfg.paths.project, vmId), cfg.paths.project, { - routing: ctx.routingConfig, - }) - if (worker.ok) restoreSchedulableIfReady(vmId) - } - return json(res, 200, panel.ok({ proxy: result.proxy, worker, timezone })) + return mutateProxyVms([String(body.vm_id || '').trim()].filter(Boolean), async (handles) => { + const vmId = body.vm_id + if (!vmId) + return json(res, 400, { + ok: false, + error: { + type: 'invalid_request_error', + code: 'missing_field', + message: 'vm_id required', + param: 'vm_id', + }, + }) + const result = await proxyPool.bind(id, vmId) + if (!result.ok) { + const message = + result.error === 'proxy_bind_limit' + ? `SOCKS5 最多绑定 ${result.max || 5} 台虚拟机` + : result.error === 'local_proxy_node_mismatch' + ? LOCAL_PROXY_MISMATCH_MESSAGE + : result.error + return json(res, 400, { + ok: false, + error: { type: 'invalid_request_error', code: result.error, message, details: result }, + }) + } + bindVmProxy(cfg.paths.project, vmId, proxyPool.getProxyForVm(vmId)) + if (egressEnabled()) { + const egress = await proxyEffect(handles, () => + (ctx.vmMutationOps?.ensureProxyEgress || ensureProxyEgress)( + cfg.paths.project, + proxyPool.getProxyForVm(vmId) || proxyPool.getProxyByIdWithAuth(id), + ), + ) + if (!egress.ok) + return json(res, 502, { + ok: false, + error: { code: egress.code || 'egress_start_unknown', message: egress.error }, + }) + } + let worker = null + // Runs before the reload so the worker starts with the exit node's zone. + const timezone = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, vmId, { proxyId: id }) + const vm = getVm(cfg.paths.project, vmId) + if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { + setVmSchedulable(cfg.paths.project, vmId, false, 'proxy_rebind_worker_reload') + worker = await vmEffect(handles.get(vmId), () => + (ctx.vmMutationOps?.reloadSlotReady || reloadSlotReady)( + getVm(cfg.paths.project, vmId), + cfg.paths.project, + { + routing: ctx.routingConfig, + }, + ), + ) + if (worker.ok) restoreSchedulableIfReady(vmId) + } + return json(res, 200, panel.ok({ proxy: result.proxy, worker, timezone })) + }) } if (req.method === 'POST' && /^\/api\/panel\/proxies\/[^/]+\/unbind$/.test(p)) { const id = p.split('/')[4] const body = await readBody(req, 64 * 1024) - const vmId = String(body.vm_id || '').trim() - const snap = proxyPool.snapshot().proxies.find((proxy) => proxy.id === id) - const targets = vmId - ? [vmId] - : snap?.bound_vm_ids?.length - ? snap.bound_vm_ids - : snap?.bound_vm_id - ? [snap.bound_vm_id] - : [] - const result = await proxyPool.unbind(id, vmId || null) - if (!result.ok) - return json(res, 404, { - ok: false, - error: { type: 'not_found_error', code: result.error, message: result.error }, - }) - for (const unboundId of result.unbound_vm_ids || targets) { - bindVmProxy(cfg.paths.project, unboundId, null) - setVmSchedulable(cfg.paths.project, unboundId, false, 'proxy_required') - } - if (egressEnabled() && !(result.proxy?.bound_vm_ids || []).length) { - stopProxyEgress(cfg.paths.project, id) - } - return json(res, 200, panel.ok(result.proxy)) + return mutateProxyVms([String(body.vm_id || '').trim()].filter(Boolean), async (handles) => { + const vmId = String(body.vm_id || '').trim() + const snap = proxyPool.snapshot().proxies.find((proxy) => proxy.id === id) + const targets = vmId + ? [vmId] + : snap?.bound_vm_ids?.length + ? snap.bound_vm_ids + : snap?.bound_vm_id + ? [snap.bound_vm_id] + : [] + const result = await proxyPool.unbind(id, vmId || null) + if (!result.ok) + return json(res, 404, { + ok: false, + error: { type: 'not_found_error', code: result.error, message: result.error }, + }) + for (const unboundId of result.unbound_vm_ids || targets) { + bindVmProxy(cfg.paths.project, unboundId, null) + setVmSchedulable(cfg.paths.project, unboundId, false, 'proxy_required') + } + if (egressEnabled() && !(result.proxy?.bound_vm_ids || []).length) { + const stopped = await proxyEffect(handles, () => + (ctx.vmMutationOps?.stopProxyEgress || stopProxyEgress)(cfg.paths.project, id), + ) + if (!stopped.ok) + return json(res, 502, { + ok: false, + error: { code: stopped.code || 'egress_stop_unknown', message: stopped.error }, + }) + } + return json(res, 200, panel.ok(result.proxy)) + }) } if (req.method === 'DELETE' && /^\/api\/panel\/proxies\/[^/]+$/.test(p)) { - const id = p.split('/').pop() - const snap = proxyPool.snapshot().proxies.find((proxy) => proxy.id === id) - const targets = snap?.bound_vm_ids?.length ? snap.bound_vm_ids : snap?.bound_vm_id ? [snap.bound_vm_id] : [] - const result = await proxyPool.remove(id) - if (!result.ok) - return json(res, 404, { - ok: false, - error: { type: 'not_found_error', code: result.error, message: result.error }, - }) - for (const unboundId of targets) { - bindVmProxy(cfg.paths.project, unboundId, null) - setVmSchedulable(cfg.paths.project, unboundId, false, 'proxy_required') - } - if (egressEnabled()) stopProxyEgress(cfg.paths.project, id) - return json(res, 200, panel.ok(result.removed)) + return mutateProxyVms([], async (handles) => { + const id = p.split('/').pop() + const snap = proxyPool.snapshot().proxies.find((proxy) => proxy.id === id) + const targets = snap?.bound_vm_ids?.length ? snap.bound_vm_ids : snap?.bound_vm_id ? [snap.bound_vm_id] : [] + const result = await proxyPool.remove(id) + if (!result.ok) + return json(res, 404, { + ok: false, + error: { type: 'not_found_error', code: result.error, message: result.error }, + }) + for (const unboundId of targets) { + bindVmProxy(cfg.paths.project, unboundId, null) + setVmSchedulable(cfg.paths.project, unboundId, false, 'proxy_required') + } + if (egressEnabled()) { + const stopped = await proxyEffect(handles, () => + (ctx.vmMutationOps?.stopProxyEgress || stopProxyEgress)(cfg.paths.project, id), + ) + if (!stopped.ok) + return json(res, 502, { + ok: false, + error: { code: stopped.code || 'egress_stop_unknown', message: stopped.error }, + }) + } + return json(res, 200, panel.ok(result.removed)) + }) } // Auto-allocate proxy for a VM if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/allocate-proxy$/.test(p)) { - const vmId = p.split('/')[4] - const allocated = await proxyPool.allocateForVm(vmId) - if (!allocated) - return json(res, 409, { - ok: false, - error: { type: 'api_error', code: 'no_free_proxy', message: 'No free SOCKS5 in pool' }, + return mutateProxyVms([p.split('/')[4]], async (handles) => { + const vmId = p.split('/')[4] + const allocated = await proxyPool.allocateForVm(vmId) + if (!allocated) + return json(res, 409, { + ok: false, + error: { type: 'api_error', code: 'no_free_proxy', message: 'No free SOCKS5 in pool' }, + }) + bindVmProxy(cfg.paths.project, vmId, proxyPool.getProxyForVm(vmId)) + if (egressEnabled()) { + const egress = await proxyEffect(handles, () => + (ctx.vmMutationOps?.ensureProxyEgress || ensureProxyEgress)( + cfg.paths.project, + proxyPool.getProxyForVm(vmId), + ), + ) + if (!egress.ok) + return json(res, 502, { + ok: false, + error: { code: egress.code || 'egress_start_unknown', message: egress.error }, + }) + } + let worker = null + const timezone = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, vmId, { + proxyId: allocated?.id || null, }) - bindVmProxy(cfg.paths.project, vmId, proxyPool.getProxyForVm(vmId)) - if (egressEnabled()) await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyForVm(vmId)) - let worker = null - const timezone = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, vmId, { - proxyId: allocated?.id || null, + const vm = getVm(cfg.paths.project, vmId) + if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { + setVmSchedulable(cfg.paths.project, vmId, false, 'proxy_rebind_worker_reload') + worker = await vmEffect(handles.get(vmId), () => + (ctx.vmMutationOps?.reloadSlotReady || reloadSlotReady)( + getVm(cfg.paths.project, vmId), + cfg.paths.project, + { + routing: ctx.routingConfig, + }, + ), + ) + if (worker.ok) restoreSchedulableIfReady(vmId) + } + return json(res, 200, panel.ok({ proxy: allocated, worker, timezone })) }) - const vm = getVm(cfg.paths.project, vmId) - if (vm?.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { - setVmSchedulable(cfg.paths.project, vmId, false, 'proxy_rebind_worker_reload') - worker = await reloadSlotReady(getVm(cfg.paths.project, vmId), cfg.paths.project, { - routing: ctx.routingConfig, - }) - if (worker.ok) restoreSchedulableIfReady(vmId) - } - return json(res, 200, panel.ok({ proxy: allocated, worker, timezone })) } return json(res, 404, { @@ -6139,9 +6286,15 @@ export function createPanelHandler(ctx) { result = await handlePanel(req, res, url) } catch (error) { if ( - ['vm_operation_active', 'vm_execution_active', 'vm_physical_target_conflict', 'invalid_vm_operation'].includes( - error.code, - ) + [ + 'vm_operation_active', + 'vm_execution_active', + 'vm_physical_target_conflict', + 'invalid_vm_operation', + 'proxy_operation_active', + 'proxy_operation_inactive', + 'vm_mutation_claim_required', + ].includes(error.code) ) { state.response = [res, error.status || 409, { ok: false, error: { code: error.code, message: error.message } }] result = true diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 6d8c13191..98bb572e3 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -163,7 +163,7 @@ test('an idle proxy binding owns the new target and releases both persistent cla const h = f.handler({ vm_id: 'vm-b' }, { vmMutationOps: { ensureProxyEgress: async () => ({ ok: true }) } }) await h.request('POST', '/api/panel/proxies/' + f.proxyId + '/bind') assert.equal(h.response.status, 200, JSON.stringify(h.response.body)) - assert.deepEqual(f.pool.getProxyByIdWithAuth(f.proxyId).bound_vm_ids, ['vm-a', 'vm-b']) + assert.deepEqual(f.pool.snapshot().proxies.find((proxy) => proxy.id === f.proxyId).bound_vm_ids, ['vm-a', 'vm-b']) const vmClaim = await f.otherOperations.tryAcquire({ vmId: 'vm-b' }) assert.equal(vmClaim.ok, true) await vmClaim.release() From 053e77db401af2bb079f718d4f771ff69e24e791 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:45:22 +0800 Subject: [PATCH 252/473] feat(slotgate): launch and bind restricted node-owned Docker executors --- .../2026-10-21-node-fencing-launch-adapter.md | 61 ++++ worker/internal/slotgate/authority.go | 104 ++++++- worker/internal/slotgate/docker_engine.go | 70 +++++ worker/internal/slotgate/docker_launch.go | 272 ++++++++++++++++++ .../internal/slotgate/docker_launch_test.go | 154 +++++++++- worker/internal/slotgate/docker_verifier.go | 48 +--- worker/internal/slotgate/ksg2.go | 6 + worker/internal/slotgate/slotgate.go | 15 +- 8 files changed, 674 insertions(+), 56 deletions(-) create mode 100644 docs/plans/2026-10-21-node-fencing-launch-adapter.md create mode 100644 worker/internal/slotgate/docker_engine.go create mode 100644 worker/internal/slotgate/docker_launch.go diff --git a/docs/plans/2026-10-21-node-fencing-launch-adapter.md b/docs/plans/2026-10-21-node-fencing-launch-adapter.md new file mode 100644 index 000000000..58552faba --- /dev/null +++ b/docs/plans/2026-10-21-node-fencing-launch-adapter.md @@ -0,0 +1,61 @@ +# 节点侧 Docker 启动接口的范围与启用条件 + +本批只实现普通 Claude Docker executor 的可信启动与绑定。它使用现有 slot image 中的 `/opt/kin/kin-kernel`、`cli-node` 和 glibc shim，不修改预编译 payload。`remote-slot.mjs` 的普通 Docker spec 和 `slot-image.mjs` 的 payload 布局是路径依据。 + +这不是生产 HA 的启用条件。fenced 模式默认关闭，现有部署仍保持单活动控制面。验证仅使用临时 Unix socket 上的合成 Engine API，没有连接真实 Docker、节点或 provider。 + +## 接口与持久状态 + +KSG2 增加 `launch` 操作。客户端只提供 `vm_id`、创建操作时捕获的 `fence` 和唯一 `command_id`。镜像、挂载、用户、网络、环境变量、命令和 HostConfig 均由节点代码生成。客户端不能提交 `bind`、`complete`、`ended` 或原始 Docker spec。 + +节点配置中的 `launch` policy 只有四项： + +| 字段 | 允许值 | +| --- | --- | +| `image_id` | 经过节点管理员批准、包含现有 payload 的完整 `sha256:` image ID；不接受可变 tag，也不自动拉取镜像 | +| `network_id` | `none`，或节点预先批准的完整 bridge network ID；不接受 host、container 或 service 网络命名空间 | +| `uid`、`gid` | 非 root 的正整数；不能由 RPC 调用者覆盖 | + +`launch` policy 缺失时拒绝启动。当前接口要求目标 VM 已有 `VACANT` 状态，且可信节点组件已准备好该 captured incarnation 的私有文件树。本批没有提供初始 allocate、文件上传或文件发布 RPC，也不会自动接管 legacy executor。 + +启动顺序为： + +1. 校验 captured fence、`VACANT` 状态、私有文件和固定镜像、网络。 +2. 持久 `BeginLaunch`，再用 `StartLaunch` 持久记录 dispatched。 +3. 在 Authority 全局锁外向 Engine create；名称和 labels 绑定 VM、generation、incarnation 和 command ID。 +4. create 返回完整 immutable ID 后，先将 ID、daemon ID、boot ID、镜像 ID 和名称保存到 pending launch，再按该完整 ID start。 +5. 节点 inspect 同一 ID 的运行状态、StartedAt、镜像和 labels，并再次核对 daemon ID。 +6. `BindExecutor` 对原 captured fence 做条件更新；成功后才将状态置为 `ACTIVE`。 + +只读预检失败不会创建 launch claim，也不会发送 create。进入 dispatched 后的 create/start/inspect 或持久化错误保留 pending，不自动重试。节点重新打开遇到 pending launch 会保留为 unknown，进入 `REVOKING`。pending launch 阻止 generation 推进。create 返回晚于 revoke 时仍保存目标 ID，但不继续 start 或 bind。 + +客户端连接关闭、超时或控制面 owner 消失均不是终止证明。正常转发目前也继续保留 pending；普通 HTTP/SSE 终态分类仍未接入。当前状态机没有自动处理未知 launch 的恢复操作。 + +## 容器与文件权限 + +启动命令固定为 `/opt/kin/kin-kernel --gateway-worker --config /run/kin/kernel.json`。容器使用非 root UID/GID、只读 rootfs、`CapDrop=ALL`、`no-new-privileges`、固定资源上限和 `RestartPolicy=no`。不开放任意 exec、设备、host PID/IPC/network 或 privileged 能力。 + +容器只能挂载以下 captured incarnation 内的路径： + +| 节点路径 | 容器路径 | 权限 | +| --- | --- | --- | +| `vms//incarnations//home` | `/home/kincli` | guest 读写 | +| `vms//incarnations//claude` | `/home/kincli/.claude` | guest 读写 | +| `vms//incarnations//run` | `/run/kin` | guest 创建 socket | +| `vms//incarnations//config/kernel.json` | `/run/kin/kernel.json` | 只读单文件挂载 | +| `vms//incarnations//config/internal.token` | `/run/kin/internal.token` | 只读单文件挂载 | + +父目录和 config 目录必须归节点服务用户所有，目录权限为私有目录。guest 可写的三个叶目录必须归批准的 guest UID 所有。config 文件必须是节点服务用户所有的普通文件，不能是 symlink 或 hardlink，不能允许 group/other 写入，并须允许批准的 guest 用户读取。启动预检只检查元数据，不读取凭据内容，也不创建缺失的 bind source。 + +Authority journal、其父目录、节点 agent 的可执行文件与配置，以及 Docker socket 不得挂载给 guest。节点文件发布接口尚未实现；后续 stage/publish 必须限制目标文件、拒绝 symlink/hardlink，并在最终 rename 前校验原 captured fence。当前 metadata 检查依赖节点权限已隔离；它不能代替权限迁移，也不能对抗仍持有原始 host 写权限的旧 owner。 + +## 实际节点迁移前仍需完成 + +- 把容器生命周期和文件写权限迁移到节点服务。runtime Pod 不得继续持有 root/docker-group SSH、raw Docker socket、可写 SFTP 或替换 gate 的权限。bootstrap 凭据须与 runtime 分离。 +- 迁移时撤销旧授权，并关闭已经建立的 SSH、SFTP、终端和 forwarding 会话。只修改新连接的认证方式不能阻止旧 owner 的延迟命令。 +- 将 KSG2、captured generation、PG admission/job/VM operation 和文件发布接入生产调用链。fenced 节点不得回退到 KSG1 或原始 SSH 路径。 +- 提供节点可信 allocate、文件发布、未知 launch 的有证据恢复，以及正常响应终态处理。恢复只能关闭同一 immutable executor/incarnation 映射的作业；不能靠 TTL、ready_slots 或 cancel ACK 释放。 +- 单独处理 KVM、原生 macOS 和 Codex 进程归属。现有 Codex Docker 容器仅持有身份和终端，停止它不能证明控制面 detached Codex 子进程已经终止。 +- 在两个不同节点上的控制面验证同时执行、旧命令迟到、节点服务崩溃、Engine 故障及安全接管。本批合成测试不能代替该验收。 + +本任务不在用户节点执行以上权限迁移、升级或资源操作。实际迁移须另行获得对应节点与操作范围的授权。 diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index c49cda587..a271e495f 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -78,14 +78,25 @@ type Fence struct { } type CommandRecord struct { - ID string `json:"id"` - Kind CommandKind `json:"kind"` - Status CommandStatus `json:"status"` - Fence Fence `json:"fence"` - Runtime string `json:"runtime"` - Token string `json:"token"` - Dispatched bool `json:"dispatched"` - TerminationID string `json:"termination_id,omitempty"` + ID string `json:"id"` + Kind CommandKind `json:"kind"` + Status CommandStatus `json:"status"` + Fence Fence `json:"fence"` + Runtime string `json:"runtime"` + Token string `json:"token"` + Dispatched bool `json:"dispatched"` + TerminationID string `json:"termination_id,omitempty"` + LaunchTarget *DockerLaunchTarget `json:"launch_target,omitempty"` +} + +// DockerLaunchTarget is recorded after create returns and before start. A +// missing StartedAt here is intentional: the container has not started yet. +type DockerLaunchTarget struct { + ContainerID string `json:"container_id"` + DockerDaemonID string `json:"docker_daemon_id"` + NodeBootID string `json:"node_boot_id"` + ImageID string `json:"image_id"` + Name string `json:"name"` } type AuthorityState struct { @@ -295,6 +306,65 @@ func (a *Authority) BeginLaunch(vm string, expected Fence, commandID string) (Co return a.begin(vm, expected, commandID, commandLaunch, PhaseVacant) } +// StartLaunch records dispatch without holding the Authority lock over Docker +// I/O. The pending launch blocks all generation advancement until a trusted +// binding completes; a crash or lost create response is never retried here. +func (a *Authority) StartLaunch(handle CommandHandle) error { + a.mu.Lock() + defer a.mu.Unlock() + state, command, err := a.command(handle) + if err != nil { + return err + } + if state.Phase != PhaseVacant || command.Kind != commandLaunch || command.Status != CommandPending { + return ErrAuthorityPhase + } + if command.Dispatched { + return ErrCommandDispatched + } + command.Dispatched = true + state.Commands[handle.ID] = command + _, err = a.commit(handle.VMID, state) + return err +} + +func (a *Authority) RecordLaunchTarget(handle CommandHandle, target DockerLaunchTarget) error { + a.mu.Lock() + defer a.mu.Unlock() + if err := validateLaunchTarget(target); err != nil { + return err + } + state, command, err := a.command(handle) + if err != nil { + return err + } + // Revocation can win while create is in progress. Still retain the exact + // returned target, but the launcher must not start or bind it afterward. + if command.Kind != commandLaunch || command.Status != CommandPending || !command.Dispatched { + return ErrAuthorityPhase + } + if command.LaunchTarget != nil && *command.LaunchTarget != target { + return ErrFenceMismatch + } + command.LaunchTarget = &target + state.Commands[handle.ID] = command + _, err = a.commit(handle.VMID, state) + return err +} + +func (a *Authority) checkLaunch(handle CommandHandle) error { + a.mu.Lock() + defer a.mu.Unlock() + state, command, err := a.command(handle) + if err != nil { + return err + } + if state.Phase != PhaseVacant || command.Kind != commandLaunch || command.Status != CommandPending || !command.Dispatched { + return ErrAuthorityPhase + } + return nil +} + // BeginCommand must commit before any bytes or mutation are sent. Command IDs // remain tombstoned across generations, preventing retry by relabeling an ID. func (a *Authority) BeginCommand(vm string, expected Fence, commandID string, kind CommandKind) (CommandHandle, error) { @@ -383,6 +453,9 @@ func (a *Authority) BindExecutor(handle CommandHandle, executor ExecutorIdentity if state.Phase != PhaseVacant || command.Kind != commandLaunch || command.Status != CommandPending || !command.Dispatched { return AuthorityState{}, ErrAuthorityPhase } + if target := command.LaunchTarget; target != nil && (target.ContainerID != executor.ContainerID || target.DockerDaemonID != executor.DockerDaemonID || target.NodeBootID != executor.NodeBootID) { + return AuthorityState{}, ErrFenceMismatch + } for _, retired := range state.Retired { if executor == retired { return AuthorityState{}, ErrFenceMismatch @@ -410,6 +483,9 @@ func (a *Authority) CompleteCommand(handle CommandHandle) error { if command.Status != CommandPending { return ErrCommandOwner } + if command.Kind == commandLaunch && command.Dispatched { + return ErrAuthorityPhase + } command.Status = CommandCompleted state.Commands[handle.ID] = command _, err = a.commit(handle.VMID, state) @@ -590,6 +666,10 @@ func (a *Authority) persist(next authorityJournal) error { func cloneState(state AuthorityState) AuthorityState { commands := make(map[string]CommandRecord, len(state.Commands)) for id, command := range state.Commands { + if command.LaunchTarget != nil { + target := *command.LaunchTarget + command.LaunchTarget = &target + } commands[id] = command } state.Commands = commands @@ -699,6 +779,14 @@ func validateJournal(journal authorityJournal) error { if pendingCommand(command) && command.Fence != state.Fence { return ErrInvalidAuthority } + if command.LaunchTarget != nil { + if command.Kind != commandLaunch || !command.Dispatched { + return ErrInvalidAuthority + } + if err := validateLaunchTarget(*command.LaunchTarget); err != nil { + return err + } + } } if state.LastTermination != nil { if err := validateExecutor(state.LastTermination.Executor); err != nil { diff --git a/worker/internal/slotgate/docker_engine.go b/worker/internal/slotgate/docker_engine.go new file mode 100644 index 000000000..207e249bc --- /dev/null +++ b/worker/internal/slotgate/docker_engine.go @@ -0,0 +1,70 @@ +package slotgate + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "path/filepath" + "time" +) + +type dockerEngine struct{ client *http.Client } + +func newDockerEngine(socket string) (*dockerEngine, error) { + if !filepath.IsAbs(socket) { + return nil, ErrInvalidAuthority + } + transport := &http.Transport{DisableKeepAlives: true, DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socket) + }} + return &dockerEngine{client: &http.Client{Transport: transport, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil +} + +func (e *dockerEngine) call(ctx context.Context, method, path string, body, out any, allowed ...int) (int, error) { + var payload io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return 0, err + } + payload = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, payload) + if err != nil { + return 0, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + res, err := e.client.Do(req) + if err != nil { + return 0, err + } + defer res.Body.Close() + ok := false + for _, status := range allowed { + if res.StatusCode == status { + ok = true + } + } + if !ok { + return res.StatusCode, fmt.Errorf("docker_status_%d", res.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(res.Body, (1<<20)+1)) + if err != nil { + return res.StatusCode, err + } + if len(data) > 1<<20 { + return res.StatusCode, ErrInvalidAuthority + } + if out != nil && len(data) != 0 { + if err = json.Unmarshal(data, out); err != nil { + return res.StatusCode, err + } + } + return res.StatusCode, nil +} diff --git a/worker/internal/slotgate/docker_launch.go b/worker/internal/slotgate/docker_launch.go new file mode 100644 index 000000000..a9c24edf4 --- /dev/null +++ b/worker/internal/slotgate/docker_launch.go @@ -0,0 +1,272 @@ +package slotgate + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" +) + +// DockerLaunchPolicy is node-owned configuration, never a client container +// spec. This first adapter only supports the existing non-KVM Claude payload. +type DockerLaunchPolicy struct { + ImageID string `json:"image_id"` + NetworkID string `json:"network_id"` + UID int `json:"uid"` + GID int `json:"gid"` +} + +type dockerLauncher struct { + engine *dockerEngine + authority *Authority + root string + boot string + policy DockerLaunchPolicy +} + +func fullDockerID(id string) bool { + decoded, err := hex.DecodeString(id) + return err == nil && len(decoded) == 32 +} +func pinnedImage(id string) bool { + return strings.HasPrefix(id, "sha256:") && fullDockerID(strings.TrimPrefix(id, "sha256:")) +} + +func validateLaunchTarget(target DockerLaunchTarget) error { + if !fullDockerID(target.ContainerID) || !pinnedImage(target.ImageID) || !validAuthorityID(target.DockerDaemonID) || !validAuthorityID(target.NodeBootID) || !validAuthorityID(target.Name) { + return ErrInvalidAuthority + } + return nil +} + +func newDockerLauncher(socket, root, boot string, a *Authority, policy DockerLaunchPolicy) (*dockerLauncher, error) { + if !filepath.IsAbs(root) || !validAuthorityID(boot) || !pinnedImage(policy.ImageID) || (policy.NetworkID != "none" && !fullDockerID(policy.NetworkID)) || policy.UID <= 0 || policy.GID <= 0 || uint64(policy.UID) > 1<<31-1 || uint64(policy.GID) > 1<<31-1 { + return nil, ErrInvalidAuthority + } + engine, err := newDockerEngine(socket) + if err != nil { + return nil, err + } + if a == nil { + return nil, ErrInvalidAuthority + } + return &dockerLauncher{engine: engine, authority: a, root: filepath.Clean(root), boot: boot, policy: policy}, nil +} + +func (l *dockerLauncher) Launch(ctx context.Context, vm string, expected Fence, id string) (AuthorityState, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + state, err := l.authority.State(vm) + if err != nil { + return AuthorityState{}, err + } + if state.Fence != expected { + return AuthorityState{}, ErrFenceMismatch + } + if state.Phase != PhaseVacant { + return AuthorityState{}, ErrAuthorityPhase + } + if hasPendingMutation(state) { + return AuthorityState{}, ErrPendingMutation + } + if !validAuthorityID(id) { + return AuthorityState{}, ErrInvalidAuthority + } + if err = l.privateFiles(vm, expected); err != nil { + return AuthorityState{}, err + } + var daemon struct { + ID string `json:"ID"` + } + if _, err = l.engine.call(ctx, "GET", "/info", nil, &daemon, 200); err != nil { + return AuthorityState{}, err + } + if !validAuthorityID(daemon.ID) { + return AuthorityState{}, ErrInvalidAuthority + } + var image struct { + ID string `json:"Id"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"Config"` + } + if _, err = l.engine.call(ctx, "GET", "/images/"+l.policy.ImageID+"/json", nil, &image, 200); err != nil { + return AuthorityState{}, err + } + if image.ID != l.policy.ImageID || image.Config.Labels["kin.slot.full"] != "1" { + return AuthorityState{}, ErrInvalidAuthority + } + if l.policy.NetworkID != "none" { + var network struct { + ID string `json:"Id"` + Driver string `json:"Driver"` + Scope string `json:"Scope"` + } + if _, err = l.engine.call(ctx, "GET", "/networks/"+l.policy.NetworkID, nil, &network, 200); err != nil { + return AuthorityState{}, err + } + if network.ID != l.policy.NetworkID || network.Driver != "bridge" || network.Scope != "local" { + return AuthorityState{}, ErrInvalidAuthority + } + } + handle, err := l.authority.BeginLaunch(vm, expected, id) + if err != nil { + return AuthorityState{}, err + } + if err = l.authority.StartLaunch(handle); err != nil { + return AuthorityState{}, err + } + if err = l.authority.checkLaunch(handle); err != nil { + return AuthorityState{}, err + } + spec, name := l.spec(vm, expected, id) + var created struct { + ID string `json:"Id"` + } + if _, err = l.engine.call(ctx, "POST", "/containers/create?name="+url.QueryEscape(name), spec, &created, 201); err != nil { + return AuthorityState{}, err + } + if !fullDockerID(created.ID) { + return AuthorityState{}, ErrInvalidAuthority + } + target := DockerLaunchTarget{ContainerID: created.ID, DockerDaemonID: daemon.ID, NodeBootID: l.boot, ImageID: l.policy.ImageID, Name: name} + if err = l.authority.RecordLaunchTarget(handle, target); err != nil { + return AuthorityState{}, err + } + var currentDaemon struct { + ID string `json:"ID"` + } + if _, err = l.engine.call(ctx, "GET", "/info", nil, ¤tDaemon, 200); err != nil { + return AuthorityState{}, err + } + if currentDaemon.ID != daemon.ID { + return AuthorityState{}, ErrTerminationProof + } + if err = l.authority.checkLaunch(handle); err != nil { + return AuthorityState{}, err + } + path := "/containers/" + created.ID + if _, err = l.engine.call(ctx, "POST", path+"/start", nil, nil, 204, 304); err != nil { + return AuthorityState{}, err + } + var inspected struct { + ID string `json:"Id"` + Image string `json:"Image"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"Config"` + State *struct { + Running *bool `json:"Running"` + Restarting *bool `json:"Restarting"` + StartedAt string `json:"StartedAt"` + } `json:"State"` + } + if _, err = l.engine.call(ctx, "GET", path+"/json", nil, &inspected, 200); err != nil { + return AuthorityState{}, err + } + if inspected.ID != created.ID || inspected.Image != l.policy.ImageID || inspected.State == nil || inspected.State.Running == nil || !*inspected.State.Running || inspected.State.Restarting == nil || *inspected.State.Restarting { + return AuthorityState{}, ErrTerminationProof + } + for key, value := range spec["Labels"].(map[string]string) { + if inspected.Config.Labels[key] != value { + return AuthorityState{}, ErrTerminationProof + } + } + started, err := time.Parse(time.RFC3339Nano, inspected.State.StartedAt) + if err != nil || started.IsZero() { + return AuthorityState{}, ErrInvalidAuthority + } + currentDaemon.ID = "" + if _, err = l.engine.call(ctx, "GET", "/info", nil, ¤tDaemon, 200); err != nil { + return AuthorityState{}, err + } + if currentDaemon.ID != daemon.ID { + return AuthorityState{}, ErrTerminationProof + } + identity := ExecutorIdentity{Kind: "container", ContainerID: created.ID, DockerDaemonID: daemon.ID, NodeBootID: l.boot, StartedAt: inspected.State.StartedAt} + return l.authority.BindExecutor(handle, identity) +} + +func (l *dockerLauncher) spec(vm string, fence Fence, id string) (map[string]any, string) { + digest := sha256.Sum256([]byte(vm + "\x00" + fence.Incarnation + "\x00" + id)) + name := "kin-fenced-" + hex.EncodeToString(digest[:16]) + private := filepath.Join(l.root, "vms", vm, "incarnations", fence.Incarnation) + return map[string]any{ + "Image": l.policy.ImageID, "Hostname": name, "User": fmt.Sprintf("%d:%d", l.policy.UID, l.policy.GID), "WorkingDir": "/home/kincli", + "Entrypoint": []string{"/opt/kin/kin-kernel"}, "Cmd": []string{"--gateway-worker", "--config", "/run/kin/kernel.json"}, + "Env": []string{"HOME=/home/kincli", "CLAUDE_CONFIG_DIR=/home/kincli/.claude", "TZ=UTC", "LANG=en_US.UTF-8", "KIN_VM_ID=" + vm}, + "Labels": map[string]string{"kin.slot.fenced": "1", "kin.vm.id": vm, "kin.fence.generation": strconv.FormatUint(fence.Generation, 10), "kin.fence.incarnation": fence.Incarnation, "kin.fence.command": id}, + "HostConfig": map[string]any{ + "Binds": []string{private + "/home:/home/kincli", private + "/claude:/home/kincli/.claude", private + "/run:/run/kin", private + "/config/kernel.json:/run/kin/kernel.json:ro", private + "/config/internal.token:/run/kin/internal.token:ro"}, + "Privileged": false, "NetworkMode": l.policy.NetworkID, "IpcMode": "private", "RestartPolicy": map[string]string{"Name": "no"}, "ReadonlyRootfs": true, + "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges"}, "Memory": int64(1 << 30), "MemorySwap": int64(1 << 30), "NanoCpus": int64(1e9), "PidsLimit": 256, + "Tmpfs": map[string]string{"/tmp": "rw,noexec,nosuid,size=32m"}, + }, + }, name +} + +// File publication is separate. Launch only accepts an already prepared +// private tree; it never follows a symlink, creates a missing bind source, or +// lets a container select a host path. No credential contents are read here. +func (l *dockerLauncher) privateFiles(vm string, fence Fence) error { + inc, err := hex.DecodeString(fence.Incarnation) + if err != nil || len(inc) != 16 || !vmIDRe.MatchString(vm) { + return ErrInvalidAuthority + } + parts := []string{l.root, filepath.Join(l.root, "vms"), filepath.Join(l.root, "vms", vm), filepath.Join(l.root, "vms", vm, "incarnations"), filepath.Join(l.root, "vms", vm, "incarnations", fence.Incarnation)} + for _, dir := range parts { + if err = launchPath(dir, true, os.Geteuid()); err != nil { + return err + } + } + base := parts[len(parts)-1] + if err = launchPath(filepath.Join(base, "config"), true, os.Geteuid()); err != nil { + return err + } + for _, dir := range []string{"run", "home", "claude"} { + if err = launchPath(filepath.Join(base, dir), true, l.policy.UID); err != nil { + return err + } + } + for _, file := range []string{"kernel.json", "internal.token"} { + path := filepath.Join(base, "config", file) + if err = launchPath(path, false, os.Geteuid()); err != nil { + return err + } + info, err := os.Lstat(path) + if err != nil { + return err + } + stat := info.Sys().(*syscall.Stat_t) + readable := info.Mode().Perm()&0o004 != 0 || (int(stat.Uid) == l.policy.UID && info.Mode().Perm()&0o400 != 0) || (int(stat.Gid) == l.policy.GID && info.Mode().Perm()&0o040 != 0) + if !readable { + return ErrInvalidAuthority + } + } + return nil +} + +func launchPath(path string, directory bool, owner int) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || int(stat.Uid) != owner || info.Mode().Perm()&0o022 != 0 || info.Mode()&os.ModeSymlink != 0 || info.IsDir() != directory { + return ErrInvalidAuthority + } + if directory && info.Mode().Perm()&0o077 != 0 { + return ErrInvalidAuthority + } + if !directory && (!info.Mode().IsRegular() || stat.Nlink != 1 || info.Size() == 0 || info.Size() > gate2RequestMax) { + return ErrInvalidAuthority + } + return nil +} diff --git a/worker/internal/slotgate/docker_launch_test.go b/worker/internal/slotgate/docker_launch_test.go index 32a540346..e67c65203 100644 --- a/worker/internal/slotgate/docker_launch_test.go +++ b/worker/internal/slotgate/docker_launch_test.go @@ -1,9 +1,12 @@ package slotgate import ( + "bufio" "context" + "crypto/tls" "encoding/json" "fmt" + "net" "net/http" "os" "path/filepath" @@ -16,6 +19,7 @@ import ( type fakeLaunchEngine struct { mu sync.Mutex image string + socket string container string startedAt string spec map[string]any @@ -25,6 +29,7 @@ type fakeLaunchEngine struct { createEntered chan struct{} createProceed chan struct{} onInspect func() + networkDriver string } func (f *fakeLaunchEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -41,6 +46,12 @@ func (f *fakeLaunchEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]any{"Id": image, "Config": map[string]any{"Labels": map[string]string{"kin.slot.full": "1"}}}) return } + if strings.HasPrefix(r.URL.Path, "/networks/") { + driver := f.networkDriver + f.mu.Unlock() + json.NewEncoder(w).Encode(map[string]any{"Id": strings.TrimPrefix(r.URL.Path, "/networks/"), "Driver": driver, "Scope": "local"}) + return + } if r.URL.Path == "/containers/create" { if err := json.NewDecoder(r.Body).Decode(&f.spec); err != nil { f.mu.Unlock() @@ -105,14 +116,28 @@ func launchFixture(t *testing.T) (*Authority, *dockerLauncher, *fakeLaunchEngine if err = os.MkdirAll(filepath.Join(private, dir), 0o700); err != nil { t.Fatal(err) } + if dir != "config" && policy.UID != os.Geteuid() { + if err = os.Chown(filepath.Join(private, dir), policy.UID, policy.GID); err != nil { + t.Fatal(err) + } + } } for _, file := range []string{"kernel.json", "internal.token"} { if err = os.WriteFile(filepath.Join(private, "config", file), []byte("synthetic-only"), 0o600); err != nil { t.Fatal(err) } + if policy.UID != os.Geteuid() { + if err = os.Chown(filepath.Join(private, "config", file), os.Geteuid(), policy.GID); err != nil { + t.Fatal(err) + } + if err = os.Chmod(filepath.Join(private, "config", file), 0o640); err != nil { + t.Fatal(err) + } + } } fake := &fakeLaunchEngine{image: policy.ImageID, container: strings.Repeat("c", 64), startedAt: "2026-10-21T01:00:00Z"} socket := filepath.Join(shortRoot(t), "docker.sock") + fake.socket = socket unixHTTP(t, socket, fake) launcher, err := newDockerLauncher(socket, root, boot, a, policy) if err != nil { @@ -161,12 +186,21 @@ func TestDockerLaunchNetworkIOLeavesOtherVMAdmissionAvailableAndPendingBlocksAdv a, launcher, engine, state, _ := launchFixture(t) other := launchForTest(t, a, "other", testExecutor("d")) engine.createEntered, engine.createProceed = make(chan struct{}), make(chan struct{}) + var released sync.Once + resume := func() { released.Do(func() { close(engine.createProceed) }) } + defer resume() finished := make(chan error, 1) go func() { _, err := launcher.Launch(context.Background(), "v", state.Fence, "slow-create") finished <- err }() - <-engine.createEntered + select { + case <-engine.createEntered: + case err := <-finished: + t.Fatalf("launch ended before create: %v", err) + case <-time.After(3 * time.Second): + t.Fatal("create was not reached") + } admitted := make(chan error, 1) go func() { _, err := a.BeginCommand("other", other.Fence, "parallel", CommandForward); admitted <- err }() select { @@ -175,7 +209,7 @@ func TestDockerLaunchNetworkIOLeavesOtherVMAdmissionAvailableAndPendingBlocksAdv t.Fatal(err) } case <-time.After(time.Second): - close(engine.createProceed) + resume() t.Fatal("Engine I/O held the Authority global lock") } if _, err := a.Revoke("v", state.Fence); err != nil { @@ -183,7 +217,7 @@ func TestDockerLaunchNetworkIOLeavesOtherVMAdmissionAvailableAndPendingBlocksAdv } _, err := a.Advance(context.Background(), "v", state.Fence) requireAuthorityError(t, err, ErrPendingMutation) - close(engine.createProceed) + resume() if err = <-finished; err == nil { t.Fatal("late launch bound after revocation") } @@ -313,3 +347,117 @@ func TestDockerLaunchRejectsRootCapabilitiesAndUnsafePrivateFilesBeforeCreate(t t.Fatal("invalid preparation reserved a launch") } } + +func TestDockerLaunchJournalFailureSendsNoCreateAndDispatchedLaunchCannotBeCompleted(t *testing.T) { + a, launcher, engine, state, root := launchFixture(t) + journal := filepath.Join(root, "authority", "authority.json") + if err := os.Rename(journal, journal+".saved"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(journal, 0o700); err != nil { + t.Fatal(err) + } + if _, err := launcher.Launch(context.Background(), "v", state.Fence, "persist-before-create"); err == nil { + t.Fatal("journal failure authorized create") + } + engine.mu.Lock() + calls := append([]string(nil), engine.calls...) + engine.mu.Unlock() + for _, call := range calls { + if strings.HasPrefix(call, "POST ") { + t.Fatalf("mutation before durable launch: %v", calls) + } + } + if err := os.Remove(journal); err != nil { + t.Fatal(err) + } + if err := os.Rename(journal+".saved", journal); err != nil { + t.Fatal(err) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, filepath.Join(root, "authority"), terminatedExecutor) + handle, err := reopened.BeginLaunch("v", state.Fence, "node-launch") + if err != nil { + t.Fatal(err) + } + if err = reopened.StartLaunch(handle); err != nil { + t.Fatal(err) + } + requireAuthorityError(t, reopened.CompleteCommand(handle), ErrAuthorityPhase) +} + +func TestKSG2LaunchCreatesAndBindsWithNodePolicyAndRejectsRawSpec(t *testing.T) { + a, launcher, engine, state, root := launchFixture(t) + if err := a.Close(); err != nil { + t.Fatal(err) + } + s, err := New(Config{Listen: "127.0.0.1:0", Root: root, Token: testToken, TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: filepath.Join(root, "authority"), DockerSocket: engine.socket, Launch: &launcher.policy}) + if err != nil { + t.Fatal(err) + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.ServeListener(ctx, tls.NewListener(ln, s.TLSConfig())) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + c, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS13}) + if err != nil { + t.Fatal(err) + } + c.SetDeadline(time.Now().Add(3 * time.Second)) + raw, _ := json.Marshal(map[string]any{"op": "launch", "vm_id": "v", "fence": state.Fence, "command_id": "raw-spec-forbidden", "HostConfig": map[string]any{"Privileged": true, "Binds": []string{"/:/host"}}}) + fmt.Fprintf(c, "KSG2 %s\n%s\n", testToken, raw) + line, _ := bufio.NewReader(c).ReadString('\n') + c.Close() + if line != "ERR bad_request\n" { + t.Fatalf("raw Docker spec accepted: %q", line) + } + engine.mu.Lock() + count := len(engine.calls) + engine.mu.Unlock() + if count != 0 { + t.Fatal("raw spec reached Engine") + } + reply := gate2Control(t, ln.Addr().String(), gate2Request{Op: "launch", VMID: "v", Fence: state.Fence, CommandID: "trusted-node-launch"}) + if !reply.OK || reply.State == nil || reply.State.Phase != PhaseActive || reply.State.Fence.Executor.ContainerID != engine.container { + t.Fatalf("KSG2 launch: %+v", reply) + } +} + +func TestDockerLaunchChecksFixedBridgeNetworkBeforeCreate(t *testing.T) { + for _, driver := range []string{"bridge", "host"} { + t.Run(driver, func(t *testing.T) { + _, launcher, engine, state, _ := launchFixture(t) + launcher.policy.NetworkID = strings.Repeat("e", 64) + engine.networkDriver = driver + _, err := launcher.Launch(context.Background(), "v", state.Fence, "fixed-network") + if driver == "bridge" { + if err != nil { + t.Fatal(err) + } + } else { + if err == nil { + t.Fatal("non-bridge network was accepted") + } + engine.mu.Lock() + calls := append([]string(nil), engine.calls...) + engine.mu.Unlock() + for _, call := range calls { + if strings.HasPrefix(call, "POST ") { + t.Fatalf("unsafe network reached create: %v", calls) + } + } + } + }) + } +} diff --git a/worker/internal/slotgate/docker_verifier.go b/worker/internal/slotgate/docker_verifier.go index cacd8dce3..059cac4d2 100644 --- a/worker/internal/slotgate/docker_verifier.go +++ b/worker/internal/slotgate/docker_verifier.go @@ -2,13 +2,7 @@ package slotgate import ( "context" - "encoding/json" - "fmt" - "io" - "net" - "net/http" "os" - "path/filepath" "strings" "time" ) @@ -29,45 +23,15 @@ func nodeBootID() (string, error) { // The old identity must already have been recorded by a trusted node launcher. // A daemon ID mismatch, malformed response or denied request is not absence. func newDockerVerifier(socket, boot string) (TerminationVerifier, error) { - if !filepath.IsAbs(socket) || !validAuthorityID(boot) { + if !validAuthorityID(boot) { return nil, ErrInvalidAuthority } - transport := &http.Transport{DisableKeepAlives: true, DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, "unix", socket) - }} - client := &http.Client{Transport: transport, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + engine, err := newDockerEngine(socket) + if err != nil { + return nil, err + } call := func(ctx context.Context, method, path string, out any, allowed ...int) (int, error) { - req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil) - if err != nil { - return 0, err - } - res, err := client.Do(req) - if err != nil { - return 0, err - } - defer res.Body.Close() - ok := false - for _, status := range allowed { - if res.StatusCode == status { - ok = true - } - } - if !ok { - return res.StatusCode, fmt.Errorf("docker_status_%d", res.StatusCode) - } - data, err := io.ReadAll(io.LimitReader(res.Body, (1<<20)+1)) - if err != nil { - return res.StatusCode, err - } - if len(data) > 1<<20 { - return res.StatusCode, ErrInvalidAuthority - } - if out != nil && res.StatusCode == 200 { - if err = json.Unmarshal(data, out); err != nil { - return res.StatusCode, err - } - } - return res.StatusCode, nil + return engine.call(ctx, method, path, nil, out, allowed...) } type inspectState struct { ID string `json:"Id"` diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index a272a9f27..67c5f6171 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -133,6 +133,12 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. state, err = s.authority.Revoke(request.VMID, request.Fence) case "advance": state, err = s.authority.Advance(ctx, request.VMID, request.Fence) + case "launch": + if s.launcher == nil || request.Kind != "" || request.Ticket != "" { + err = ErrInvalidAuthority + break + } + state, err = s.launcher.Launch(ctx, request.VMID, request.Fence, request.CommandID) default: err = ErrInvalidAuthority } diff --git a/worker/internal/slotgate/slotgate.go b/worker/internal/slotgate/slotgate.go index a3f307f50..ebb038f92 100644 --- a/worker/internal/slotgate/slotgate.go +++ b/worker/internal/slotgate/slotgate.go @@ -32,9 +32,10 @@ type Config struct { TLSDir string `json:"tls_dir"` // Experimental node fencing. Raw SSH/Docker/SFTP permissions and controller // admission wiring must be migrated before this can be enabled for HA. - Fenced bool `json:"fenced,omitempty"` - AuthorityDir string `json:"authority_dir,omitempty"` - DockerSocket string `json:"docker_socket,omitempty"` + Fenced bool `json:"fenced,omitempty"` + AuthorityDir string `json:"authority_dir,omitempty"` + DockerSocket string `json:"docker_socket,omitempty"` + Launch *DockerLaunchPolicy `json:"launch,omitempty"` } // Preface: "KSG1 \n"; reply "OK\n" or "ERR \n". @@ -58,6 +59,7 @@ type Server struct { slots chan struct{} wg sync.WaitGroup authority *Authority + launcher *dockerLauncher } func New(cfg Config) (*Server, error) { @@ -104,6 +106,13 @@ func New(cfg Config) (*Server, error) { if err != nil { return nil, err } + if cfg.Launch != nil { + srv.launcher, err = newDockerLauncher(socket, cfg.Root, boot, srv.authority, *cfg.Launch) + if err != nil { + srv.authority.Close() + return nil, err + } + } } return srv, nil } From 97e00cf3fcecec42b9088c006b6fb3846ced211d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:30:30 +0800 Subject: [PATCH 253/473] test(logs): reproduce PostgreSQL NUL persistence failures --- test/e2e/request-log.e2e.test.mjs | 41 +++++++++++++++++++++++++++++ test/unit/request-log.test.mjs | 43 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/test/e2e/request-log.e2e.test.mjs b/test/e2e/request-log.e2e.test.mjs index bf2864cf4..1abd7bf22 100644 --- a/test/e2e/request-log.e2e.test.mjs +++ b/test/e2e/request-log.e2e.test.mjs @@ -4,6 +4,47 @@ import { startGateway, api } from '../harness.mjs' const MODEL = 'claude-haiku-4-5-20251001' +test('NUL-containing rejected requests preserve panel queries and backups', async () => { + const gw = await startGateway({ env: { KIN_REQUEST_LOG_MODE: 'normal', KIN_BACKUP_DISABLED: '1' } }) + try { + for (const mode of ['normal', 'debug']) { + const model = 'invalid\0model' + const rejected = await api(gw, 'POST', '/v1/messages', { + headers: { 'x-kin-log': mode }, + body: { model, max_tokens: 8, messages: [{ role: 'user', content: 'hi' }] }, + }) + assert.equal(rejected.status, 400, rejected.text) + const rid = rejected.headers.get('x-request-id') + const logs = await api(gw, 'GET', '/api/panel/request-logs?mode=normal&limit=10') + assert.equal(logs.status, 200, logs.text) + const row = logs.json.items.find((item) => item.request_id === rid) + assert.ok(row, `rejected ${mode} request must remain logged`) + assert.equal(row.status, 400) + assert.equal(row.model, 'invalid\\u0000model') + assert.equal(row.requested_model, 'invalid\\u0000model') + if (mode === 'debug') { + const debug = await api(gw, 'GET', `/api/panel/request-logs/${rid}`) + assert.equal(debug.status, 200, debug.text) + assert.equal(debug.json.item.inbound_body.model, model) + } + const keys = await api(gw, 'GET', '/api/panel/api-keys') + assert.equal(keys.status, 200, keys.text) + } + const backup = await api(gw, 'POST', '/api/panel/backups') + assert.equal(backup.status, 201, backup.text) + assert.equal(backup.json.item.status, 'ok') + const valid = await api(gw, 'POST', '/v1/messages', { + body: { model: MODEL, max_tokens: 8, messages: [{ role: 'user', content: 'after rejected requests' }] }, + }) + assert.equal(valid.status, 200, valid.text) + const logs = await api(gw, 'GET', '/api/panel/request-logs?mode=normal&limit=10') + assert.equal(logs.status, 200, logs.text) + assert.ok(logs.json.items.some((item) => item.request_id === valid.headers.get('x-request-id'))) + } finally { + await gw.stop() + } +}) + test('normal mode writes summary log with X-Request-ID', async () => { const gw = await startGateway({ mockText: 'logged', env: { KIN_REQUEST_LOG_MODE: 'normal' } }) try { diff --git a/test/unit/request-log.test.mjs b/test/unit/request-log.test.mjs index 482fc6937..3f6f53c6f 100644 --- a/test/unit/request-log.test.mjs +++ b/test/unit/request-log.test.mjs @@ -14,6 +14,7 @@ import { } from '../../src/lib/admin/request-log.mjs' import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' import { GroupsRepo } from '../../src/lib/db/repos/groups-repo.mjs' +import { trackPendingWrite, flushPendingWrites } from '../../src/lib/db/pending-writes.mjs' import { gateVerdict } from '../../src/lib/protocol/intercept-stats.mjs' function tmpStore(mode = 'normal') { @@ -73,6 +74,48 @@ test('normal mode writes jsonl without body', async () => { assert.equal((await store.listDebug({ limit: 5 })).length, 0) }) +for (const mode of ['normal', 'debug']) { + test(`${mode} logs escape NUL fields and leave response writes drainable`, async () => { + const store = tmpStore(mode) + const ctx = store.start( + { method: 'POST', headers: {}, socket: {} }, + { protocol: 'anthropic.messages', pathName: '/v1/messages' }, + ) + const model = 'invalid\0model' + const inbound = { model, messages: [{ role: 'user', content: '文本\0content' }] } + const summary = await trackPendingWrite( + store.finish(ctx, { + status: 400, + model, + requested_model: model, + upstream_model: 'upstream\0model', + error_message: 'invalid model: ' + model, + session_id: 'session\0id', + inbound_body: inbound, + }), + ) + await flushPendingWrites() + await flushPendingWrites() + const row = await store.repo.getByRequestId(ctx.request_id) + for (const [field, expected] of Object.entries({ + model: 'invalid\\u0000model', + requested_model: 'invalid\\u0000model', + upstream_model: 'upstream\\u0000model', + error_message: 'invalid model: invalid\\u0000model', + session_id: 'session\\u0000id', + })) { + assert.equal(row[field], expected, field) + assert.equal(summary[field], expected, field) + } + if (mode === 'debug') { + const debug = await store.getDebug(ctx.request_id) + assert.equal(debug.model, summary.model) + assert.deepEqual(debug.inbound_body, inbound) + } + assert.equal(inbound.model, model, 'logging must not mutate the request') + }) +} + test('summary persists protocol usage detail (cache/model/ttft/stop_reason)', async () => { const store = tmpStore('normal') const ctx = store.start( From 1c034cc654afd6ed4f8fb51e28723e5d3471720c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 17:31:14 +0800 Subject: [PATCH 254/473] fix(logs): escape NUL characters before PostgreSQL writes --- src/lib/admin/request-log.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/admin/request-log.mjs b/src/lib/admin/request-log.mjs index 2d004cb0f..a047c4405 100644 --- a/src/lib/admin/request-log.mjs +++ b/src/lib/admin/request-log.mjs @@ -585,6 +585,11 @@ export class RequestLogStore { const classified = classifyRequestError(summary) if (classified?.error_class) summary.error_class = classified.error_class + // PostgreSQL text rejects NUL; JSON debug bodies can retain the original request. + for (const [key, value] of Object.entries(summary)) { + if (typeof value === 'string') summary[key] = value.replaceAll('\0', '\\u0000') + } + await this.repo.insertSummaryIfAbsent(summary) if (this.jsonlMirror) this._appendJsonl(summary) this._mem.push(summary) From 2a58a9e97080324ecc03a2becbb9b983c1496259 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:46:56 +0800 Subject: [PATCH 255/473] test: reject package writes before VM ownership is acquired --- test/unit/panel-proxy-vm-mutations.test.mjs | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 98bb572e3..0e7cea9af 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -10,6 +10,7 @@ import { ProxyPool } from '../../src/lib/vm/proxy-pool.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { SharedProxyOperations } from '../../src/lib/vm/shared-proxy-operations.mjs' +import { buildVmPackage } from '../../src/lib/vm/vm-package.mjs' async function fixture(t) { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-proxy-claims-')) @@ -219,3 +220,31 @@ test('proxy unbind awaits confirmed helper stop before releasing its VM claim', assert.equal(next.ok, true) await next.release() }) + +test('package update rejects global executions before its first metadata or credential write', async (t) => { + const f = await fixture(t) + const file = path.join(f.project, 'vms', 'vm-a.json') + const before = fs.readFileSync(file, 'utf8') + await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + vmId: 'vm-a', + accountId: 'synthetic', + vmLimit: 1, + accountLimit: 1, + }) + const body = buildVmPackage({ vm: { id: 'vm-a', name: 'must not be written', timezone: 'UTC' } }) + const h = f.handler(body, { packageImportBringUp: false }) + await h.request('PUT', '/api/panel/vms/vm-a/package') + assert.equal(h.response.status, 409) + assert.equal(fs.readFileSync(file, 'utf8'), before) +}) + +test('package create rejects a busy physical alias before assigning an identity', async (t) => { + const f = await fixture(t) + await f.otherOperations.tryAcquire({ vmId: 'vm-new' }) + const body = buildVmPackage({ vm: { id: 'new', name: 'synthetic', timezone: 'UTC' } }) + const h = f.handler(body, { packageImportBringUp: false }) + await h.request('POST', '/api/panel/vms/package') + assert.equal(h.response.status, 409) + assert.equal(h.response.body.error.code, 'vm_operation_active') + assert.equal(fs.existsSync(path.join(f.project, 'vms', 'new.json')), false) +}) From a6ae7dfe68d9a3a0136d3f0e0d9b49655039e675 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:47:04 +0800 Subject: [PATCH 256/473] test: provide complete execution limits for package conflict --- test/unit/panel-proxy-vm-mutations.test.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 0e7cea9af..9d9fecdaa 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -230,6 +230,8 @@ test('package update rejects global executions before its first metadata or cred accountId: 'synthetic', vmLimit: 1, accountLimit: 1, + fableLimit: 0, + rpmLimit: 0, }) const body = buildVmPackage({ vm: { id: 'vm-a', name: 'must not be written', timezone: 'UTC' } }) const h = f.handler(body, { packageImportBringUp: false }) From f666c9234e5b55a42384ced566465e88da1332e3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:47:34 +0800 Subject: [PATCH 257/473] fix: admit background monitor preparation after startup scope closes --- src/lib/admin/health-probe.mjs | 9 +- src/lib/admin/vm-test-schedule.mjs | 58 ++++++++---- src/lib/oauth/credential-refresh-monitor.mjs | 9 +- src/lib/oauth/usage-probe-monitor.mjs | 9 +- .../unit/maintenance-monitor-startup.test.mjs | 88 +++++++++++++++++++ 5 files changed, 154 insertions(+), 19 deletions(-) diff --git a/src/lib/admin/health-probe.mjs b/src/lib/admin/health-probe.mjs index 776f49ccc..ad117fade 100644 --- a/src/lib/admin/health-probe.mjs +++ b/src/lib/admin/health-probe.mjs @@ -14,6 +14,7 @@ import { detectInboundPlatform } from '../protocol/platform-detect.mjs' import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { getSharedRoutingState } from './shared-routing-config.mjs' +import { getActivityCoordinator } from '../db/database.mjs' export const HEALTH_REAL_HEADER = 'x-kin-health-real' export const HEALTH_VIA_CACHE = 'health-cache' @@ -831,7 +832,13 @@ export function createHealthProbeMonitor(opts = {}) { const routing = getSharedRoutingState(sharedTasks?.repo.db) if (!routing) throw Object.assign(new Error('shared routing unavailable'), { code: 'routing_config_unavailable' }) - await routing.refresh() + const coordinator = getActivityCoordinator(routing.db) + if (!coordinator) + throw Object.assign(new Error('activity coordinator unavailable'), { code: 'maintenance_unavailable' }) + await coordinator.runActivity({ kind: 'health_probe_configuration' }, async () => ({ + ended: true, + value: await routing.refresh(), + })) } if (scheduled && (current !== generation || !config.enabled)) return getSnapshot('anthropic') const probe = async () => { diff --git a/src/lib/admin/vm-test-schedule.mjs b/src/lib/admin/vm-test-schedule.mjs index 02105eb40..9002a9416 100644 --- a/src/lib/admin/vm-test-schedule.mjs +++ b/src/lib/admin/vm-test-schedule.mjs @@ -7,6 +7,8 @@ import { getVm, listVms } from '../vm/vm-registry.mjs' import { listTestableModels } from './vm-test-chat.mjs' import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' import { VmTestScheduleResultsRepo } from '../db/repos/vm-test-schedule-results-repo.mjs' +import { getActivityCoordinator } from '../db/database.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' export const VM_TEST_SCHEDULE_PREFIX = 'vm_test_schedule:' export const VM_TEST_SCHEDULE_DEFAULT_KEY = 'vm_test_schedule_default' @@ -375,6 +377,25 @@ export class VmTestScheduleRunner { return settings } + async _activity(kind, callback) { + const coordinator = this.sharedTasks && getActivityCoordinator(this.sharedTasks.db) + if (!coordinator && isKubernetesMode()) + throw Object.assign(new Error('activity coordinator unavailable'), { code: 'maintenance_unavailable' }) + if (coordinator) { + try { + // Restore rebind is already owned by the exact maintenance operation; + // admitting an ordinary activity would correctly be refused in REBINDING. + const scope = coordinator.assertActivity() + if (scope.operation || scope.transferredTo) return callback() + } catch (error) { + if (error.code !== 'maintenance_activity_closed') throw error + } + } + return coordinator + ? coordinator.runActivity({ kind }, async () => ({ ended: true, value: await callback() })) + : callback() + } + _vm(vmId) { return this.listVms().find((vm) => vm.id === vmId) || getVm(this.projectRoot, vmId) } @@ -647,7 +668,7 @@ export class VmTestScheduleRunner { reload() { const generation = ++this.discoveryGeneration - const pending = this._reload(generation) + const pending = this._activity('vm_test_discovery', () => this._reload(generation)) this.discoveries.add(pending) pending.then( () => this.discoveries.delete(pending), @@ -708,21 +729,26 @@ export class VmTestScheduleRunner { if (!this.started || this.running.has(vmId)) return this.running.add(vmId) try { - const settings = this._settings() - if (this.sharedTasks) await settings.refresh?.() - if (!this.started) return - if (!this._vm(vmId)) { - await this.remove(vmId) - return - } - const schedule = this._stored(vmId) - if (!schedule?.enabled) return - if (expectedRevision != null && expectedRevision !== schedule.revision) { - this.discoveryGeneration++ - this._syncJob(vmId, schedule) - return - } - const startedAt = this.sharedTasks ? await this.sharedTasks.now() : this.now() + const prepared = await this._activity('vm_test_prepare', async () => { + const settings = this._settings() + if (this.sharedTasks) await settings.refresh?.() + if (!this.started) return + if (!this._vm(vmId)) { + await this.remove(vmId) + return + } + const schedule = this._stored(vmId) + if (!schedule?.enabled) return + if (expectedRevision != null && expectedRevision !== schedule.revision) { + this.discoveryGeneration++ + this._syncJob(vmId, schedule) + return + } + const startedAt = this.sharedTasks ? await this.sharedTasks.now() : this.now() + return { settings, schedule, startedAt } + }) + if (!prepared) return + const { settings, schedule, startedAt } = prepared const execute = async () => { if (!this.started || (this.resultRepo && !(await this.resultRepo.isCurrent(vmId, schedule.revision)))) { return { notStarted: true } diff --git a/src/lib/oauth/credential-refresh-monitor.mjs b/src/lib/oauth/credential-refresh-monitor.mjs index 968e9765e..28f829eed 100644 --- a/src/lib/oauth/credential-refresh-monitor.mjs +++ b/src/lib/oauth/credential-refresh-monitor.mjs @@ -10,6 +10,7 @@ import { hasRefreshPresence, needsRefresh } from './oauth-credentials.mjs' import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { getSharedRoutingState } from '../admin/shared-routing-config.mjs' +import { getActivityCoordinator } from '../db/database.mjs' export const DEFAULT_CREDENTIAL_REFRESH = Object.freeze({ enabled: true, @@ -118,7 +119,13 @@ export function createCredentialRefreshMonitor(opts = {}) { const routing = getSharedRoutingState(sharedTasks?.repo.db) if (!routing) throw Object.assign(new Error('shared routing unavailable'), { code: 'routing_config_unavailable' }) - await routing.refresh() + const coordinator = getActivityCoordinator(routing.db) + if (!coordinator) + throw Object.assign(new Error('activity coordinator unavailable'), { code: 'maintenance_unavailable' }) + await coordinator.runActivity({ kind: 'credential_refresh_configuration' }, async () => ({ + ended: true, + value: await routing.refresh(), + })) } if (scheduled && (current !== generation || !config.enabled)) return lastRun const started = nowFn() diff --git a/src/lib/oauth/usage-probe-monitor.mjs b/src/lib/oauth/usage-probe-monitor.mjs index fc66254f7..2280fc8a3 100644 --- a/src/lib/oauth/usage-probe-monitor.mjs +++ b/src/lib/oauth/usage-probe-monitor.mjs @@ -17,6 +17,7 @@ import { hasLiveExtraSample, hasOfficialUsageSample, parseResetMs } from '../poo import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { getSharedRoutingState } from '../admin/shared-routing-config.mjs' +import { getActivityCoordinator } from '../db/database.mjs' export const DEFAULT_USAGE_PROBE = Object.freeze({ enabled: false, interval_sec: 60, @@ -212,7 +213,13 @@ export function createUsageProbeMonitor(opts = {}) { if (!isKubernetesMode()) return const routing = getSharedRoutingState(sharedTasks?.repo.db) if (!routing) throw Object.assign(new Error('shared routing unavailable'), { code: 'routing_config_unavailable' }) - await routing.refresh() + const coordinator = getActivityCoordinator(routing.db) + if (!coordinator) + throw Object.assign(new Error('activity coordinator unavailable'), { code: 'maintenance_unavailable' }) + await coordinator.runActivity({ kind: 'usage_probe_configuration' }, async () => ({ + ended: true, + value: await routing.refresh(), + })) } const listTargets = () => { diff --git a/test/unit/maintenance-monitor-startup.test.mjs b/test/unit/maintenance-monitor-startup.test.mjs index 3ad6d7352..02c4276b2 100644 --- a/test/unit/maintenance-monitor-startup.test.mjs +++ b/test/unit/maintenance-monitor-startup.test.mjs @@ -181,3 +181,91 @@ test('VM schedule cron preparation enters a fresh activity before refreshing set await runner._run(vm.id, plan.revision) assert.equal(calls, 1) }) + +test('a monitor can replace the closed initialization scope inherited by an asynchronous callback', async (t) => { + const { db, coordinator, tasks } = await fixture(t) + let resume + const delayed = new Promise((resolve) => { + resume = resolve + }) + let calls = 0 + const monitor = createCredentialRefreshMonitor({ + sharedTasks: tasks, + listTargets: () => [vm], + refreshOne: async () => { + await db.query('SELECT 1') + calls++ + return { ok: true } + }, + }) + t.after(() => monitor.stop()) + const initialization = await coordinator.admit({ kind: 'synthetic_initialization' }) + const background = initialization.run(() => delayed.then(() => monitor.runOnce())) + await initialization.complete({ ended: true }) + resume() + await background + assert.equal(calls, 1) +}) + +test('DRAINING rejects monitor configuration and VM discovery before any callback or claim', async (t) => { + const { db, root, coordinator, tasks } = await fixture(t) + let calls = 0 + const callback = async () => { + calls++ + return { ok: true } + } + const credential = createCredentialRefreshMonitor({ + sharedTasks: tasks, + listTargets: () => [vm], + refreshOne: callback, + }) + const health = createHealthProbeMonitor({ sharedTasks: tasks, listTargets: () => [vm], runChat: callback }) + const usage = createUsageProbeMonitor({ sharedTasks: tasks, listTargets: () => [vm], probeOne: callback }) + const runner = new VmTestScheduleRunner({ + projectRoot: root, + sharedTasks: tasks, + settings: () => new RuntimeSettings(db), + list: () => [vm], + runTest: callback, + }) + const operation = await coordinator.beginMaintenance({ kind: 'backup' }) + try { + await assert.rejects(credential.runOnce(), { code: 'database_maintenance' }) + await assert.rejects(health.runRealProbe(), { code: 'database_maintenance' }) + await assert.rejects(usage.runOnce(), { code: 'database_maintenance' }) + await assert.rejects(runner.start(), { code: 'database_maintenance' }) + assert.equal(calls, 0) + assert.equal((await db.pool.query('SELECT count(*)::integer AS n FROM background_task_runs')).rows[0].n, 0) + } finally { + await Promise.all([credential.stop(), health.stop(), usage.stop(), runner.stop()]) + await operation.close() + await coordinator.cancelDrain(operation.operationId) + } +}) + +test('the exact restore owner can reload VM schedules inside its trusted rebind scope', async (t) => { + const { db, root, coordinator, tasks } = await fixture(t) + const runner = new VmTestScheduleRunner({ + projectRoot: root, + sharedTasks: tasks, + settings: () => new RuntimeSettings(db), + list: () => [vm], + runTest: async () => ({ ok: true }), + }) + t.after(() => runner.stop()) + const request = await coordinator.admit({ kind: 'http' }) + await request.run(async () => { + const operation = await coordinator.beginMaintenanceFromCurrentActivity({ kind: 'restore' }) + try { + await operation.beginRestore() + await operation.beginRebind() + await operation.runAdmin(() => runner.reload()) + await runner.reload() + await coordinator.acknowledgeEpoch(2) + await operation.resume() + } finally { + await operation.close() + } + }) + await request.complete({ ended: true }) +}) From e5a7cf9eddaa973706aa48dff4342d0f317c6ad2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:48:13 +0800 Subject: [PATCH 258/473] fix: acquire package target claims before writing VM files --- src/lib/admin/panel-routes.mjs | 157 ++++++++++++-------- src/lib/vm/vm-package.mjs | 3 +- test/unit/panel-proxy-vm-mutations.test.mjs | 23 +++ 3 files changed, 123 insertions(+), 60 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 56f9ebf1c..a7ebfb81c 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -64,7 +64,12 @@ import { oauthToSub2apiExport, parseCredentialEdit, } from '../oauth/sub2api-account.mjs' -import { officialCcHome, readOfficialCcStatus, scheduleOfficialCcBootstrap } from '../oauth/official-cc-bootstrap.mjs' +import { + officialCcHome, + readOfficialCcStatus, + runOfficialCcBootstrap, + scheduleOfficialCcBootstrap, +} from '../oauth/official-cc-bootstrap.mjs' import { invalidateLiveCredentialCache } from './panel-live-credentials.mjs' import { ensureCodexKernel, stopCodexKernel, finishCodexRecycle } from '../transport/codex-kernel-supervisor.mjs' import { listOfficialModels, seedModelCatalog } from '../protocol/models.mjs' @@ -785,14 +790,16 @@ export function createPanelHandler(ctx) { return { ok: true, via, node_id: host.nodeId || null, proxy_id: proxy.id || null, geo: r.geo } } - async function bringUpImportedVm(id) { + async function bringUpImportedVm(id, operation) { let vm = getVm(cfg.paths.project, id) let startError = null if (!vm) return { start_error: 'vm not found', official_cc_bootstrap: null, probe: null } if (!hasBoundExit(vm.proxy)) { startError = 'no_exit' } else { - const boot = await startSlotReady(vm, cfg.paths.project, { routing: ctx.routingConfig }) + const boot = await vmEffect(operation, () => + (ctx.vmMutationOps?.startSlotReady || startSlotReady)(vm, cfg.paths.project, { routing: ctx.routingConfig }), + ) vm = getVm(cfg.paths.project, id) || vm if (!boot?.ok) { startError = boot?.error || boot?.reason || 'runtime start failed' @@ -813,8 +820,10 @@ export function createPanelHandler(ctx) { else if (!vmHasClaudeCredential(vm)) bootstrap = { scheduled: false, reason: 'no_credential' } else if (!canOfficialCc(credentialModeOfVm(vm))) { bootstrap = { scheduled: false, reason: 'credential_mode_unsupported' } + } else if (!slotHost(vm).supports('official_cc')) { + bootstrap = { scheduled: false, reason: 'remote_unsupported' } } else { - bootstrap = scheduleOfficialCcBootstrap({ + const options = { vmId: id, projectRoot: cfg.paths.project, vm, @@ -825,7 +834,13 @@ export function createPanelHandler(ctx) { routingFile: routingConfigPath, config: ctx.routingConfig?.official_cc, credentialMode: credentialModeOfVm(vm), - }) + } + if (isKubernetesMode()) { + const completed = await vmEffect(operation, () => + (ctx.vmMutationOps?.runOfficialCcBootstrap || runOfficialCcBootstrap)(options), + ) + bootstrap = { scheduled: true, vmId: id, completed: true, ok: completed?.ok === true } + } else bootstrap = scheduleOfficialCcBootstrap(options) } const probed = await panel.buildProbeOne({ cfg, accountQuota, id, routingConfig: ctx.routingConfig }) const probe = probed?.status @@ -3404,7 +3419,7 @@ export function createPanelHandler(ctx) { // POST /api/panel/vms/:id/circuit/reset — close a tripped Claude unit circuit if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/circuit\/reset$/.test(p)) { const id = decodeURIComponent(p.split('/')[4]) - const result = panel.resetVmCircuit({ cfg, poolScheduler: ctx.poolScheduler, id }) + const result = await panel.resetVmCircuit({ cfg, poolScheduler: ctx.poolScheduler, id }) if (result.status) return json(res, result.status, result.body) return json(res, 200, result) } @@ -4022,38 +4037,49 @@ export function createPanelHandler(ctx) { if (req.method === 'PUT' && /^\/api\/panel\/vms\/[^/]+\/package$/.test(p)) { const id = p.split('/')[4] const body = await readBody(req, 256 * 1024).catch(() => null) - const result = await commitVmPackage({ - projectRoot: cfg.paths.project, - parsed: parseVmPackage(body), - mode: 'update', - targetId: id, - owner: { role: panelIdentity(req).role, userId: req.panelUserId }, - proxyPool, - }) - if (!result.ok) return json(res, result.status, { ok: false, error: result.error }) - await fillTimezoneFromExit(id) - let worker = null - // The package may swap credentials or exit; a running worker keeps the old ones until reloaded. - if (ctx.packageImportBringUp !== false && result.vm.status === 'running' && process.env.KIN_CRS_MOCK !== '1') { - setVmSchedulable(cfg.paths.project, id, false, 'package_update_worker_reload') - worker = await reloadSlotReady(getVm(cfg.paths.project, id), cfg.paths.project, { - routing: ctx.routingConfig, + const parsed = parseVmPackage(body) + if (!parsed.ok) return json(res, 400, { ok: false, error: parsed.error }) + return mutateProxyVms([id], async (handles) => { + const result = await commitVmPackage({ + projectRoot: cfg.paths.project, + parsed, + mode: 'update', + targetId: id, + owner: { role: panelIdentity(req).role, userId: req.panelUserId }, + proxyPool, }) - if (worker.ok) restoreSchedulableIfReady(id) - } - const saved = getVm(cfg.paths.project, id) || result.vm - return json( - res, - 200, - panel.ok({ - vm: summarizeVm(saved, cfg.paths.project, ctx.routingConfig), - worker_reload: worker ? { ok: !!worker.ok, error: worker.ok ? null : worker.error || null } : null, - }), - ) + if (!result.ok) return json(res, result.status, { ok: false, error: result.error }) + await fillTimezoneFromExit(id) + let worker = null + // The package may swap credentials or exit; a running worker keeps the old ones until reloaded. + if ( + ctx.packageImportBringUp !== false && + result.vm.status === 'running' && + process.env.KIN_CRS_MOCK !== '1' + ) { + setVmSchedulable(cfg.paths.project, id, false, 'package_update_worker_reload') + worker = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.reloadSlotReady || reloadSlotReady)(getVm(cfg.paths.project, id), cfg.paths.project, { + routing: ctx.routingConfig, + }), + ) + if (worker.ok) restoreSchedulableIfReady(id) + } + const saved = getVm(cfg.paths.project, id) || result.vm + return json( + res, + 200, + panel.ok({ + vm: summarizeVm(saved, cfg.paths.project, ctx.routingConfig), + worker_reload: worker ? { ok: !!worker.ok, error: worker.ok ? null : worker.error || null } : null, + }), + ) + }) } if (req.method === 'POST' && p === '/api/panel/vms/package') { const body = await readBody(req, 256 * 1024).catch(() => null) const parsed = parseVmPackage(body) + if (!parsed.ok) return json(res, 400, { ok: false, error: parsed.error }) const ident = panelIdentity(req) if (parsed.ok && ident.role === 'user') { const ownerId = normalizeOwnerId(req.panelUserId) @@ -4070,32 +4096,45 @@ export function createPanelHandler(ctx) { { id: a.vm_id }, { id: a.account_id }, ]) - const result = await commitVmPackage({ - projectRoot: cfg.paths.project, - parsed, - mode: 'create', - owner: { role: ident.role, userId: req.panelUserId }, - proxyPool, - occupied: historic, - }) - if (!result.ok) return json(res, result.status, { ok: false, error: result.error }) - await fillTimezoneFromExit(result.vm.id) - const brought = - ctx.packageImportBringUp === false - ? { start_error: null, official_cc_bootstrap: null, probe: null } - : await bringUpImportedVm(result.vm.id) - const saved = getVm(cfg.paths.project, result.vm.id) || result.vm - return json( - res, - result.status, - panel.ok({ - vm: summarizeVm(saved, cfg.paths.project, ctx.routingConfig), - renamed_from: result.renamed_from, - start_error: brought.start_error, - official_cc_bootstrap: brought.official_cc_bootstrap, - probe: brought.probe, - }), - ) + const fixedId = isKubernetesMode() || ctx.vmOperations ? parsed.value.id : null + const commit = async (handles) => { + if (fixedId) { + const aliases = physicalVmAliases(fixedId) + const committed = await getDb() + .prepare('SELECT id FROM vms WHERE id = ANY($1::text[]) LIMIT 1') + .get(aliases) + if (committed || listVms(cfg.paths.project).some((vm) => aliases.includes(vm.id))) + return json(res, 409, { ok: false, error: { code: 'vm_id_exists', message: 'VM identity exists' } }) + } + const result = await commitVmPackage({ + projectRoot: cfg.paths.project, + parsed, + mode: 'create', + targetId: fixedId, + owner: { role: ident.role, userId: req.panelUserId }, + proxyPool, + occupied: historic, + }) + if (!result.ok) return json(res, result.status, { ok: false, error: result.error }) + await fillTimezoneFromExit(result.vm.id) + const brought = + ctx.packageImportBringUp === false + ? { start_error: null, official_cc_bootstrap: null, probe: null } + : await bringUpImportedVm(result.vm.id, handles.get(result.vm.id)) + const saved = getVm(cfg.paths.project, result.vm.id) || result.vm + return json( + res, + result.status, + panel.ok({ + vm: summarizeVm(saved, cfg.paths.project, ctx.routingConfig), + renamed_from: result.renamed_from, + start_error: brought.start_error, + official_cc_bootstrap: brought.official_cc_bootstrap, + probe: brought.probe, + }), + ) + } + return fixedId ? mutateProxyVms([fixedId], commit) : commit(new Map()) } // POST /api/panel/vms/create — configurable seed VM + pure Claude Code home diff --git a/src/lib/vm/vm-package.mjs b/src/lib/vm/vm-package.mjs index f88c57729..d49366ef1 100644 --- a/src/lib/vm/vm-package.mjs +++ b/src/lib/vm/vm-package.mjs @@ -602,10 +602,11 @@ export async function commitVmPackage({ // Tenants cannot read or write seed-settings; a package must not smuggle them in either. if (owner.role === 'user') spec.vm.seed_policy = null const create = mode === 'create' - let id = create ? spec.id : targetId + let id = create ? targetId || spec.id : targetId let renamedFrom = null if (!isValidVmId(id)) return { ok: false, status: 400, error: { code: 'invalid_id', message: 'vm.id 无效' } } if (create && fs.existsSync(path.join(projectRoot, 'vms', `${id}.json`))) { + if (targetId) return { ok: false, status: 409, error: { code: 'vm_id_exists', message: 'VM identity exists' } } renamedFrom = id id = nextFreePackageVmId(projectRoot, occupied) const oldLabel = slotLabel(renamedFrom) diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 9d9fecdaa..76b47a3bc 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -1,3 +1,4 @@ +import '../fixtures/kin-computer-test-env.mjs' import test from 'node:test' import assert from 'node:assert/strict' import fs from 'node:fs' @@ -250,3 +251,25 @@ test('package create rejects a busy physical alias before assigning an identity' assert.equal(h.response.body.error.code, 'vm_operation_active') assert.equal(fs.existsSync(path.join(f.project, 'vms', 'new.json')), false) }) + +test('Kubernetes package creation never renames outside its claimed identity', async (t) => { + const f = await fixture(t) + const before = fs.readFileSync(path.join(f.project, 'vms', 'vm-a.json'), 'utf8') + const h = f.handler(buildVmPackage({ vm: { id: 'vm-a', name: 'must not rename', timezone: 'UTC' } }), { + packageImportBringUp: false, + }) + await h.request('POST', '/api/panel/vms/package') + assert.equal(h.response.status, 409) + assert.equal(h.response.body.error.code, 'vm_id_exists') + assert.deepEqual(fs.readdirSync(path.join(f.project, 'vms')), ['vm-a.json']) + assert.equal(fs.readFileSync(path.join(f.project, 'vms', 'vm-a.json'), 'utf8'), before) +}) + +test('Kubernetes package creation refuses a committed alias missing from this Pod files', async (t) => { + const f = await fixture(t) + await f.peer.prepare("INSERT INTO vms (id, name) VALUES ('vm-new', 'synthetic')").run() + const h = f.handler(buildVmPackage({ vm: { id: 'new', timezone: 'UTC' } }), { packageImportBringUp: false }) + await h.request('POST', '/api/panel/vms/package') + assert.equal(h.response.status, 409) + assert.equal(fs.existsSync(path.join(f.project, 'vms', 'new.json')), false) +}) From 4fed2928f00a91a2738a6eb8bf7a09d3358c0928 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:48:41 +0800 Subject: [PATCH 259/473] test: seed complete committed VM package fixture --- test/unit/panel-proxy-vm-mutations.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 76b47a3bc..ddfa79b59 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -12,6 +12,7 @@ import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { SharedProxyOperations } from '../../src/lib/vm/shared-proxy-operations.mjs' import { buildVmPackage } from '../../src/lib/vm/vm-package.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' async function fixture(t) { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-proxy-claims-')) @@ -267,7 +268,7 @@ test('Kubernetes package creation never renames outside its claimed identity', a test('Kubernetes package creation refuses a committed alias missing from this Pod files', async (t) => { const f = await fixture(t) - await f.peer.prepare("INSERT INTO vms (id, name) VALUES ('vm-new', 'synthetic')").run() + await new VmsRepo(f.peer).upsertFromVmJson({ id: 'vm-new', status: 'stopped' }) const h = f.handler(buildVmPackage({ vm: { id: 'new', timezone: 'UTC' } }), { packageImportBringUp: false }) await h.request('POST', '/api/panel/vms/package') assert.equal(h.response.status, 409) From c5ed6d1bda6d64d1f3f6697b74c72dd4196cd8b1 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:48:58 +0800 Subject: [PATCH 260/473] test: reproduce process-local circuit admission across controllers --- test/unit/shared-unit-circuit.test.mjs | 85 ++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 test/unit/shared-unit-circuit.test.mjs diff --git a/test/unit/shared-unit-circuit.test.mjs b/test/unit/shared-unit-circuit.test.mjs new file mode 100644 index 000000000..b56977b35 --- /dev/null +++ b/test/unit/shared-unit-circuit.test.mjs @@ -0,0 +1,85 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' +import { UnitCircuit } from '../../src/lib/pool/unit-circuit.mjs' + +const candidate = { + accountId: 'circuit-account', + vmId: 'circuit-vm', + model: 'claude-test', + vm: { id: 'circuit-vm', policy: { maxConcurrency: 8 } }, + account: { account_id: 'circuit-account' }, + maxConcurrency: 8, +} + +async function replicas(threshold = 1) { + const firstDb = getDb() + const secondDb = await createRepositoryTestDatabase({ + connectionString: process.env.VM2API_TEST_DATABASE_URL, + schema: firstDb.schema, + }) + return [firstDb, secondDb].map((db) => { + const pool = new PoolScheduler({ + executionAdmissions: new SharedExecutionAdmissions(db), + accountQuota: { + db, + canAccept: () => ({ ok: true }), + limitFor: () => 8, + policyFor: () => ({}), + rpmLimitFor: () => 0, + }, + config: { circuit_failure_threshold: threshold, circuit_open_ms: 1000 }, + }) + // Separate process-local objects model two control-plane processes. + pool.unitCircuit = new UnitCircuit({ failureThreshold: threshold, openMs: 1000 }) + return { pool, runner: new FailoverRunner({ scheduler: pool }), db } + }) +} + +async function kubernetes(fn) { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + try { + await fn() + } finally { + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + } +} + +test('a circuit opened by one control plane blocks another control plane', () => + kubernetes(async () => { + const [first, second] = await replicas() + const lease = await first.pool.reserve(candidate) + assert.ok(lease) + await first.runner.noteUnitHealth({ ...candidate, ...lease }, { circuit: true }, { ok: false }) + await lease.release() + const next = await second.pool.reserve(candidate) + try { + assert.equal(next, null, 'an independent control plane must observe the shared open circuit') + } finally { + await next?.release() + } + })) + +test('failure streaks combine across control planes before the threshold', () => + kubernetes(async () => { + const peers = await replicas(3) + for (let index = 0; index < 3; index++) { + const peer = peers[index % 2] + const lease = await peer.pool.reserve(candidate) + assert.ok(lease) + await peer.runner.noteUnitHealth({ ...candidate, ...lease }, { circuit: true }, { ok: false }) + await lease.release() + } + const next = await peers[1].pool.reserve(candidate) + try { + assert.equal(next, null, 'three failures must open one shared circuit') + } finally { + await next?.release() + } + })) From 3e5b3b1ab51b61dad08257483090399d9d5ef996 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:50:50 +0800 Subject: [PATCH 261/473] test: retain official bootstrap ownership beyond HTTP scheduling --- .../official-cc-shared-lifecycle.test.mjs | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 test/unit/official-cc-shared-lifecycle.test.mjs diff --git a/test/unit/official-cc-shared-lifecycle.test.mjs b/test/unit/official-cc-shared-lifecycle.test.mjs new file mode 100644 index 000000000..db60f3264 --- /dev/null +++ b/test/unit/official-cc-shared-lifecycle.test.mjs @@ -0,0 +1,120 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb, getActivityCoordinator } from '../../src/lib/db/database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { + officialCcHome, + readOfficialCcStatus, + scheduleOfficialCcBootstrap, +} from '../../src/lib/oauth/official-cc-bootstrap.mjs' + +async function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'official-shared-lifecycle-')) + fs.mkdirSync(path.join(root, 'vms')) + // The baseline returns before seeing this credential-less record. It cannot run a real command. + const vm = { id: 'vm-synthetic', status: 'running', claude: { mode: 'oauth' } } + fs.writeFileSync(path.join(root, 'vms', vm.id + '.json'), JSON.stringify(vm)) + const db = getDb() + await withDatabaseActivities([db], () => Promise.resolve()) + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const mode = process.env.VM2API_CONTROL_PLANE_MODE + const mock = process.env.KIN_CRS_MOCK + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + delete process.env.KIN_CRS_MOCK + const tasks = new SharedBackgroundTasks(db) + const operations = new SharedVmOperations(db) + const other = new SharedVmOperations(peer) + let finish + const pending = new Promise((resolve) => { + finish = resolve + }) + const options = { + vmId: vm.id, + projectRoot: root, + vm, + manual: true, + force: true, + config: { resident: false }, + backgroundTasks: tasks, + vmOperations: operations, + runBootstrap: async () => pending, + } + t.after(async () => { + finish({ ok: true }) + await tasks.flush() + await peer.close() + if (mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + if (mock == null) delete process.env.KIN_CRS_MOCK + else process.env.KIN_CRS_MOCK = mock + fs.rmSync(root, { recursive: true, force: true }) + }) + return { db, peer, tasks, other, options, finish, vm, root } +} + +test('manual bootstrap rejects a busy VM before queued status or background effects are written', async (t) => { + const f = await fixture(t) + await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + vmId: f.vm.id, + accountId: 'synthetic', + vmLimit: 1, + accountLimit: 1, + fableLimit: 0, + rpmLimit: 0, + }) + await assert.rejects(async () => scheduleOfficialCcBootstrap(f.options), { code: 'vm_execution_active' }) + assert.equal(readOfficialCcStatus(officialCcHome(f.root, f.vm.id)), null) +}) + +test('scheduled bootstrap keeps an independent activity and VM claim until the original promise completes', async (t) => { + const f = await fixture(t) + const scheduled = await scheduleOfficialCcBootstrap(f.options) + assert.equal(scheduled.scheduled, true) + assert.equal( + (await f.peer.prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get()).n, + 1, + ) + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) + const coordinator = getActivityCoordinator(f.db) + assert.equal( + ( + await f.db.pool.query( + `SELECT count(*) AS n FROM ${coordinator.qualified}.activities WHERE kind='background' AND completed_at IS NULL`, + ) + ).rows[0].n, + '1', + ) + let drained = false + const flush = f.tasks.flush().then(() => { + drained = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(drained, false) + f.finish({ ok: true }) + await flush + const next = await f.other.tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, true) + await next.release() + assert.equal( + (await f.peer.prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get()).n, + 0, + ) +}) + +test('ordinary error and timeout results keep bootstrap task and VM ownership without TTL', async (t) => { + const f = await fixture(t) + await scheduleOfficialCcBootstrap(f.options) + f.finish({ ok: false, timed_out: true, error: 'synthetic timeout' }) + await f.tasks.flush() + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) + const claim = await f.peer.prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.notEqual(claim.unknown_at, null) + assert.equal(claim.finished_at, null) +}) From a81ca923d3c1ebdc5594f47b421bf9e5da5804e2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:51:27 +0800 Subject: [PATCH 262/473] test(slotgate): require durable Claude terminal acknowledgement and retain unknown responses --- .../internal/slotgate/claude_terminal_test.go | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 worker/internal/slotgate/claude_terminal_test.go diff --git a/worker/internal/slotgate/claude_terminal_test.go b/worker/internal/slotgate/claude_terminal_test.go new file mode 100644 index 000000000..0a8bae7cd --- /dev/null +++ b/worker/internal/slotgate/claude_terminal_test.go @@ -0,0 +1,189 @@ +package slotgate + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "sync" + "testing" + "time" +) + +const normalClaudeJSON = `{"id":"msg_normal","type":"message","role":"assistant","content":[{"type":"text","text":"done"}],"stop_reason":"end_turn"}` +const normalClaudeSSE = "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_normal\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[]}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" + +func claudeWire(id string) string { + body, _ := json.Marshal(map[string]any{"request_id": id, "body": map[string]any{"model": "synthetic-claude", "messages": []any{}}}) + return fmt.Sprintf("POST /internal/v1/messages HTTP/1.1\r\nHost: kernel\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s", len(body), body) +} + +func waitCommandStatus(t *testing.T, a *Authority, id string, want CommandStatus) { + t.Helper() + until := time.Now().Add(time.Second) + for { + state, err := a.State("v") + if err != nil { + t.Fatal(err) + } + if state.Commands[id].Status == want { + return + } + if time.Now().After(until) { + t.Fatalf("command status=%s, want %s", state.Commands[id].Status, want) + } + time.Sleep(time.Millisecond) + } +} + +func TestKSG2ClaudeNormalProtocolCompletionIsPersistedBeforeStatusAcknowledgement(t *testing.T) { + for _, mode := range []string{"json", "sse"} { + t.Run(mode, func(t *testing.T) { + addr, root, server, state, _ := fencedGate(t) + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mode == "json" { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, normalClaudeJSON) + } else { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, normalClaudeSSE) + } + })) + req := forwardRequest(t, addr, state, "normal-"+mode) + c, reader := gate2Connect(t, addr, testToken, req, claudeWire(req.CommandID)) + line, err := reader.ReadString('\n') + if err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + if _, err = io.Copy(io.Discard, response.Body); err != nil { + t.Fatal(err) + } + response.Body.Close() + c.Close() + waitCommandStatus(t, server.authority, req.CommandID, CommandCompleted) + req.Op = "command" + c, reader = gate2Connect(t, addr, testToken, req, "") + var reply struct { + OK bool `json:"ok"` + Command struct { + ID string `json:"id"` + Status CommandStatus `json:"status"` + } `json:"command"` + } + if err = json.NewDecoder(reader).Decode(&reply); err != nil { + t.Fatal(err) + } + c.Close() + if !reply.OK || reply.Command.ID != req.CommandID || reply.Command.Status != CommandCompleted { + t.Fatalf("node status: %+v", reply) + } + }) + } +} + +func TestKSG2ClaudeMessageStopWithoutHTTPCompletionRemainsPending(t *testing.T) { + addr, root, server, state, _ := fencedGate(t) + wrote, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + resume := func() { once.Do(func() { close(release) }) } + defer resume() + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, normalClaudeSSE) + w.(http.Flusher).Flush() + close(wrote) + select { + case <-release: + case <-r.Context().Done(): + } + })) + req := forwardRequest(t, addr, state, "delayed-eof") + c, reader := gate2Connect(t, addr, testToken, req, claudeWire(req.CommandID)) + defer c.Close() + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + <-wrote + current, err := server.authority.State("v") + if err != nil || current.Commands[req.CommandID].Status != CommandPending { + t.Fatalf("message_stop alone completed: %+v %v", current, err) + } + resume() + if _, err = io.Copy(io.Discard, response.Body); err != nil { + t.Fatal(err) + } + response.Body.Close() + waitCommandStatus(t, server.authority, req.CommandID, CommandCompleted) +} + +func TestKSG2ClaudeUnknownResponsesRemainPending(t *testing.T) { + for _, tc := range []struct { + name, contentType, body string + status int + truncate bool + trailer string + requestID string + }{ + {"cancel-json", "application/json", `{"ok":false,"error":{"type":"worker_error","code":"provider_error","message":"provider error: client disconnected; 429 limit"}}`, 502, false, "", ""}, + {"generic-error", "application/json", `{"type":"error","error":{"type":"api_error","message":"provider error"}}`, 200, false, "", ""}, + {"verified-without-terminal", "application/json", `{"type":"message","role":"assistant","content":[]}`, 200, false, "", ""}, + {"truncated-json", "application/json", normalClaudeJSON, 200, true, "", ""}, + {"stop-no-http-end", "text/event-stream", normalClaudeSSE, 200, true, "", ""}, + {"missing-message-stop", "text/event-stream", "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n", 200, false, "", ""}, + {"error-after-stop", "text/event-stream", normalClaudeSSE + "data: {\"type\":\"error\",\"error\":{\"type\":\"worker_error\",\"message\":\"client disconnected\"}}\n\n", 200, false, "", ""}, + {"incomplete-trailer", "text/event-stream", normalClaudeSSE, 200, false, "incomplete", ""}, + {"wrong-job-id", "application/json", normalClaudeJSON, 200, false, "", "other-request"}, + } { + t.Run(tc.name, func(t *testing.T) { + addr, root, server, state, _ := fencedGate(t) + finished := make(chan struct{}) + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(finished) + w.Header().Set("Content-Type", tc.contentType) + w.Header().Set("X-Terminal-State", "verified") + if tc.truncate { + w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+20)) + } + if tc.trailer != "" { + w.Header().Set("Trailer", "X-Terminal-State") + } + w.WriteHeader(tc.status) + fmt.Fprint(w, tc.body) + if tc.trailer != "" { + w.Header().Set("X-Terminal-State", tc.trailer) + } + })) + req := forwardRequest(t, addr, state, tc.name) + id := req.CommandID + if tc.requestID != "" { + id = tc.requestID + } + c, reader := gate2Connect(t, addr, testToken, req, claudeWire(id)) + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, response.Body) + response.Body.Close() + c.Close() + <-finished + time.Sleep(10 * time.Millisecond) + current, err := server.authority.State("v") + if err != nil || current.Commands[req.CommandID].Status != CommandPending { + t.Fatalf("unknown response released: %+v %v", current, err) + } + }) + } +} From e5e4849209ce527c1ff9f5097999d9181142c5bb Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:52:25 +0800 Subject: [PATCH 263/473] test: preserve explicit unknown official bootstrap results --- .../official-cc-shared-lifecycle.test.mjs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/test/unit/official-cc-shared-lifecycle.test.mjs b/test/unit/official-cc-shared-lifecycle.test.mjs index db60f3264..bb3083fc9 100644 --- a/test/unit/official-cc-shared-lifecycle.test.mjs +++ b/test/unit/official-cc-shared-lifecycle.test.mjs @@ -89,7 +89,7 @@ test('scheduled bootstrap keeps an independent activity and VM claim until the o `SELECT count(*) AS n FROM ${coordinator.qualified}.activities WHERE kind='background' AND completed_at IS NULL`, ) ).rows[0].n, - '1', + 1, ) let drained = false const flush = f.tasks.flush().then(() => { @@ -118,3 +118,47 @@ test('ordinary error and timeout results keep bootstrap task and VM ownership wi assert.notEqual(claim.unknown_at, null) assert.equal(claim.finished_at, null) }) + +test('a normally returned bootstrap error remains unknown even without a timeout flag', async (t) => { + const f = await fixture(t) + await scheduleOfficialCcBootstrap(f.options) + f.finish({ ok: false, error: 'synthetic worker ACK missing' }) + await f.tasks.flush() + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) +}) + +test('a successful response with an unknown auxiliary operation retains ownership', async (t) => { + const f = await fixture(t) + await scheduleOfficialCcBootstrap(f.options) + f.finish({ ok: true, vmOperationUncertain: true }) + await f.tasks.flush() + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) +}) + +test('successful bootstrap keeps ownership until queued persistence has completed', async (t) => { + const f = await fixture(t) + let finishFlush + let enteredFlush + const flushing = new Promise((resolve) => { + finishFlush = resolve + }) + const entered = new Promise((resolve) => { + enteredFlush = resolve + }) + f.options.flush = async () => { + enteredFlush() + await flushing + } + await scheduleOfficialCcBootstrap(f.options) + f.finish({ ok: true }) + await entered + try { + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) + } finally { + finishFlush() + } + await f.tasks.flush() + const next = await f.other.tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, true) + await next.release() +}) From 55b52bc540fbf756a5a8a3cdc66c1d84eddd8e5b Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:54:04 +0800 Subject: [PATCH 264/473] test: keep bootstrap claims after auxiliary profile timeouts --- test/unit/official-cc-shared-lifecycle.test.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/unit/official-cc-shared-lifecycle.test.mjs b/test/unit/official-cc-shared-lifecycle.test.mjs index bb3083fc9..7b70976d5 100644 --- a/test/unit/official-cc-shared-lifecycle.test.mjs +++ b/test/unit/official-cc-shared-lifecycle.test.mjs @@ -135,6 +135,14 @@ test('a successful response with an unknown auxiliary operation retains ownershi assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) }) +test('successful hello with an auxiliary profile timeout still retains ownership', async (t) => { + const f = await fixture(t) + await scheduleOfficialCcBootstrap(f.options) + f.finish({ ok: true, status: { profile_error: 'synthetic timeout', models_error: null } }) + await f.tasks.flush() + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) +}) + test('successful bootstrap keeps ownership until queued persistence has completed', async (t) => { const f = await fixture(t) let finishFlush From 157dbb86e31dfe5f1dadcd0ca36b9db1367eb3c3 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:54:34 +0800 Subject: [PATCH 265/473] test: release bootstrap validation failures before commands start --- test/unit/official-cc-shared-lifecycle.test.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/unit/official-cc-shared-lifecycle.test.mjs b/test/unit/official-cc-shared-lifecycle.test.mjs index 7b70976d5..07d9b3839 100644 --- a/test/unit/official-cc-shared-lifecycle.test.mjs +++ b/test/unit/official-cc-shared-lifecycle.test.mjs @@ -143,6 +143,16 @@ test('successful hello with an auxiliary profile timeout still retains ownership assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) }) +test('the real bootstrap can release a credential validation failure before any command is sent', async (t) => { + const f = await fixture(t) + delete f.options.runBootstrap + await scheduleOfficialCcBootstrap(f.options) + await f.tasks.flush() + const next = await f.other.tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, true) + await next.release() +}) + test('successful bootstrap keeps ownership until queued persistence has completed', async (t) => { const f = await fixture(t) let finishFlush From 3304c452ecd3d91913d60ca4746ab253315a2b22 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:54:47 +0800 Subject: [PATCH 266/473] fix: persist official bootstrap activity for its original promise --- src/lib/admin/panel-routes.mjs | 6 +- src/lib/oauth/official-cc-bootstrap.mjs | 120 +++++++++++++++++++++++- 2 files changed, 119 insertions(+), 7 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index a7ebfb81c..cf1199ade 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -4005,12 +4005,14 @@ export function createPanelHandler(ctx) { }) } const body = await readBody(req, 8 * 1024) - const scheduled = scheduleOfficialCcBootstrap({ + const scheduled = await scheduleOfficialCcBootstrap({ vmId: id, projectRoot: cfg.paths.project, force: body.force !== false, collectIdentity: collectSlotIdentity, - onStats: officialCcStatsHandler(id, vm.claude?.account_uuid), + onStats: isKubernetesMode() ? undefined : officialCcStatsHandler(id, vm.claude?.account_uuid), + onStatsFactory: isKubernetesMode() ? () => officialCcStatsHandler(id, vm.claude?.account_uuid) : undefined, + flush: flushPanelWrites, routingFile: routingConfigPath, config: ctx.routingConfig.official_cc, credentialMode: vm.claude?.mode, diff --git a/src/lib/oauth/official-cc-bootstrap.mjs b/src/lib/oauth/official-cc-bootstrap.mjs index 74aca43c3..fd7f7b24c 100644 --- a/src/lib/oauth/official-cc-bootstrap.mjs +++ b/src/lib/oauth/official-cc-bootstrap.mjs @@ -21,6 +21,9 @@ import { spawn, execFileSync } from 'node:child_process' import fs from 'node:fs' import path from 'node:path' import { fileURLToPath } from 'node:url' +import { randomUUID } from 'node:crypto' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' import { parseVmIndex, @@ -48,7 +51,9 @@ import { OFFICIAL_CLI_VERSION, loadVmIdentity, persistVmSettings } from '../iden import { applyOfficialFingerprintToVm, readOfficialCcIdentity } from '../identity/official-fingerprint.mjs' import { writeSlotSeedFiles, inferProjectRootFromCliHome } from '../vm/slot-seed.mjs' import { touchTelemetrySession } from '../vm/worker-telemetry.mjs' -import { atomicWriteJson, listVmRecordFiles } from '../vm/vm-file.mjs' +import { atomicWriteJson, listVmRecordFiles, withVmLock } from '../vm/vm-file.mjs' +import { currentVmOperationIds } from '../vm/shared-vm-operations.mjs' +import { flushVmDbSync } from '../vm/vm-db-sync.mjs' import { CONTAINER_CLI_NODE_BIN, normalizeOfficialCcInference, resolveOfficialCcInference } from '../vm/slot-engine.mjs' import { getVm } from '../vm/vm-registry.mjs' import { slotHost } from '../vm/slot-host.mjs' @@ -1084,15 +1089,15 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { slotOauth, retryDelayMs, } = applyOfficialCcConfig(rawOpts, config) - if (!vmId || !projectRoot) return { ok: false, error: 'vmId and projectRoot required' } + if (!vmId || !projectRoot) return { ok: false, error: 'vmId and projectRoot required', vmOperationNotStarted: true } const vmPath = path.join(projectRoot, 'vms', `${vmId}.json`) - if (!fs.existsSync(vmPath)) return { ok: false, error: 'vm not found' } + if (!fs.existsSync(vmPath)) return { ok: false, error: 'vm not found', vmOperationNotStarted: true } let vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) const homeDir = officialCcHome(projectRoot, vmId) const { uid, gid } = officialCcUidGid(vmId) const startedAt = new Date().toISOString() if (!canOfficialCc(credentialModeOfVm(vm))) { - return { ok: false, error: 'official Claude Code init requires full OAuth' } + return { ok: false, error: 'official Claude Code init requires full OAuth', vmOperationNotStarted: true } } if (!(hasAccessPresence(vm.claude) || hasRefreshPresence(vm.claude))) { const credFile = path.join(homeDir, '.claude', 'credentials.json') @@ -1103,7 +1108,7 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { live = !!(oauth.accessToken || oauth.access_token || oauth.refreshToken || oauth.refresh_token) } catch {} if (!live) { - return { ok: false, error: 'slot has no live OAuth credential' } + return { ok: false, error: 'slot has no live OAuth credential', vmOperationNotStarted: true } } } @@ -1454,6 +1459,7 @@ export function scheduleOfficialCcBootstrap(opts = {}) { return { scheduled: false, reason: 'already_initialized' } } } + if (isKubernetesMode()) return scheduleSharedOfficialCcBootstrap(merged) if (merged.projectRoot) { try { writeOfficialCcStatus(officialCcHome(merged.projectRoot, vmId), { @@ -1485,3 +1491,107 @@ export function scheduleOfficialCcBootstrap(opts = {}) { running.set(vmId, job) return { scheduled: true, vmId } } + +/** The background activity owns the original Promise, independently of the HTTP request. */ +async function scheduleSharedOfficialCcBootstrap(options) { + const { vmId, projectRoot } = options + const tasks = backgroundTasksFor(options.backgroundTasks) + const nested = currentVmOperationIds(tasks.db).size > 0 + let resolveReady + let rejectReady + const ready = new Promise((resolve, reject) => { + resolveReady = resolve + rejectReady = reject + }) + const job = tasks.run({ taskKey: `official_cc_bootstrap:${vmId}`, executionId: randomUUID() }, async () => { + let entered = false + try { + return await withVmLock( + path.join(projectRoot, 'vms', `${vmId}.json`), + async (operation) => { + entered = true + const vm = getVm(projectRoot, vmId) + if (!vm || !slotHost(vm).supports('official_cc')) { + throw Object.assign(new Error(vm ? 'remote official bootstrap unsupported' : 'VM not found'), { + code: vm ? 'remote_unsupported' : 'vm_not_found', + status: vm ? 409 : 404, + vmOperationNotStarted: true, + taskNotStarted: true, + }) + } + writeOfficialCcStatus(officialCcHome(projectRoot, vmId), { + status: 'running', + vm_id: vmId, + started_at: new Date().toISOString(), + force: options.force !== false, + step: 'queued', + }) + if (!nested) resolveReady({ scheduled: true, vmId }) + try { + let auxiliaryUnknown = false + const observe = (callback) => + callback && + (async (...args) => { + try { + const value = await callback(...args) + if (value?.ok === false) auxiliaryUnknown = true + return value + } catch (error) { + auxiliaryUnknown = true + throw error + } + }) + const result = await (options.runBootstrap || runOfficialCcBootstrap)({ + ...options, + vm, + collectIdentity: observe(options.collectIdentity), + onStats: observe(options.onStatsFactory ? options.onStatsFactory() : options.onStats), + }) + await options.flush?.() + await flushVmDbSync() + // Normal successful foreground CLI completion follows the existing protocol. + // A resident CLI remains active; a timeout/error supplies no termination proof. + const notStarted = result?.notStarted === true || result?.vmOperationNotStarted === true + const terminal = + !auxiliaryUnknown && + !result?.status?.profile_error && + !result?.status?.models_error && + !(options.syncTelemetry && result?.status?.telemetry_enabled && !result.status.sidecar_reloaded) && + result?.vmOperationUncertain !== true && + (notStarted || (result?.ok === true && !result.timed_out && options.resident !== true)) + if (!terminal) await operation.markUncertain() + return { terminal, value: result } + } catch (error) { + writeOfficialCcStatus(officialCcHome(projectRoot, vmId), { + status: 'error', + vm_id: vmId, + finished_at: new Date().toISOString(), + error: String(error.message || error).slice(0, 300), + }) + if (error.vmOperationNotStarted === true) error.taskNotStarted = true + throw error + } + }, + { operations: options.vmOperations, kind: 'official-cc-bootstrap' }, + ) + } catch (error) { + if (!entered) error.taskNotStarted = true + throw error + } + }) + running.set(vmId, job) + job + .then( + (result) => + resolveReady( + result.ran + ? { scheduled: true, vmId, ...(nested ? { completed: true } : {}) } + : { scheduled: false, reason: result.code }, + ), + rejectReady, + ) + .finally(() => { + if (running.get(vmId) === job) running.delete(vmId) + }) + return ready +} From 2da7e9f1720f3786d8bbb828ab307ed3417be32e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:56:39 +0800 Subject: [PATCH 267/473] test: reject VM metadata and identity writes during global executions --- test/unit/panel-vm-shared-mutations.test.mjs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/unit/panel-vm-shared-mutations.test.mjs b/test/unit/panel-vm-shared-mutations.test.mjs index 7ac6297c9..5a2f2d39b 100644 --- a/test/unit/panel-vm-shared-mutations.test.mjs +++ b/test/unit/panel-vm-shared-mutations.test.mjs @@ -86,6 +86,12 @@ for (const [method, suffix, body] of [ ['POST', '/stop', {}], ['DELETE', '', {}], ['POST', '/schedulable', { schedulable: false }], + ['PATCH', '/owner', { user_id: null }], + ['POST', '/reset-fingerprint', {}], + ['POST', '/reconcile-fingerprint', {}], + ['PUT', '/seed-settings', { theme: 'light' }], + ['POST', '/wrap-cli/repair', { mode: 'keep' }], + ['POST', '/collect-identity', {}], ]) { test(`panel ${method} VM${suffix} rejects a global execution admission before any mutation`, async (t) => { const f = await fixture(t) @@ -107,6 +113,19 @@ for (const [method, suffix, body] of [ }) } +test('batch fingerprint reconciliation acquires every target before changing any file', async (t) => { + const f = await fixture(t) + f.seed('vm-a') + f.seed('vm-b') + await f.otherOperations.tryAcquire({ vmId: 'vm-b' }) + const h = f.handler() + await h.request('POST', '/api/panel/vms/reconcile-fingerprints') + assert.equal(h.response.status, 409) + const next = await f.otherOperations.tryAcquire({ vmId: 'vm-a' }) + assert.equal(next.ok, true) + await next.release() +}) + test('create claims the new physical identity before writing a VM or fingerprint', async (t) => { const f = await fixture(t) await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' }) From 3cab77e4617b8f55a6842d35fc114a77fad982f0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:57:38 +0800 Subject: [PATCH 268/473] test: require execution receipts for Kubernetes fleet helper stops --- test/unit/egress-stop-proof.test.mjs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs index 5087fde4d..7a60cff02 100644 --- a/test/unit/egress-stop-proof.test.mjs +++ b/test/unit/egress-stop-proof.test.mjs @@ -269,3 +269,25 @@ test('a Kubernetes Codex exit-policy stop cannot infer another Pod kernel is abs assert.equal(result.ok, false) assert.equal(result.vmOperationUncertain, true) }) + +test('Kubernetes fleet telemetry cannot bypass the persistent helper execution receipt', async () => { + const source = fs.readFileSync(new URL('../../src/lib/vm/fleet-update.mjs', import.meta.url), 'utf8') + const body = source.slice(source.indexOf('export async function restartLocalEgress('), source.indexOf('async function restartLocalServices(')).replace('export ', '') + let legacySignals = 0 + let starts = 0 + const restart = vmContext.runInNewContext(`${body}; restartLocalEgress`, { + isLocalEgressProxy: () => false, isKubernetesMode: () => true, + runExclusiveProxyEgress: (_root, _id, callback) => callback(), + stopEgressProcessConfirmed: async () => ({ ok: false, code: 'egress_owner_unreachable', vmOperationUncertain: true }), + inspectEgressProcess: () => ({ ok: false }), + stopEgressProcess: () => { legacySignals++; return { ok: true } }, + egressPidsForProxyConfig: () => [], drainEgressForProxyConfig: async () => ({ ok: true }), + readEgressListenTcp: () => '', + startProxyEgress: async () => { starts++; return { ok: true } }, + }) + const result = await restart('/synthetic', { proxy: { id: 'synthetic' } }) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + assert.equal(legacySignals, 0) + assert.equal(starts, 0) +}) From b5d04411258913590cc59c5315eb40c1ebbe83bd Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:58:51 +0800 Subject: [PATCH 269/473] feat(slotgate): persist validated Claude terminal completion and expose authenticated status --- .../2026-10-21-node-fencing-launch-adapter.md | 18 +- worker/internal/slotgate/claude_terminal.go | 320 ++++++++++++++++++ .../internal/slotgate/claude_terminal_test.go | 126 ++++++- worker/internal/slotgate/ksg2.go | 49 ++- 4 files changed, 502 insertions(+), 11 deletions(-) create mode 100644 worker/internal/slotgate/claude_terminal.go diff --git a/docs/plans/2026-10-21-node-fencing-launch-adapter.md b/docs/plans/2026-10-21-node-fencing-launch-adapter.md index 58552faba..92a82ede1 100644 --- a/docs/plans/2026-10-21-node-fencing-launch-adapter.md +++ b/docs/plans/2026-10-21-node-fencing-launch-adapter.md @@ -29,7 +29,21 @@ KSG2 增加 `launch` 操作。客户端只提供 `vm_id`、创建操作时捕获 只读预检失败不会创建 launch claim，也不会发送 create。进入 dispatched 后的 create/start/inspect 或持久化错误保留 pending，不自动重试。节点重新打开遇到 pending launch 会保留为 unknown，进入 `REVOKING`。pending launch 阻止 generation 推进。create 返回晚于 revoke 时仍保存目标 ID，但不继续 start 或 bind。 -客户端连接关闭、超时或控制面 owner 消失均不是终止证明。正常转发目前也继续保留 pending；普通 HTTP/SSE 终态分类仍未接入。当前状态机没有自动处理未知 launch 的恢复操作。 +客户端连接关闭、超时或控制面 owner 消失均不是终止证明。未知响应和不受支持的协议继续保留 pending。当前状态机没有自动处理未知 launch 的恢复操作。 + +## 正常 Claude 终态与持久确认 + +节点只为 kernel 的 `POST /internal/v1/messages` 识别正常 Claude 终态。请求 envelope 的 `request_id` 必须与 node command ID 相同。其他 endpoint、worker、Codex、错误响应以及 ID 不符时继续 pending。 + +- JSON 必须是完整 HTTP 200 响应，包含合法 assistant message、非空 message ID、受支持的 content block 和明确 stop reason。当前只接受 `end_turn`、`tool_use`、`max_tokens`、`stop_sequence`。 +- SSE 必须包含合法 `message_start` 和 `message_stop`，content block 的 start/delta/stop 顺序须有效，且完整读取 HTTP body 与 trailers。仅 stop_reason 加 EOF 不会完成。message_stop 后仍会继续检查错误、未知帧及 trailers。 +- header 的 `verified`、HTTP EOF、普通拒绝、cancel ACK 和健康计数都不会单独完成命令。取消、内部错误、传输截断、已声明 terminal trailer 缺失或失效、重复 terminal、尾随错误、未知内容或超出解析上限时继续 pending。 + +解析器按增量观察原响应，不修改响应内容。JSON 上限为 1 MiB；SSE 每行上限为 64 KiB、单事件上限为 1 MiB。超过上限仍可转发响应，但不会确认完成。新 stop reason、内容块类型和协议目前不会自动纳入允许范围。 + +正常终态仅在 `CompleteCommand` 持久成功后成为 node 的 `completed`。新增只读 `command` RPC，要求原 captured fence、command ID、目标 kind 与 ticket，并只向已认证 token 返回状态，不暴露内部 nonce。客户端不能提交完成状态。 + +已转发的 HTTP body 不等于持久确认。未来 KSG2 客户端必须在释放 PG job 前取得 node 的持久 `completed`。若 node 写盘失败，状态查询会失败，不能据 HTTP 成功释放。此客户端与 PG 绑定尚未接线。 ## 容器与文件权限 @@ -54,7 +68,7 @@ Authority journal、其父目录、节点 agent 的可执行文件与配置， - 把容器生命周期和文件写权限迁移到节点服务。runtime Pod 不得继续持有 root/docker-group SSH、raw Docker socket、可写 SFTP 或替换 gate 的权限。bootstrap 凭据须与 runtime 分离。 - 迁移时撤销旧授权，并关闭已经建立的 SSH、SFTP、终端和 forwarding 会话。只修改新连接的认证方式不能阻止旧 owner 的延迟命令。 - 将 KSG2、captured generation、PG admission/job/VM operation 和文件发布接入生产调用链。fenced 节点不得回退到 KSG1 或原始 SSH 路径。 -- 提供节点可信 allocate、文件发布、未知 launch 的有证据恢复，以及正常响应终态处理。恢复只能关闭同一 immutable executor/incarnation 映射的作业；不能靠 TTL、ready_slots 或 cancel ACK 释放。 +- 提供节点可信 allocate、文件发布、未知 launch 的有证据恢复，以及尚未覆盖的响应终态处理。恢复只能关闭同一 immutable executor/incarnation 映射的作业；不能靠 TTL、ready_slots 或 cancel ACK 释放。 - 单独处理 KVM、原生 macOS 和 Codex 进程归属。现有 Codex Docker 容器仅持有身份和终端，停止它不能证明控制面 detached Codex 子进程已经终止。 - 在两个不同节点上的控制面验证同时执行、旧命令迟到、节点服务崩溃、Engine 故障及安全接管。本批合成测试不能代替该验收。 diff --git a/worker/internal/slotgate/claude_terminal.go b/worker/internal/slotgate/claude_terminal.go new file mode 100644 index 000000000..0635d9626 --- /dev/null +++ b/worker/internal/slotgate/claude_terminal.go @@ -0,0 +1,320 @@ +package slotgate + +import ( + "bytes" + "encoding/json" + "io" + "mime" + "net/http" + "strings" +) + +const claudeTerminalJSONMax = 1 << 20 +const claudeTerminalLineMax = 64 << 10 +const claudeTerminalEventMax = 1 << 20 + +// claudeTerminal observes the existing Claude protocol without changing the +// forwarded bytes. It is intentionally stricter than the display assembler: +// stop_reason plus EOF without message_stop is not accepted for SSE. Errors, +// unknown frames and parser limits retain the command. No error text is used +// to turn an internal cancellation into a normal provider rejection. +type claudeTerminal struct { + body io.ReadCloser + mode string + eligible bool + invalid bool + eof bool + read int64 + jsonBody []byte + line []byte + data []byte + event string + started bool + stopped bool + blocks map[int]bool + nextBlock int +} + +func observeClaudeTerminal(response *http.Response, request *http.Request, body []byte, commandID, kind string) *claudeTerminal { + observer := &claudeTerminal{body: response.Body, blocks: map[int]bool{}} + if kind != "kernel" || request.Method != http.MethodPost || request.URL.Path != "/internal/v1/messages" || request.URL.RawQuery != "" || response.StatusCode != http.StatusOK { + return observer + } + var envelope struct { + RequestID string `json:"request_id"` + } + if err := json.Unmarshal(body, &envelope); err != nil || envelope.RequestID != commandID { + return observer + } + media, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) + if err != nil || (media != "application/json" && media != "text/event-stream") { + return observer + } + observer.mode = media + observer.eligible = true + return observer +} + +func (o *claudeTerminal) Read(p []byte) (int, error) { + n, err := o.body.Read(p) + o.read += int64(n) + if n > 0 && o.eligible && !o.invalid { + o.observe(p[:n]) + } + if err == io.EOF { + o.eof = true + } else if err != nil { + o.reject() + } + return n, err +} + +func (o *claudeTerminal) Close() error { return o.body.Close() } + +func (o *claudeTerminal) reject() { + o.invalid = true + o.jsonBody = nil + o.line = nil + o.data = nil + o.blocks = nil +} + +func (o *claudeTerminal) observe(data []byte) { + if o.mode == "application/json" { + if len(o.jsonBody)+len(data) > claudeTerminalJSONMax { + o.reject() + return + } + o.jsonBody = append(o.jsonBody, data...) + return + } + for _, b := range data { + if b == '\n' { + line := bytes.TrimSuffix(o.line, []byte{'\r'}) + o.observeLine(line) + o.line = o.line[:0] + if o.invalid { + return + } + } else { + if len(o.line) >= claudeTerminalLineMax { + o.reject() + return + } + o.line = append(o.line, b) + } + } +} + +func (o *claudeTerminal) observeLine(line []byte) { + if len(line) == 0 { + if len(o.data) > 0 { + o.observeEvent() + } else if o.event != "" && o.event != "ping" { + o.reject() + return + } + o.data = o.data[:0] + o.event = "" + return + } + if line[0] == ':' { + return + } + name, value, found := bytes.Cut(line, []byte{':'}) + if !found { + o.reject() + return + } + value = bytes.TrimPrefix(value, []byte{' '}) + switch string(name) { + case "data": + if len(o.data)+len(value)+1 > claudeTerminalEventMax { + o.reject() + return + } + if len(o.data) > 0 { + o.data = append(o.data, '\n') + } + o.data = append(o.data, value...) + case "event": + o.event = string(value) + case "id", "retry": + // SSE transport metadata is not evidence of completion. + default: + o.reject() + } +} + +func normalClaudeStop(reason string) bool { + switch reason { + case "end_turn", "tool_use", "max_tokens", "stop_sequence": + return true + } + return false +} + +func presentJSON(raw json.RawMessage) bool { + return len(raw) != 0 && !bytes.Equal(bytes.TrimSpace(raw), []byte("null")) +} + +type claudeMessage struct { + ID string `json:"id"` + Type string `json:"type"` + Role string `json:"role"` + Content json.RawMessage `json:"content"` + StopReason string `json:"stop_reason"` + Error json.RawMessage `json:"error"` + Response struct { + Error json.RawMessage `json:"error"` + } `json:"response"` +} + +func normalClaudeBlock(kind string) bool { + switch kind { + case "text", "thinking", "redacted_thinking", "tool_use", "server_tool_use": + return true + } + return false +} + +func assistantMessage(message *claudeMessage) bool { + if message == nil || message.ID == "" || message.Type != "message" || message.Role != "assistant" || presentJSON(message.Error) || presentJSON(message.Response.Error) { + return false + } + var content []json.RawMessage + if err := json.Unmarshal(message.Content, &content); err != nil || content == nil { + return false + } + for _, raw := range content { + var block struct { + Type string `json:"type"` + } + if err := json.Unmarshal(raw, &block); err != nil || !normalClaudeBlock(block.Type) { + return false + } + } + return true +} + +func (o *claudeTerminal) observeEvent() { + var event struct { + Type string `json:"type"` + Message *claudeMessage `json:"message"` + Index *int `json:"index"` + Delta json.RawMessage `json:"delta"` + ContentBlock json.RawMessage `json:"content_block"` + Error json.RawMessage `json:"error"` + Response struct { + Error json.RawMessage `json:"error"` + } `json:"response"` + } + if err := json.Unmarshal(o.data, &event); err != nil || presentJSON(event.Error) || presentJSON(event.Response.Error) || (o.event != "" && o.event != event.Type) { + o.reject() + return + } + switch event.Type { + case "message_start": + if o.started || o.stopped || !assistantMessage(event.Message) { + o.reject() + return + } + o.started = true + case "message_stop": + if !o.started || o.stopped || len(o.blocks) != 0 { + o.reject() + return + } + o.stopped = true + case "content_block_start": + if !o.started || o.stopped || event.Index == nil || *event.Index != o.nextBlock || o.blocks[*event.Index] { + o.reject() + return + } + var block struct { + Type string `json:"type"` + } + if err := json.Unmarshal(event.ContentBlock, &block); err != nil || !normalClaudeBlock(block.Type) { + o.reject() + return + } + o.blocks[*event.Index] = true + o.nextBlock++ + case "content_block_delta", "content_block_stop": + if !o.started || o.stopped || event.Index == nil || !o.blocks[*event.Index] { + o.reject() + return + } + if event.Type == "content_block_stop" { + delete(o.blocks, *event.Index) + } else { + var delta struct { + Type string `json:"type"` + } + if err := json.Unmarshal(event.Delta, &delta); err != nil { + o.reject() + return + } + switch delta.Type { + case "text_delta", "thinking_delta", "input_json_delta", "signature_delta": + default: + o.reject() + return + } + } + case "message_delta": + if !o.started || o.stopped || len(o.blocks) != 0 { + o.reject() + return + } + var delta struct { + StopReason string `json:"stop_reason"` + } + if !bytes.HasPrefix(bytes.TrimSpace(event.Delta), []byte{'{'}) { + o.reject() + return + } + if err := json.Unmarshal(event.Delta, &delta); err != nil || (delta.StopReason != "" && !normalClaudeStop(delta.StopReason)) { + o.reject() + return + } + case "ping", "response_headers": + // Kernel metadata may precede a message or follow message_stop. An + // error/unknown event still invalidates even an earlier message_stop. + default: + o.reject() + } +} + +func (o *claudeTerminal) completed(response *http.Response) bool { + if !o.eligible || o.invalid { + return false + } + if response.ContentLength >= 0 { + if o.read != response.ContentLength { + return false + } + } else if !o.eof { + return false + } + terminal, declared := response.Trailer[http.CanonicalHeaderKey("X-Terminal-State")] + if declared && len(terminal) == 0 { + return false + } + if !declared { + terminal = response.Header.Values("X-Terminal-State") + } + for _, value := range terminal { + if (declared && value == "") || (value != "" && value != "verified") { + return false + } + } + if o.mode == "application/json" { + var message claudeMessage + if err := json.Unmarshal(o.jsonBody, &message); err != nil { + return false + } + return assistantMessage(&message) && normalClaudeStop(message.StopReason) + } + return o.eof && o.started && o.stopped && len(o.blocks) == 0 && len(o.line) == 0 && len(o.data) == 0 && strings.TrimSpace(o.event) == "" +} diff --git a/worker/internal/slotgate/claude_terminal_test.go b/worker/internal/slotgate/claude_terminal_test.go index 0a8bae7cd..89e94c6e5 100644 --- a/worker/internal/slotgate/claude_terminal_test.go +++ b/worker/internal/slotgate/claude_terminal_test.go @@ -1,11 +1,14 @@ package slotgate import ( + "context" "encoding/json" "fmt" "io" "net/http" + "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -142,6 +145,12 @@ func TestKSG2ClaudeUnknownResponsesRemainPending(t *testing.T) { {"error-after-stop", "text/event-stream", normalClaudeSSE + "data: {\"type\":\"error\",\"error\":{\"type\":\"worker_error\",\"message\":\"client disconnected\"}}\n\n", 200, false, "", ""}, {"incomplete-trailer", "text/event-stream", normalClaudeSSE, 200, false, "incomplete", ""}, {"wrong-job-id", "application/json", normalClaudeJSON, 200, false, "", "other-request"}, + {"nested-internal-error", "application/json", `{"id":"msg_x","type":"message","role":"assistant","content":[],"stop_reason":"end_turn","response":{"error":{"type":"worker_error","message":"client disconnected"}}}`, 200, false, "", ""}, + {"empty-error-frame", "text/event-stream", normalClaudeSSE + "event: error\n\n", 200, false, "", ""}, + {"unknown-stop-reason", "application/json", `{"id":"msg_x","type":"message","role":"assistant","content":[],"stop_reason":"cancelled"}`, 200, false, "", ""}, + {"duplicate-terminal", "text/event-stream", normalClaudeSSE + "data: {\"type\":\"message_stop\"}\n\n", 200, false, "", ""}, + {"trailing-json-error", "application/json", normalClaudeJSON + `{"type":"error","error":{"message":"client disconnected"}}`, 200, false, "", ""}, + {"missing-declared-trailer", "text/event-stream", normalClaudeSSE, 200, false, "missing", ""}, } { t.Run(tc.name, func(t *testing.T) { addr, root, server, state, _ := fencedGate(t) @@ -159,7 +168,11 @@ func TestKSG2ClaudeUnknownResponsesRemainPending(t *testing.T) { w.WriteHeader(tc.status) fmt.Fprint(w, tc.body) if tc.trailer != "" { - w.Header().Set("X-Terminal-State", tc.trailer) + if tc.trailer == "missing" { + w.Header().Del("X-Terminal-State") + } else { + w.Header().Set("X-Terminal-State", tc.trailer) + } } })) req := forwardRequest(t, addr, state, tc.name) @@ -187,3 +200,114 @@ func TestKSG2ClaudeUnknownResponsesRemainPending(t *testing.T) { }) } } + +func TestKSG2CommandStatusCannotRelabelOldTicketWithCurrentGeneration(t *testing.T) { + addr, _, server, state, _ := fencedGate(t) + req := forwardRequest(t, addr, state, "captured-old-ticket") + if _, err := server.authority.Revoke("v", state.Fence); err != nil { + t.Fatal(err) + } + next, err := server.authority.Advance(context.Background(), "v", state.Fence) + if err != nil { + t.Fatal(err) + } + req.Op = "command" + if reply := gate2Control(t, addr, req); reply.OK || reply.Error != ErrFenceMismatch.Error() { + t.Fatalf("old fence status: %+v", reply) + } + req.Fence = next.Fence + if reply := gate2Control(t, addr, req); reply.OK || reply.Error != "ticket_invalid" { + t.Fatalf("old ticket relabeled: %+v", reply) + } +} + +func TestKSG2ClaudeCompletionPersistenceFailureCannotAcknowledgeCompleted(t *testing.T) { + addr, root, server, state, _ := fencedGate(t) + journal := filepath.Join(server.cfg.AuthorityDir, "authority.json") + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := os.Rename(journal, journal+".saved"); err != nil { + t.Error(err) + } + if err := os.Mkdir(journal, 0o700); err != nil { + t.Error(err) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, normalClaudeJSON) + })) + req := forwardRequest(t, addr, state, "commit-failure") + c, reader := gate2Connect(t, addr, testToken, req, claudeWire(req.CommandID)) + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + if _, err = io.Copy(io.Discard, response.Body); err != nil { + t.Fatal(err) + } + response.Body.Close() + c.Close() + req.Op = "command" + until := time.Now().Add(time.Second) + for { + c, reader = gate2Connect(t, addr, testToken, req, "") + var reply struct { + OK bool `json:"ok"` + Error string `json:"error"` + Command struct { + Status CommandStatus `json:"status"` + } `json:"command"` + } + err = json.NewDecoder(reader).Decode(&reply) + c.Close() + if err != nil { + t.Fatal(err) + } + if reply.Command.Status == CommandCompleted { + t.Fatal("HTTP success was acknowledged despite failed node commit") + } + if !reply.OK && reply.Error == ErrAuthorityUnavailable.Error() { + break + } + if time.Now().After(until) { + t.Fatalf("completion persistence failure not surfaced: %+v", reply) + } + } +} + +type oneByteReader struct{ io.Reader } + +func (r oneByteReader) Read(p []byte) (int, error) { + if len(p) > 1 { + p = p[:1] + } + return r.Reader.Read(p) +} + +func TestClaudeTerminalObserverIsIncrementalAndBounded(t *testing.T) { + for _, tc := range []struct { + name, media, body string + complete bool + }{ + {"fragmented-json", "application/json", normalClaudeJSON, true}, + {"fragmented-sse", "text/event-stream", normalClaudeSSE, true}, + {"oversized-json", "application/json", strings.Repeat("x", claudeTerminalJSONMax+1), false}, + {"oversized-sse-line", "text/event-stream", "data: " + strings.Repeat("x", claudeTerminalLineMax+1) + "\n\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + req, _ := http.NewRequest("POST", "http://kernel/internal/v1/messages", nil) + response := &http.Response{StatusCode: 200, Header: http.Header{"Content-Type": []string{tc.media}}, ContentLength: int64(len(tc.body)), Body: io.NopCloser(oneByteReader{strings.NewReader(tc.body)})} + observer := observeClaudeTerminal(response, req, []byte(`{"request_id":"command"}`), "command", "kernel") + if _, err := io.Copy(io.Discard, observer); err != nil { + t.Fatal(err) + } + if got := observer.completed(response); got != tc.complete { + t.Fatalf("complete=%v, want %v", got, tc.complete) + } + if len(observer.jsonBody) > claudeTerminalJSONMax || len(observer.line) > claudeTerminalLineMax || len(observer.data) > claudeTerminalEventMax { + t.Fatal("parser exceeded its memory bounds") + } + }) + } +} diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index 67c5f6171..7f03a8654 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -22,6 +22,8 @@ import ( const gate2LineMax = 8192 const gate2RequestMax = 16 << 20 +var errGateTicketInvalid = errors.New("ticket_invalid") + // KSG2 \n followed by one bounded JSON command line. Only forward // commands continue with one parsed HTTP/1 request and an HTTP response after // OK\n. KSG1 raw splicing is never accepted when fenced mode is enabled. @@ -68,7 +70,7 @@ func gate2Line(reader *bufio.Reader) ([]byte, error) { } func gate2Code(err error) string { - for _, known := range []error{ErrAuthorityUnavailable, ErrAuthorityPhase, ErrFenceMismatch, ErrCommandExists, ErrCommandOwner, ErrCommandDispatched, ErrPendingMutation, ErrInvalidAuthority, ErrTerminationProof} { + for _, known := range []error{ErrAuthorityUnavailable, ErrAuthorityPhase, ErrFenceMismatch, ErrCommandExists, ErrCommandOwner, ErrCommandDispatched, ErrPendingMutation, ErrInvalidAuthority, ErrTerminationProof, errGateTicketInvalid} { if errors.Is(err, known) { return known.Error() } @@ -139,6 +141,26 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. break } state, err = s.launcher.Launch(ctx, request.VMID, request.Fence, request.CommandID) + case "command": + var handle CommandHandle + handle, err = s.forwardHandle(request) + if err != nil { + break + } + state, err = s.authority.State(handle.VMID) + if err == nil { + command, exists := state.Commands[handle.ID] + if !exists || state.Fence != request.Fence || command.Fence != request.Fence { + err = ErrFenceMismatch + break + } + reply["command"] = struct { + ID string `json:"id"` + Fence Fence `json:"fence"` + Status CommandStatus `json:"status"` + Dispatched bool `json:"dispatched"` + }{command.ID, command.Fence, command.Status, command.Dispatched} + } default: err = ErrInvalidAuthority } @@ -146,7 +168,7 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["error"] = gate2Code(err) } else { reply["ok"] = true - if request.Op != "prepare" { + if request.Op != "prepare" && request.Op != "command" { reply["state"] = publicGateState(state) } } @@ -168,7 +190,7 @@ func gate2Ticket(nonce string, request gate2Request) string { func (s *Server) forwardHandle(request gate2Request) (CommandHandle, error) { if _, ok := kinds[request.Kind]; !ok { - return CommandHandle{}, errors.New("ticket_invalid") + return CommandHandle{}, errGateTicketInvalid } state, err := s.authority.State(request.VMID) if err != nil { @@ -179,11 +201,11 @@ func (s *Server) forwardHandle(request gate2Request) (CommandHandle, error) { } command, exists := state.Commands[request.CommandID] if !exists || command.Kind != CommandForward || command.Fence != request.Fence { - return CommandHandle{}, errors.New("ticket_invalid") + return CommandHandle{}, errGateTicketInvalid } wanted := gate2Ticket(command.Token, request) if !hmac.Equal([]byte(wanted), []byte(request.Ticket)) { - return CommandHandle{}, errors.New("ticket_invalid") + return CommandHandle{}, errGateTicketInvalid } return CommandHandle{VMID: request.VMID, ID: request.CommandID, Fence: request.Fence, runtime: command.Runtime, token: command.Token}, nil } @@ -265,7 +287,18 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio return } response.Close = true - response.Write(conn) - // Even a full HTTP response remains pending in this batch. Classifying a - // kernel's normal terminal response is a separate trusted protocol adapter. + terminal := observeClaudeTerminal(response, req, body, request.CommandID, request.Kind) + response.Body = terminal + if err = response.Write(conn); err != nil { + return + } + if terminal.completed(response) { + // The HTTP body is not a durable acknowledgement. A KSG2 client must + // read the authenticated command status before releasing its PG job. + // If this commit fails the Authority poisons itself and cannot report + // completed; no new execution or retry is issued here. + if err = s.authority.CompleteCommand(handle); err != nil { + return + } + } } From f020616cc34c266d38ca3a56b9f9fce929b17375 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:58:52 +0800 Subject: [PATCH 270/473] test: reproduce shutdown losing global background callbacks --- test/e2e/background-shutdown.e2e.test.mjs | 78 +++++++++++++++++++++++ test/fixtures/background-shutdown.mjs | 37 +++++++++++ 2 files changed, 115 insertions(+) create mode 100644 test/e2e/background-shutdown.e2e.test.mjs create mode 100644 test/fixtures/background-shutdown.mjs diff --git a/test/e2e/background-shutdown.e2e.test.mjs b/test/e2e/background-shutdown.e2e.test.mjs new file mode 100644 index 000000000..beb884555 --- /dev/null +++ b/test/e2e/background-shutdown.e2e.test.mjs @@ -0,0 +1,78 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import pg from 'pg' +import { startGateway } from '../harness.mjs' +import { coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' +import { quoteIdentifier } from '../../src/lib/db/database.mjs' + +async function fixture(t, graceMs = 3000) { + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-background-shutdown-')) + fs.mkdirSync(path.join(project, 'vms')) + fs.writeFileSync(path.join(project, 'vms', 'active.json'), '{}') + const preload = fileURLToPath(new URL('../fixtures/background-shutdown.mjs', import.meta.url)) + const gw = await startGateway({ + project, + env: { + VM2API_CONTROL_PLANE_MODE: 'kubernetes', + KIN_BACKUP_DISABLED: '1', + VM2API_SHUTDOWN_TIMEOUT_MS: String(graceMs), + VM2API_SYNTHETIC_BACKGROUND_DIRECTORY: project, + NODE_OPTIONS: `--import=${preload}`, + }, + }) + t.after(async () => { + fs.writeFileSync(path.join(project, 'release'), '1') + await gw.stop() + const client = new pg.Client({ connectionString: gw.connectionString }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinationSchemaFor(gw.schema))} CASCADE`) + } finally { + await client.end() + fs.rmSync(project, { recursive: true, force: true }) + } + }) + const deadline = Date.now() + 3000 + while (!fs.existsSync(path.join(project, 'started')) && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 10)) + assert.ok(fs.existsSync(path.join(project, 'started')), gw.stderr()) + return gw +} + +const exitOf = (child) => + child.exitCode !== null ? Promise.resolve(child.exitCode) : new Promise((resolve) => child.once('exit', resolve)) + +test('shutdown awaits a global shared background callback and its committed terminal state', async (t) => { + const gw = await fixture(t) + const exit = exitOf(gw.child) + gw.child.kill('SIGTERM') + await new Promise((resolve) => setTimeout(resolve, 100)) + assert.equal(gw.child.exitCode, null, gw.stderr()) + fs.writeFileSync(path.join(gw.project, 'release'), '1') + assert.equal(await exit, 0, gw.stderr()) + assert.ok( + (await gw.db.query("SELECT finished_at FROM background_task_runs WHERE task_key='synthetic-shutdown-background'")) + .rows[0].finished_at, + ) + assert.equal( + (await gw.db.query("SELECT value FROM settings WHERE key='synthetic-background-ended'")).rows[0].value, + 'true', + ) +}) + +test('an unfinished global background callback remains claimed when the shutdown deadline expires', async (t) => { + const gw = await fixture(t, 100) + const exit = exitOf(gw.child) + const started = Date.now() + gw.child.kill('SIGTERM') + assert.equal(await exit, 1, gw.stderr()) + assert.ok(Date.now() - started < 6500) + const row = ( + await gw.db.query("SELECT finished_at FROM background_task_runs WHERE task_key='synthetic-shutdown-background'") + ).rows[0] + assert.equal(row.finished_at, null) +}) diff --git a/test/fixtures/background-shutdown.mjs b/test/fixtures/background-shutdown.mjs new file mode 100644 index 000000000..d9582f277 --- /dev/null +++ b/test/fixtures/background-shutdown.mjs @@ -0,0 +1,37 @@ +import fs from 'node:fs' +import path from 'node:path' +import { getDb, getActivityCoordinator } from '../../src/lib/db/database.mjs' +import { getSharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' + +const directory = process.env.VM2API_SYNTHETIC_BACKGROUND_DIRECTORY +if (directory) { + let checking = false + const timer = setInterval(async () => { + if (checking) return + checking = true + let db + try { + db = getDb() + if (!(await getActivityCoordinator(db)?.isReady())) { + checking = false + return + } + } catch { + checking = false + return + } + clearInterval(timer) + getSharedBackgroundTasks(db).run({ taskKey: 'synthetic-shutdown-background', executionId: 'one' }, async () => { + fs.writeFileSync(path.join(directory, 'started'), '1') + await new Promise((resolve) => { + const release = setInterval(() => { + if (!fs.existsSync(path.join(directory, 'release'))) return + clearInterval(release) + resolve() + }, 10) + }) + await db.query("INSERT INTO settings(key,value) VALUES ('synthetic-background-ended','true')") + return { terminal: true } + }).catch((error) => fs.writeFileSync(path.join(directory, 'error'), error.code || error.name)) + }, 10) +} From 0efdbb3d078762a48a7678af21b3f91a3a6fc01c Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 22:58:59 +0800 Subject: [PATCH 271/473] fix: guard VM metadata and identity mutations with persistent claims --- src/lib/admin/panel-routes.mjs | 429 ++++++++++++---------- src/lib/identity/official-fingerprint.mjs | 4 +- src/lib/vm/fleet-update.mjs | 9 + test/unit/egress-stop-proof.test.mjs | 31 +- 4 files changed, 261 insertions(+), 212 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index cf1199ade..39a97e184 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -2383,7 +2383,8 @@ export function createPanelHandler(ctx) { } // POST /api/panel/vms/fleet-update if (req.method === 'POST' && p === '/api/panel/vms/reconcile-fingerprints') { - return json(res, 200, panel.ok(reconcileOfficialFingerprints(cfg.paths.project))) + const ids = listVms(cfg.paths.project).map((vm) => vm.id) + return mutateVms(ids, () => json(res, 200, panel.ok(reconcileOfficialFingerprints(cfg.paths.project, { ids })))) } if (req.method === 'POST' && p === '/api/panel/vms/fleet-update') { const body = await readBody(req, 32 * 1024).catch(() => ({})) @@ -2748,40 +2749,46 @@ export function createPanelHandler(ctx) { // POST /api/panel/vms/:id/collect-identity if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/collect-identity$/.test(p)) { const id = p.split('/')[4] - const vm = getVm(cfg.paths.project, id) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const collected = await collectSlotIdentity(cfg.paths.project, vm) - if (!collected.ok) { - return json(res, 502, { - ok: false, - error: { message: collected.error || 'collect failed', code: collected.code }, - }) - } - return json(res, 200, panel.ok(collected)) + return mutateVms([id], async (handles) => { + const vm = getVm(cfg.paths.project, id) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const collected = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.collectSlotIdentity || collectSlotIdentity)(cfg.paths.project, vm), + ) + if (!collected.ok) { + return json(res, 502, { + ok: false, + error: { message: collected.error || 'collect failed', code: collected.code }, + }) + } + return json(res, 200, panel.ok(collected)) + }) } if (req.method === 'PATCH' && /^\/api\/panel\/vms\/[^/]+\/owner$/.test(p)) { if (panelIdentity(req).role !== 'admin') { return json(res, 403, { ok: false, error: { code: 'forbidden', message: '当前权限无法执行此操作' } }) } const id = p.split('/')[4] - const vm = getVm(cfg.paths.project, id) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const body = await readBody(req, 8192).catch(() => ({})) - const nextOwner = - body.user_id === null || body.user_id === '' ? null : normalizeOwnerId(body.user_id || body.owner_user_id) - if (nextOwner && !(await panelUsers.getById(nextOwner))) { - return json(res, 404, { ok: false, error: { message: 'user not found' } }) - } - const currentOrigin = vm.origin || VM_ORIGIN.platform - if (!nextOwner && currentOrigin === VM_ORIGIN.userCreated) { - return json(res, 400, { - ok: false, - error: { code: 'cannot_unassign_user_created', message: '自建 VM 不能收回进平台池，请删除' }, - }) - } - const origin = assignOriginForOwner(nextOwner, { previousOrigin: currentOrigin, actorRole: 'admin' }) - const saved = persistVmOwner(cfg.paths.project, id, { ownerUserId: nextOwner, origin }) - return json(res, 200, panel.ok({ vm: summarizeVm(saved, cfg.paths.project, ctx.routingConfig) })) + return mutateVms([id], async (handles) => { + const vm = getVm(cfg.paths.project, id) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const body = await readBody(req, 8192).catch(() => ({})) + const nextOwner = + body.user_id === null || body.user_id === '' ? null : normalizeOwnerId(body.user_id || body.owner_user_id) + if (nextOwner && !(await panelUsers.getById(nextOwner))) { + return json(res, 404, { ok: false, error: { message: 'user not found' } }) + } + const currentOrigin = vm.origin || VM_ORIGIN.platform + if (!nextOwner && currentOrigin === VM_ORIGIN.userCreated) { + return json(res, 400, { + ok: false, + error: { code: 'cannot_unassign_user_created', message: '自建 VM 不能收回进平台池，请删除' }, + }) + } + const origin = assignOriginForOwner(nextOwner, { previousOrigin: currentOrigin, actorRole: 'admin' }) + const saved = persistVmOwner(cfg.paths.project, id, { ownerUserId: nextOwner, origin }) + return json(res, 200, panel.ok({ vm: summarizeVm(saved, cfg.paths.project, ctx.routingConfig) })) + }) } // GET /api/panel/vms/:id @@ -3735,45 +3742,49 @@ export function createPanelHandler(ctx) { // POST /api/panel/vms/:id/reset-fingerprint if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/reset-fingerprint$/.test(p)) { const id = p.split('/')[4] - const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) - if (!fs.existsSync(vmPath)) { - return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - let vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) - const generated = assignComputerFingerprint(cfg.paths.project, vm, { - taken: takenFingerprintKeys(listVms(cfg.paths.project), { exceptId: id }), + return mutateVms([id], async (handles) => { + const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) + if (!fs.existsSync(vmPath)) { + return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + let vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) + const generated = assignComputerFingerprint(cfg.paths.project, vm, { + taken: takenFingerprintKeys(listVms(cfg.paths.project), { exceptId: id }), + }) + vm.timezone = generated.timezone + vm.locale = generated.locale + vm.fingerprint = applyGeneratedFingerprint({}, generated) + vm.updated_at = generated.reset_at + atomicWriteJson(vmPath, vm, { mode: 0o600 }) + writeGuestMachineIdFile(cfg.paths.project, id, generated.guest_machine_id) + applyOfficialFingerprintToVm(vmPath, path.join(cfg.paths.project, 'vms', id, 'cli-home')) + try { + vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) + } catch {} + return json(res, 200, panel.ok({ id, fingerprint: vm.fingerprint })) }) - vm.timezone = generated.timezone - vm.locale = generated.locale - vm.fingerprint = applyGeneratedFingerprint({}, generated) - vm.updated_at = generated.reset_at - atomicWriteJson(vmPath, vm, { mode: 0o600 }) - writeGuestMachineIdFile(cfg.paths.project, id, generated.guest_machine_id) - applyOfficialFingerprintToVm(vmPath, path.join(cfg.paths.project, 'vms', id, 'cli-home')) - try { - vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) - } catch {} - return json(res, 200, panel.ok({ id, fingerprint: vm.fingerprint })) } if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/reconcile-fingerprint$/.test(p)) { const id = p.split('/')[4] - const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) - if (!fs.existsSync(vmPath)) { - return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - } - const result = applyOfficialFingerprintToVm(vmPath, officialCcHome(cfg.paths.project, id)) - const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) - return json( - res, - 200, - panel.ok({ - vm_id: id, - ...result, - identity_source: vm.fingerprint?.identity_source || null, - has_official_user: !!vm.fingerprint?.official_user_id, - has_official_machine: !!vm.fingerprint?.official_machine_id, - }), - ) + return mutateVms([id], async (handles) => { + const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) + if (!fs.existsSync(vmPath)) { + return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + } + const result = applyOfficialFingerprintToVm(vmPath, officialCcHome(cfg.paths.project, id)) + const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) + return json( + res, + 200, + panel.ok({ + vm_id: id, + ...result, + identity_source: vm.fingerprint?.identity_source || null, + has_official_user: !!vm.fingerprint?.official_user_id, + has_official_machine: !!vm.fingerprint?.official_machine_id, + }), + ) + }) } // GET /api/panel/vms/:id/seed-settings @@ -3817,104 +3828,107 @@ export function createPanelHandler(ctx) { // PUT /api/panel/vms/:id/seed-settings if (req.method === 'PUT' && /^\/api\/panel\/vms\/[^/]+\/seed-settings$/.test(p)) { const id = p.split('/')[4] - const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) - if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const body = await readBody(req, 256 * 1024) - const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) - const telemetryWasOff = !isTelemetryEnabled(vm.seed_policy || {}) - const merged = { ...(vm.seed_policy || {}), ...(body.seed_policy || {}) } - for (const k of ['telemetry_disabled', 'reject_client_settings', 'reject_client_metadata_identity']) { - if (body[k] !== undefined) merged[k] = !!body[k] - } - if (body.theme !== undefined) merged.theme = body.theme - if (body.extra_env !== undefined) merged.extra_env = body.extra_env - if (body.settings_json_override !== undefined) merged.settings_json_override = body.settings_json_override - if (body.claude_setting !== undefined) merged.claude_setting = body.claude_setting - if (body['claude.setting'] !== undefined) merged.claude_setting = body['claude.setting'] - const next = defaultSeedPolicy(merged) - if (body.settings_json_override !== undefined) next.settings_json_override = body.settings_json_override - if (body.extra_env !== undefined) next.extra_env = body.extra_env || {} - const followProxyTz = body?.timezone_follow_proxy === true || body?.timezoneFollowProxy === true - const hasTimezone = body && Object.prototype.hasOwnProperty.call(body, 'timezone') - const hasLocale = body && Object.prototype.hasOwnProperty.call(body, 'locale') - let nextLocale = null - if (hasLocale) { - const requestedLocale = body.locale == null || body.locale === '' ? STANDARD_LOCALE : String(body.locale) - if (!SLOT_LOCALES.includes(requestedLocale)) { - return json(res, 400, { - ok: false, - error: { code: 'invalid_locale', message: `语言必须是 ${SLOT_LOCALES.join('、')}` }, - }) + return mutateProxyVms([id], async (handles) => { + const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) + if (!fs.existsSync(vmPath)) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + const body = await readBody(req, 256 * 1024) + const vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) + const telemetryWasOff = !isTelemetryEnabled(vm.seed_policy || {}) + const merged = { ...(vm.seed_policy || {}), ...(body.seed_policy || {}) } + for (const k of ['telemetry_disabled', 'reject_client_settings', 'reject_client_metadata_identity']) { + if (body[k] !== undefined) merged[k] = !!body[k] + } + if (body.theme !== undefined) merged.theme = body.theme + if (body.extra_env !== undefined) merged.extra_env = body.extra_env + if (body.settings_json_override !== undefined) merged.settings_json_override = body.settings_json_override + if (body.claude_setting !== undefined) merged.claude_setting = body.claude_setting + if (body['claude.setting'] !== undefined) merged.claude_setting = body['claude.setting'] + const next = defaultSeedPolicy(merged) + if (body.settings_json_override !== undefined) next.settings_json_override = body.settings_json_override + if (body.extra_env !== undefined) next.extra_env = body.extra_env || {} + const followProxyTz = body?.timezone_follow_proxy === true || body?.timezoneFollowProxy === true + const hasTimezone = body && Object.prototype.hasOwnProperty.call(body, 'timezone') + const hasLocale = body && Object.prototype.hasOwnProperty.call(body, 'locale') + let nextLocale = null + if (hasLocale) { + const requestedLocale = body.locale == null || body.locale === '' ? STANDARD_LOCALE : String(body.locale) + if (!SLOT_LOCALES.includes(requestedLocale)) { + return json(res, 400, { + ok: false, + error: { code: 'invalid_locale', message: `语言必须是 ${SLOT_LOCALES.join('、')}` }, + }) + } + nextLocale = requestedLocale } - nextLocale = requestedLocale - } - if (hasTimezone) { - const zone = validTimezone(body.timezone) - if (!zone) { - return json(res, 400, { - ok: false, - error: { - type: 'invalid_request_error', - code: 'invalid_timezone', - message: '时区必须是有效的 IANA 名称，如 Asia/Tokyo', - param: 'timezone', - }, - }) + if (hasTimezone) { + const zone = validTimezone(body.timezone) + if (!zone) { + return json(res, 400, { + ok: false, + error: { + type: 'invalid_request_error', + code: 'invalid_timezone', + message: '时区必须是有效的 IANA 名称，如 Asia/Tokyo', + param: 'timezone', + }, + }) + } + const pinned = persistVmTimezone(cfg.paths.project, id, zone, { source: 'manual' }) + if (!pinned) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + Object.assign(vm, pinned) + } else if (followProxyTz) { + const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, id, { force: true }) + if (!synced.ok) { + return json(res, 400, { + ok: false, + error: { + type: 'invalid_request_error', + code: synced.reason || 'proxy_timezone_unknown', + message: '未能从代理节点取到时区，请先检测代理地理位置', + }, + }) + } + if (synced.applied) { + const fresh = getVm(cfg.paths.project, id) + if (fresh) Object.assign(vm, fresh) + } else if (synced.timezone) { + vm.timezone = synced.timezone + vm.timezone_source = 'proxy_geo' + } } - const pinned = persistVmTimezone(cfg.paths.project, id, zone, { source: 'manual' }) - if (!pinned) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - Object.assign(vm, pinned) - } else if (followProxyTz) { - const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, id, { force: true }) - if (!synced.ok) { - return json(res, 400, { - ok: false, - error: { - type: 'invalid_request_error', - code: synced.reason || 'proxy_timezone_unknown', - message: '未能从代理节点取到时区，请先检测代理地理位置', - }, + if (nextLocale) vm.locale = nextLocale + vm.seed_policy = next + vm.updated_at = new Date().toISOString() + atomicWriteJson(vmPath, vm, { mode: 0o600 }) + const written = writeSlotSeedFiles(cfg.paths.project, vm, next) + let service = null + if ( + vm.status === 'running' && + (next.telemetry_disabled !== telemetryWasOff || body.telemetry_service_restart === true) + ) { + const report = await (ctx.vmMutationOps?.runFleetUpdate || runFleetUpdate)(cfg.paths.project, { + action: 'telemetry', + ids: [id], + concurrency: 1, + routing: ctx.routingConfig, }) + service = report.items?.[0] || report + await vmEffect(handles.get(id), async () => service) } - if (synced.applied) { - const fresh = getVm(cfg.paths.project, id) - if (fresh) Object.assign(vm, fresh) - } else if (synced.timezone) { - vm.timezone = synced.timezone - vm.timezone_source = 'proxy_geo' - } - } - if (nextLocale) vm.locale = nextLocale - vm.seed_policy = next - vm.updated_at = new Date().toISOString() - atomicWriteJson(vmPath, vm, { mode: 0o600 }) - const written = writeSlotSeedFiles(cfg.paths.project, vm, next) - let service = null - if ( - vm.status === 'running' && - (next.telemetry_disabled !== telemetryWasOff || body.telemetry_service_restart === true) - ) { - const report = await runFleetUpdate(cfg.paths.project, { - action: 'telemetry', - ids: [id], - concurrency: 1, - routing: ctx.routingConfig, - }) - service = report.items?.[0] || report - } - return json( - res, - 200, - panel.ok({ - vm_id: id, - seed_policy: next, - ...seedTelemetryContract(next), - settings_json: written.settings || null, - claude_setting: written.claude_setting || '', - kin_seed: written.kin_seed || null, - service, - }), - ) + return json( + res, + 200, + panel.ok({ + vm_id: id, + seed_policy: next, + ...seedTelemetryContract(next), + settings_json: written.settings || null, + claude_setting: written.claude_setting || '', + kin_seed: written.kin_seed || null, + service, + }), + ) + }) } if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/wrap-cli\/promote$/.test(p)) { @@ -3928,56 +3942,63 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/wrap-cli\/repair$/.test(p)) { const id = p.split('/')[4] - const body = await readBody(req, 8 * 1024).catch(() => ({})) - const mode = body.mode === 'traits' ? 'traits' : 'keep' - const vm = getVm(cfg.paths.project, id) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - if (isCodexVm(vm)) { - return json(res, 409, { - ok: false, - error: { code: 'codex_kernel_only', message: 'Codex 槽只使用控制面 Codex kernel' }, - }) - } - if (slotHost(vm).bakedKernel) { - const refreshed = await refreshCluster(cfg.paths.project, [vm], { routing: ctx.routingConfig }) - const item = refreshed.items[0] || { - ok: false, - code: 'slot_payload_refresh_failed', - error: '集群槽换镜像失败', - } - if (!item.ok) return json(res, 400, { ok: false, error: { code: item.code, message: item.error } }) - return json( - res, - 200, - panel.ok({ mode, wrap: { ok: true, baked: true, image: item.image }, kernel: item.kernel }), - ) - } - if (mode === 'traits') { - const gone = await destroySlot(vm) - if (!gone.ok) - return json(res, 500, { ok: false, error: { code: gone.code, message: gone.error || '重建新特征失败' } }) - const rebuilt = recreateVmFiles(cfg.paths.project, vm, { mode: 'traits' }) - const boot = await startSlotReady(rebuilt.vm, cfg.paths.project, { - recreate: true, - routing: ctx.routingConfig, - }) - if (!boot.ok) - return json(res, 500, { + return mutateVms([id], async (handles) => { + const body = await readBody(req, 8 * 1024).catch(() => ({})) + const mode = body.mode === 'traits' ? 'traits' : 'keep' + const vm = getVm(cfg.paths.project, id) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + if (isCodexVm(vm)) { + return json(res, 409, { ok: false, - error: { code: boot.code, message: boot.error || '重建新特征后启动失败' }, + error: { code: 'codex_kernel_only', message: 'Codex 槽只使用控制面 Codex kernel' }, }) - return json(res, 200, panel.ok({ mode, rebuild: 'traits', boot: panel.publicSlotBoot(boot) })) - } - const wrap = materializeWrapCli(cfg.paths.project, vm) - if (!wrap.ok) return json(res, 400, { ok: false, error: { code: wrap.code, message: wrap.error } }) - writeKernelConfig(cfg.paths.project, vm, { routing: ctx.routingConfig }) + } + if (slotHost(vm).bakedKernel) { + const refreshed = await refreshCluster(cfg.paths.project, [vm], { routing: ctx.routingConfig }) + const item = refreshed.items[0] || { + ok: false, + code: 'slot_payload_refresh_failed', + error: '集群槽换镜像失败', + } + await vmEffect(handles.get(id), async () => item) + if (!item.ok) return json(res, 400, { ok: false, error: { code: item.code, message: item.error } }) + return json( + res, + 200, + panel.ok({ mode, wrap: { ok: true, baked: true, image: item.image }, kernel: item.kernel }), + ) + } + if (mode === 'traits') { + const gone = await vmEffect(handles.get(id), () => (ctx.vmMutationOps?.destroySlot || destroySlot)(vm)) + if (!gone.ok) + return json(res, 500, { ok: false, error: { code: gone.code, message: gone.error || '重建新特征失败' } }) + const rebuilt = recreateVmFiles(cfg.paths.project, vm, { mode: 'traits' }) + const boot = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.startSlotReady || startSlotReady)(rebuilt.vm, cfg.paths.project, { + recreate: true, + routing: ctx.routingConfig, + }), + ) + if (!boot.ok) + return json(res, 500, { + ok: false, + error: { code: boot.code, message: boot.error || '重建新特征后启动失败' }, + }) + return json(res, 200, panel.ok({ mode, rebuild: 'traits', boot: panel.publicSlotBoot(boot) })) + } + const wrap = materializeWrapCli(cfg.paths.project, vm) + if (!wrap.ok) return json(res, 400, { ok: false, error: { code: wrap.code, message: wrap.error } }) + writeKernelConfig(cfg.paths.project, vm, { routing: ctx.routingConfig }) - let kernel = { ok: false, skipped: true, reason: 'engine_not_rust' } - if (resolveInferenceEngine(vm, ctx.routingConfig) === 'rust') { - kernel = await restartRustKernel(slotExec(cfg.paths.project, vm)) - } + let kernel = { ok: false, skipped: true, reason: 'engine_not_rust' } + if (resolveInferenceEngine(vm, ctx.routingConfig) === 'rust') { + kernel = await vmEffect(handles.get(id), () => + (ctx.vmMutationOps?.restartRustKernel || restartRustKernel)(slotExec(cfg.paths.project, vm)), + ) + } - return json(res, 200, panel.ok({ mode, wrap, kernel })) + return json(res, 200, panel.ok({ mode, wrap, kernel })) + }) } if (req.method === 'GET' && /^\/api\/panel\/vms\/[^/]+\/official-cc-bootstrap$/.test(p)) { diff --git a/src/lib/identity/official-fingerprint.mjs b/src/lib/identity/official-fingerprint.mjs index 9fdc1f5c6..61d746984 100644 --- a/src/lib/identity/official-fingerprint.mjs +++ b/src/lib/identity/official-fingerprint.mjs @@ -133,12 +133,14 @@ export function applyOfficialFingerprintToVm(vmPath, homeDir) { } } -export function reconcileOfficialFingerprints(projectRoot) { +export function reconcileOfficialFingerprints(projectRoot, { ids } = {}) { const vmsDir = path.join(projectRoot, 'vms') const summary = { scanned: 0, reconciled: 0, leftover_removed: 0, replaced_device: 0, skipped: 0 } if (!fs.existsSync(vmsDir)) return summary + const wanted = ids ? new Set(ids) : null for (const name of listVmRecordFiles(vmsDir)) { const vmId = name.slice(0, -5) + if (wanted && !wanted.has(vmId)) continue const vmPath = path.join(vmsDir, name) const homeDir = path.join(vmsDir, vmId, 'cli-home') summary.scanned += 1 diff --git a/src/lib/vm/fleet-update.mjs b/src/lib/vm/fleet-update.mjs index fb303f288..bd65f0d66 100644 --- a/src/lib/vm/fleet-update.mjs +++ b/src/lib/vm/fleet-update.mjs @@ -11,6 +11,7 @@ import { rustKernelHealth } from '../transport/rust-kernel-client.mjs' import { restartTelemetrySidecar } from './slot-process-status.mjs' import { isNativeNodeVm, vmNodeId } from '../cluster/placement.mjs' import { slotHost } from './slot-host.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { restartRemoteServices } from '../cluster/remote-slot.mjs' import { drainEgressForProxyConfig, @@ -21,6 +22,7 @@ import { runExclusiveProxyEgress, startProxyEgress, stopEgressProcess, + stopEgressProcessConfirmed, waitEgressExit, waitPortReleased, } from './egress.mjs' @@ -31,6 +33,13 @@ export async function restartLocalEgress(projectRoot, vm, opts = {}) { if (isLocalEgressProxy(vm.proxy)) return { ok: true, egress: 'local' } const proxyId = vm.proxy?.id return runExclusiveProxyEgress(projectRoot, proxyId, async () => { + if (isKubernetesMode()) { + const stopped = await stopEgressProcessConfirmed(projectRoot, proxyId) + if (!stopped.ok) return { ...stopped, action: 'egress_stop_failed' } + const started = await startProxyEgress(projectRoot, vm.proxy, opts) + if (!started?.ok) return { ...started, ok: false, action: 'egress_start_failed', vmOperationUncertain: true } + return { ok: true, egress: 'restarted', pid: started.pid } + } const running = inspectEgressProcess(projectRoot, proxyId) const stopped = stopEgressProcess(projectRoot, proxyId) if (!stopped.ok) return { ok: false, action: 'egress_stop_failed', error: stopped.error } diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs index 7a60cff02..3a1a1f9b4 100644 --- a/test/unit/egress-stop-proof.test.mjs +++ b/test/unit/egress-stop-proof.test.mjs @@ -127,7 +127,7 @@ test('confirmed egress stop waits for its recorded execution and clears its rece const f = fixture(t) assert.equal((await f.start()).ok, true) const result = await egress.stopEgressProcessConfirmed(f.root, 'synthetic', { termTimeoutMs: 500, killGraceMs: 500 }) - assert.equal(result.ok, true) + assert.equal(result.ok, true, JSON.stringify(result)) assert.equal(result.ended, true) assert.equal(fs.existsSync(path.join(f.dir, 'egress.owner.json')), false) assert.equal(fs.existsSync(path.join(f.dir, 'egress.pid')), false) @@ -272,18 +272,35 @@ test('a Kubernetes Codex exit-policy stop cannot infer another Pod kernel is abs test('Kubernetes fleet telemetry cannot bypass the persistent helper execution receipt', async () => { const source = fs.readFileSync(new URL('../../src/lib/vm/fleet-update.mjs', import.meta.url), 'utf8') - const body = source.slice(source.indexOf('export async function restartLocalEgress('), source.indexOf('async function restartLocalServices(')).replace('export ', '') + const body = source + .slice( + source.indexOf('export async function restartLocalEgress('), + source.indexOf('async function restartLocalServices('), + ) + .replace('export ', '') let legacySignals = 0 let starts = 0 const restart = vmContext.runInNewContext(`${body}; restartLocalEgress`, { - isLocalEgressProxy: () => false, isKubernetesMode: () => true, + isLocalEgressProxy: () => false, + isKubernetesMode: () => true, runExclusiveProxyEgress: (_root, _id, callback) => callback(), - stopEgressProcessConfirmed: async () => ({ ok: false, code: 'egress_owner_unreachable', vmOperationUncertain: true }), + stopEgressProcessConfirmed: async () => ({ + ok: false, + code: 'egress_owner_unreachable', + vmOperationUncertain: true, + }), inspectEgressProcess: () => ({ ok: false }), - stopEgressProcess: () => { legacySignals++; return { ok: true } }, - egressPidsForProxyConfig: () => [], drainEgressForProxyConfig: async () => ({ ok: true }), + stopEgressProcess: () => { + legacySignals++ + return { ok: true } + }, + egressPidsForProxyConfig: () => [], + drainEgressForProxyConfig: async () => ({ ok: true }), readEgressListenTcp: () => '', - startProxyEgress: async () => { starts++; return { ok: true } }, + startProxyEgress: async () => { + starts++ + return { ok: true } + }, }) const result = await restart('/synthetic', { proxy: { id: 'synthetic' } }) assert.equal(result.ok, false) From 8270189c1b80b7a5ec2bbfd1cedc787b7344ace9 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:00:07 +0800 Subject: [PATCH 272/473] test: require fleet ownership and preserve partial reload failures --- test/unit/fleet-update.test.mjs | 14 ++++++++++++++ test/unit/panel-vm-shared-mutations.test.mjs | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/test/unit/fleet-update.test.mjs b/test/unit/fleet-update.test.mjs index 14aafff1d..47d525fbe 100644 --- a/test/unit/fleet-update.test.mjs +++ b/test/unit/fleet-update.test.mjs @@ -83,6 +83,20 @@ test('degraded worker without OAuth still counts as up', () => { assert.equal(isWorkerProcessUp({ code: 'worker_unavailable' }), false) }) +test('fleet does not treat a partial Rust reload as completed successfully', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-partial-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + writeSlot(root, 'vm-synthetic') + const report = await runFleetUpdate(root, { + action: 'roll', + reloadFn: async () => ({ ok: true, rust_ok: false, vmOperationUncertain: true }), + collectFn: async () => ({ ok: true }), + healthFn: async () => ({ ok: true }), + }) + assert.equal(report.failed_count, 1) + assert.equal(report.items[0].vmOperationUncertain, true) +}) + test('fleet telemetry uses injected telemetryFn with options object', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-tel-')) writeSlot(root, 'vm-01', { node_id: 'node-a' }) diff --git a/test/unit/panel-vm-shared-mutations.test.mjs b/test/unit/panel-vm-shared-mutations.test.mjs index 5a2f2d39b..eb13fdac7 100644 --- a/test/unit/panel-vm-shared-mutations.test.mjs +++ b/test/unit/panel-vm-shared-mutations.test.mjs @@ -126,6 +126,19 @@ test('batch fingerprint reconciliation acquires every target before changing any await next.release() }) +test('fleet update claims every selected target before starting the first command', async (t) => { + const f = await fixture(t) + f.seed('vm-a') + f.seed('vm-b') + await f.otherOperations.tryAcquire({ vmId: 'vm-b' }) + const h = f.handler({ action: 'collect', ids: ['vm-a', 'vm-b'] }) + await h.request('POST', '/api/panel/vms/fleet-update') + assert.equal(h.response.status, 409) + const next = await f.otherOperations.tryAcquire({ vmId: 'vm-a' }) + assert.equal(next.ok, true) + await next.release() +}) + test('create claims the new physical identity before writing a VM or fingerprint', async (t) => { const f = await fixture(t) await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' }) From 11df85c4be7792c7c6f7857f9af86aa0c8219ea9 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:01:03 +0800 Subject: [PATCH 273/473] fix: await shared background work within shutdown deadline --- src/server.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server.mjs b/src/server.mjs index 35361c446..d1213a86c 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -106,6 +106,7 @@ import { handleUserCountTokens, handleUserUsage } from './lib/protocol/user-coun import { createPanelHandler } from './lib/admin/panel-routes.mjs' import { isKubernetesMode, shutdownTimeoutMs } from './lib/core/control-plane-mode.mjs' import { ControlPlaneLifecycle, withTimeout } from './lib/core/control-plane-lifecycle.mjs' +import { getSharedBackgroundTasks } from './lib/core/shared-background-tasks.mjs' import { ControlPlaneLock } from './lib/db/control-plane-lock.mjs' const FEATURES = [ @@ -1650,6 +1651,7 @@ async function shutdown(signal, { failed = false } = {}) { await pausePriceCatalog() for (const monitor of maintenanceMonitors()) await monitor?.flush?.() await vmTestSchedule.flush() + if (kubernetesMode) await getSharedBackgroundTasks(db).flush() await flushRuntimeWrites() stopAllRustKernels() await clusterManager.stop() From e02b762eb7cd544cb4b3ea938b404902081152f4 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:02:23 +0800 Subject: [PATCH 274/473] fix: retain fleet mutation claims through all target work --- src/lib/admin/panel-routes.mjs | 19 +++++--- src/lib/vm/fleet-update.mjs | 10 ++++- test/unit/fleet-update.test.mjs | 17 +++++++ test/unit/panel-vm-shared-mutations.test.mjs | 47 ++++++++++++++++++++ 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 39a97e184..4cddae914 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -2388,13 +2388,20 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && p === '/api/panel/vms/fleet-update') { const body = await readBody(req, 32 * 1024).catch(() => ({})) - const report = await runFleetUpdate(cfg.paths.project, { - action: body.action || 'roll', - ids: Array.isArray(body.ids) ? body.ids : null, - concurrency: body.concurrency, - routing: ctx.routingConfig, + const wanted = Array.isArray(body.ids) && body.ids.length ? new Set(body.ids.map(String)) : null + const ids = listVms(cfg.paths.project) + .filter((vm) => !wanted || wanted.has(vm.id)) + .map((vm) => vm.id) + return mutateProxyVms(ids, async (handles) => { + const report = await (ctx.vmMutationOps?.runFleetUpdate || runFleetUpdate)(cfg.paths.project, { + action: body.action || 'roll', + targetIds: ids, + concurrency: body.concurrency, + routing: ctx.routingConfig, + }) + for (const item of report.items || []) await vmEffect(handles.get(item.id), async () => item) + return json(res, 200, panel.ok(report)) }) - return json(res, 200, panel.ok(report)) } if (req.method === 'GET' && p === '/api/panel/wrap-cli') { const sample = describeWrapSample(cfg.paths.project) diff --git a/src/lib/vm/fleet-update.mjs b/src/lib/vm/fleet-update.mjs index bd65f0d66..fc0e6e0b5 100644 --- a/src/lib/vm/fleet-update.mjs +++ b/src/lib/vm/fleet-update.mjs @@ -139,7 +139,8 @@ export async function updateOneSlot( if (action === 'reload' || action === 'roll') { const reloaded = await reloadFn(vm, projectRoot, { routing }) result.reload = reloaded - if (!reloaded?.ok) { + if (!reloaded?.ok || reloaded.rust_ok === false || reloaded.vmOperationUncertain) { + result.vmOperationUncertain = reloaded?.vmOperationUncertain === true || reloaded?.rust_ok === false result.error = reloaded?.error || 'reload failed' result.code = reloaded?.code || 'reload_failed' return result @@ -149,6 +150,7 @@ export async function updateOneSlot( healthFn, }) result.ready = !!ready.ok + if (!ready.ok) result.vmOperationUncertain = true if (!ready.ok && action === 'reload') { result.error = ready.error || 'worker_not_ready' return result @@ -166,6 +168,7 @@ export async function updateOneSlot( ? await restartRemoteServices(vmRef, projectRoot, { telemetryEnabled: enabled }) : await restartLocalServices({ projectRoot, vm: vmRef, enabled }) result.telemetry = changed + if (changed?.vmOperationUncertain) result.vmOperationUncertain = true if (!changed?.ok) { result.error = changed?.error || 'telemetry service update failed' result.code = changed?.code || 'telemetry_update_failed' @@ -175,6 +178,7 @@ export async function updateOneSlot( if (action === 'collect' || action === 'roll') { const collected = await collectFn(projectRoot, getVm(projectRoot, id) || vm) result.collect = collected + if (collected?.vmOperationUncertain) result.vmOperationUncertain = true if (!collected?.ok) { result.error = collected?.error || 'collect failed' result.code = collected?.code || 'collect_failed' @@ -194,6 +198,7 @@ export async function runFleetUpdate( { action = 'roll', ids = null, + targetIds, concurrency = 4, routing = {}, reloadFn, @@ -206,7 +211,8 @@ export async function runFleetUpdate( ) { const nextAction = ACTIONS.has(action) ? action : 'roll' const conc = Math.max(1, Math.min(8, Number(concurrency) || 4)) - const targets = selectFleetTargets(projectRoot, ids) + const wanted = targetIds ? new Set(targetIds) : null + const targets = selectFleetTargets(projectRoot, ids).filter((vm) => !wanted || wanted.has(vm.id)) const items = await mapPool(targets, conc, (vm) => updateOneSlot(projectRoot, vm, { action: nextAction, diff --git a/test/unit/fleet-update.test.mjs b/test/unit/fleet-update.test.mjs index 47d525fbe..6191ab5b4 100644 --- a/test/unit/fleet-update.test.mjs +++ b/test/unit/fleet-update.test.mjs @@ -97,6 +97,23 @@ test('fleet does not treat a partial Rust reload as completed successfully', asy assert.equal(report.items[0].vmOperationUncertain, true) }) +test('fleet targetIds never expands a frozen or empty claimed target set', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-fixed-targets-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + writeSlot(root, 'vm-a') + writeSlot(root, 'vm-b') + const calls = [] + const collectFn = async (_root, vm) => { + calls.push(vm.id) + return { ok: true } + } + await runFleetUpdate(root, { action: 'collect', targetIds: ['vm-a'], collectFn }) + assert.deepEqual(calls, ['vm-a']) + const empty = await runFleetUpdate(root, { action: 'collect', targetIds: [], collectFn }) + assert.equal(empty.total, 0) + assert.deepEqual(calls, ['vm-a']) +}) + test('fleet telemetry uses injected telemetryFn with options object', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-fleet-tel-')) writeSlot(root, 'vm-01', { node_id: 'node-a' }) diff --git a/test/unit/panel-vm-shared-mutations.test.mjs b/test/unit/panel-vm-shared-mutations.test.mjs index eb13fdac7..93151ec17 100644 --- a/test/unit/panel-vm-shared-mutations.test.mjs +++ b/test/unit/panel-vm-shared-mutations.test.mjs @@ -139,6 +139,53 @@ test('fleet update claims every selected target before starting the first comman await next.release() }) +test('fleet update waits for all original work before releasing its global VM claims', async (t) => { + const f = await fixture(t) + f.seed('vm-a') + f.seed('vm-b') + let finish + let enter + const pending = new Promise((resolve) => { + finish = resolve + }) + const entered = new Promise((resolve) => { + enter = resolve + }) + const report = { + total: 2, + failed_count: 0, + items: [ + { id: 'vm-a', ok: true }, + { id: 'vm-b', ok: true }, + ], + } + const h = f.handler( + { action: 'collect' }, + { + vmMutationOps: { + runFleetUpdate: async (_root, options) => { + assert.deepEqual(options.targetIds, ['vm-a', 'vm-b']) + enter() + return pending + }, + }, + }, + ) + const request = h.request('POST', '/api/panel/vms/fleet-update') + try { + await entered + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-a' })).ok, false) + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-b' })).ok, false) + } finally { + finish(report) + await request + } + assert.equal(h.response.status, 200) + const next = await f.otherOperations.tryAcquire({ vmId: 'vm-b' }) + assert.equal(next.ok, true) + await next.release() +}) + test('create claims the new physical identity before writing a VM or fingerprint', async (t) => { const f = await fixture(t) await f.otherOperations.tryAcquire({ vmId: 'vm-synthetic' }) From 027483644a2fefbf7cf8217857d266a02d9c3c97 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:03:04 +0800 Subject: [PATCH 275/473] test: specify shared credential refresh admission and observation guards --- test/unit/shared-refresh-admission.test.mjs | 150 ++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 test/unit/shared-refresh-admission.test.mjs diff --git a/test/unit/shared-refresh-admission.test.mjs b/test/unit/shared-refresh-admission.test.mjs new file mode 100644 index 000000000..59717bee1 --- /dev/null +++ b/test/unit/shared-refresh-admission.test.mjs @@ -0,0 +1,150 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { BackgroundTaskRunsRepo } from '../../src/lib/db/repos/background-task-runs-repo.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => peer.close()) + return { db, peer } +} + +const execution = (patch = {}) => ({ + accountId: 'refresh-account', + vmId: 'refresh-vm', + accountLimit: 10, + vmLimit: 10, + rpmLimit: 0, + fableLimit: 0, + ...patch, +}) + +test('a stale background observation cannot create a second attempt after its winner completed', async (t) => { + const { db, peer } = await fixture(t) + const first = new BackgroundTaskRunsRepo(db) + const second = new BackgroundTaskRunsRepo(peer) + const one = await first.tryClaim({ taskKey: 'refresh-observation', executionId: 'one', ownerId: 'pod-a' }) + await first.complete(one) + const two = await first.tryClaim({ + taskKey: 'refresh-observation', + executionId: 'two', + ownerId: 'pod-a', + ifPreviousExecutionId: 'one', + }) + await first.complete(two) + const stale = await second.tryClaim({ + taskKey: 'refresh-observation', + executionId: 'three', + ownerId: 'pod-b', + ifPreviousExecutionId: 'one', + }) + assert.equal(stale.ok, false) + assert.equal(stale.code, 'background_task_changed') + const later = await second.tryClaim({ + taskKey: 'refresh-observation', + executionId: 'four', + ownerId: 'pod-b', + ifPreviousExecutionId: 'two', + }) + assert.equal(later.ok, true) + await second.complete(later) +}) + +test('a refresh can hold its exact completed admission while blocking every new admission', async (t) => { + const { db, peer } = await fixture(t) + const gate = await new ExecutionAdmissionsRepo(db).tryReserve(execution()) + await gate.bindExecution({ requestId: 'refresh-ended-job' }) + await gate.completeExecution({ + requestId: 'refresh-ended-job', + started: true, + ended: true, + reason: 'http_body_complete', + }) + const operation = await new SharedVmOperations(peer).tryAcquire({ + vmId: gate.vmId, + accountId: gate.accountId, + kind: 'oauth_refresh', + admission: { db, ...gate }, + }) + try { + assert.equal(operation.ok, true) + assert.equal((await new ExecutionAdmissionsRepo(db).tryReserve(execution())).code, 'vm_operation_active') + } finally { + await operation.release?.() + await gate.release() + } +}) + +test('refresh never excludes a pending job or another active admission', async (t) => { + const { db, peer } = await fixture(t) + const gate = await new ExecutionAdmissionsRepo(db).tryReserve(execution()) + await gate.bindExecution({ requestId: 'refresh-pending-job' }) + const operations = new SharedVmOperations(peer) + assert.equal( + (await operations.tryAcquire({ vmId: gate.vmId, accountId: gate.accountId, admission: { db, ...gate } })).ok, + false, + ) + await gate.completeExecution({ + requestId: 'refresh-pending-job', + started: true, + ended: true, + reason: 'http_body_complete', + }) + const sibling = await new ExecutionAdmissionsRepo(db).tryReserve(execution()) + try { + assert.equal( + (await operations.tryAcquire({ vmId: gate.vmId, accountId: gate.accountId, admission: { db, ...gate } })).ok, + false, + ) + assert.equal( + ( + await operations.tryAcquire({ + vmId: gate.vmId, + accountId: gate.accountId, + admission: { db, ...gate, ownerId: 'wrong-owner' }, + }) + ).ok, + false, + ) + } finally { + await sibling.release() + await gate.release() + } +}) + +test('a refresh is blocked by the same account executing on a different VM', async (t) => { + const { db, peer } = await fixture(t) + const other = await new ExecutionAdmissionsRepo(db).tryReserve(execution({ vmId: 'another-refresh-vm' })) + const operation = await new SharedVmOperations(peer).tryAcquire({ + vmId: 'refresh-vm', + accountId: other.accountId, + kind: 'oauth_refresh', + }) + try { + assert.equal(operation.ok, false) + assert.equal(operation.code, 'vm_execution_active') + } finally { + await operation.release?.() + await other.release() + } +}) + +test('a nested refresh cannot reuse a VM operation already marked uncertain', async (t) => { + const { db, peer } = await fixture(t) + const outer = new SharedVmOperations(db) + const nested = new SharedVmOperations(peer) + let effects = 0 + await outer.run('refresh-vm', async (operation) => { + await operation.markUncertain() + await assert.rejects( + nested.run('refresh-vm', async () => { + effects++ + }), + { code: 'vm_operation_inactive' }, + ) + }) + assert.equal(effects, 0) +}) From 07bb3a8527aba8b7f71300b4019adb9d2b31e787 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:05:13 +0800 Subject: [PATCH 276/473] test: require global artifact and affected VM ownership --- test/unit/panel-vm-artifacts.test.mjs | 115 ++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 test/unit/panel-vm-artifacts.test.mjs diff --git a/test/unit/panel-vm-artifacts.test.mjs b/test/unit/panel-vm-artifacts.test.mjs new file mode 100644 index 000000000..0c2e8913b --- /dev/null +++ b/test/unit/panel-vm-artifacts.test.mjs @@ -0,0 +1,115 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { getSharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { BackgroundTaskRunsRepo } from '../../src/lib/db/repos/background-task-runs-repo.mjs' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' + +const TASK_KEY = 'vm_artifact_mutation' +const elf = Buffer.alloc(64) +elf.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]) +elf.writeUInt16LE(62, 18) + +async function fixture(t, { empty = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-vm-artifacts-')) + fs.mkdirSync(path.join(root, 'vms')) + if (!empty) + fs.writeFileSync( + path.join(root, 'vms', 'vm-a.json'), + JSON.stringify({ id: 'vm-a', status: 'stopped', inference_engine: 'go' }), + ) + const db = getDb() + await withDatabaseActivities([db], () => Promise.resolve()) + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const mode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + const tasks = getSharedBackgroundTasks(db) + t.after(async () => { + await tasks.flush() + await peer.close() + if (mode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + fs.rmSync(root, { recursive: true, force: true }) + }) + const other = new SharedVmOperations(peer) + function handler(body = {}, extra = {}) { + const response = {} + const handle = createPanelHandler({ + cfg: { paths: { project: root } }, + routingConfig: {}, + vmOperations: new SharedVmOperations(db), + backgroundTasks: tasks, + proxyPool: { refresh: async () => {} }, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + json(_res, status, payload) { + response.status = status + response.body = payload + return true + }, + readBody: async () => body, + readRawBody: async () => elf, + fetchImpl: async () => { + throw new Error('synthetic no network') + }, + persistRoutingPatch: async () => {}, + ...extra, + }) + return { response, request: (route) => handle({ method: 'POST' }, {}, new URL('http://localhost' + route)) } + } + return { root, db, peer, tasks, other, handler } +} + +for (const [route, body] of [ + ['/api/panel/dataplane', { dataplane: 'wrap' }], + ['/api/panel/wrap-cli/crag-kernel', {}], + ['/api/panel/wrap-cli/make', {}], + ['/api/panel/wrap-cli/kernel/release', { tag: 'v1.2.3' }], + ['/api/panel/wrap-cli/kernel', {}], + ['/api/panel/wrap-cli/sync', {}], + ['/api/panel/vms/vm-a/wrap-cli/promote', {}], +]) { + test(`artifact ${route} claims affected VMs before its first file or remote effect`, async (t) => { + const f = await fixture(t) + await f.other.tryAcquire({ vmId: 'vm-a' }) + const h = f.handler(body) + await h.request(route) + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(fs.existsSync(path.join(f.root, 'share')), false) + assert.equal(fs.existsSync(path.join(f.root, 'bin')), false) + }) +} + +test('an empty fleet still respects a persistent artifact owner', async (t) => { + const f = await fixture(t, { empty: true }) + await new BackgroundTaskRunsRepo(f.peer).tryClaim({ taskKey: TASK_KEY, executionId: 'foreign', ownerId: 'synthetic' }) + const h = f.handler() + await h.request('/api/panel/wrap-cli/kernel') + assert.equal(h.response.status, 409) + assert.equal(h.response.body.error.code, 'vm_artifact_operation_active') + assert.equal(fs.existsSync(path.join(f.root, 'share')), false) +}) + +test('an idle artifact upload completes its durable operation and releases VM ownership', async (t) => { + const f = await fixture(t) + const h = f.handler() + await h.request('/api/panel/wrap-cli/kernel') + assert.equal(h.response.status, 200, JSON.stringify(h.response.body)) + const row = await f.peer + .prepare('SELECT finished_at, unknown_at FROM background_task_runs WHERE task_key = $1') + .get(TASK_KEY) + assert.notEqual(row?.finished_at, null) + assert.equal(row?.unknown_at, null) + const next = await f.other.tryAcquire({ vmId: 'vm-a' }) + assert.equal(next.ok, true) + await next.release() +}) From 05fde38d16d30d22efe33d192b16bdd7b455c812 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:05:23 +0800 Subject: [PATCH 277/473] test: define persistent shared refresh lifecycle and generation guarantees --- test/unit/shared-credential-refresh.test.mjs | 191 +++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 test/unit/shared-credential-refresh.test.mjs diff --git a/test/unit/shared-credential-refresh.test.mjs b/test/unit/shared-credential-refresh.test.mjs new file mode 100644 index 000000000..1e0d807c2 --- /dev/null +++ b/test/unit/shared-credential-refresh.test.mjs @@ -0,0 +1,191 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' + +async function fixture(t) { + const { SharedCredentialRefresh } = await import('../../src/lib/oauth/shared-credential-refresh.mjs') + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-refresh-')) + const file = path.join(root, 'credential.json') + fs.writeFileSync( + file, + JSON.stringify({ + generation: 'generation-one', + credential: { access_token: 'SYNTHETIC_AT_ONE', refresh_token: 'SYNTHETIC_RT_ONE' }, + }), + ) + t.after(async () => { + await peer.close() + fs.rmSync(root, { recursive: true, force: true }) + }) + const readCurrent = async () => JSON.parse(fs.readFileSync(file, 'utf8')) + const publish = async (_current, value) => { + const next = { + generation: 'generation-two', + credential: { access_token: value.access_token, refresh_token: value.refresh_token }, + } + const pending = file + '.next' + fs.writeFileSync(pending, JSON.stringify(next)) + fs.renameSync(pending, file) + return readCurrent() + } + return { + db, + peer, + file, + readCurrent, + publish, + first: new SharedCredentialRefresh(db), + second: new SharedCredentialRefresh(peer), + } +} + +const input = () => ({ + platform: 'codex', + vmId: 'refresh-vm', + accountId: 'refresh-account', + expectedGeneration: 'generation-one', +}) +const success = () => ({ + finished: true, + value: { ok: true, access_token: 'SYNTHETIC_AT_TWO', refresh_token: 'SYNTHETIC_RT_TWO' }, +}) + +test('two connections share one actual refresh and read published credentials without storing tokens in PostgreSQL', async (t) => { + const f = await fixture(t) + let calls = 0 + const refresh = async () => { + calls++ + await new Promise((resolve) => setTimeout(resolve, 30)) + return success() + } + const results = await Promise.all( + [f.first, f.second].map((service) => + service.run(input(), { readCurrent: f.readCurrent, publish: f.publish, refresh }), + ), + ) + assert.equal(calls, 1) + assert.ok(results.every((result) => result.finished && result.current.generation === 'generation-two')) + const stored = JSON.stringify((await f.db.query('SELECT shared_result FROM background_task_runs')).rows) + assert.equal(stored.includes('SYNTHETIC_AT'), false) + assert.equal(stored.includes('SYNTHETIC_RT'), false) +}) + +test('an expected generation superseded before admission adopts the new credential without refreshing', async (t) => { + const f = await fixture(t) + await f.publish(null, success().value) + let calls = 0 + const result = await f.first.run(input(), { + readCurrent: f.readCurrent, + publish: f.publish, + refresh: async () => { + calls++ + return success() + }, + }) + assert.equal(calls, 0) + assert.equal(result.shared, true) + assert.equal(result.current.generation, 'generation-two') +}) + +test('an uncertain refresh retains both claims and cannot retry after any timeout age', async (t) => { + const f = await fixture(t) + let calls = 0 + const callbacks = { + readCurrent: f.readCurrent, + publish: f.publish, + refresh: async () => { + calls++ + return { finished: false, value: { ok: false, error: 'synthetic_transport' } } + }, + } + const first = await f.first.run(input(), callbacks) + assert.equal(first.unknown, true) + await f.db.exec( + "UPDATE background_task_runs SET claimed_at=clock_timestamp()-interval '10 years'; UPDATE vm_operations SET created_at=clock_timestamp()-interval '10 years'", + ) + const second = await f.second.run(input(), callbacks) + assert.equal(second.unknown, true) + assert.equal(calls, 1) + const task = (await f.db.query('SELECT finished_at,unknown_at FROM background_task_runs')).rows[0] + const operation = (await f.db.query('SELECT released_at,uncertain_at FROM vm_operations')).rows[0] + assert.equal(task.finished_at, null) + assert.ok(task.unknown_at) + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) +}) + +test('a later caller may retry a definitively ended failure with the same credential generation', async (t) => { + const f = await fixture(t) + let calls = 0 + const callbacks = { + readCurrent: f.readCurrent, + publish: f.publish, + refresh: async () => { + calls++ + return { finished: true, value: { ok: false, error: 'synthetic_rejection' } } + }, + } + await f.first.run(input(), callbacks) + await f.second.run(input(), callbacks) + assert.equal(calls, 2) + assert.equal( + (await f.db.query('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NOT NULL')).rows[0].n, + 2, + ) +}) + +test('missing expected generation or unreadable current credential never falls back to a raw refresh', async (t) => { + const f = await fixture(t) + let calls = 0 + const callbacks = { + readCurrent: f.readCurrent, + publish: f.publish, + refresh: async () => { + calls++ + return success() + }, + } + await assert.rejects(f.first.run({ ...input(), expectedGeneration: null }, callbacks), { + code: 'credential_generation_required', + }) + await assert.rejects( + f.first.run(input(), { + ...callbacks, + readCurrent: async () => { + throw new Error('synthetic missing credential') + }, + }), + /synthetic missing credential/, + ) + assert.equal(calls, 0) +}) + +test('a failed terminal commit after publication keeps refresh ownership unavailable', async (t) => { + const f = await fixture(t) + await f.db.exec(`CREATE FUNCTION refuse_refresh_completion() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.finished_at IS NOT NULL THEN RAISE EXCEPTION 'synthetic completion failure' USING ERRCODE='P6487'; END IF; RETURN NEW; END $$; + CREATE TRIGGER refuse_refresh_completion BEFORE UPDATE ON background_task_runs FOR EACH ROW EXECUTE FUNCTION refuse_refresh_completion()`) + let calls = 0 + await assert.rejects( + f.first.run(input(), { + readCurrent: f.readCurrent, + publish: f.publish, + refresh: async () => { + calls++ + return success() + }, + }), + { code: 'P6487' }, + ) + assert.equal(calls, 1) + const operation = (await f.db.query('SELECT released_at,uncertain_at FROM vm_operations')).rows[0] + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + const task = (await f.db.query('SELECT finished_at FROM background_task_runs')).rows[0] + assert.equal(task.finished_at, null) +}) From b76829a820414e0042f16083b24f937b7eb84322 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:06:35 +0800 Subject: [PATCH 278/473] fix: serialize shared VM artifacts with persistent task claims --- src/lib/admin/panel-routes.mjs | 392 +++++++++++++++----------- test/unit/panel-vm-artifacts.test.mjs | 64 +++++ 2 files changed, 297 insertions(+), 159 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 4cddae914..e7b53e017 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -6,6 +6,9 @@ import { isKubernetesMode } from '../core/control-plane-mode.mjs' */ import fs from 'node:fs' import path from 'node:path' +import { randomUUID } from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' +import { backgroundTasksFor } from '../core/shared-background-tasks.mjs' import { verifyPanelLogin, createPanelSession, @@ -301,6 +304,7 @@ function logsOwner(req, u) { } const LOCAL_PROXY_MISMATCH_MESSAGE = '本地代理只能绑定到它所在 VPS 上的槽位' +const artifactCallbacks = new AsyncLocalStorage() function clusterNodesOf(ctx) { try { @@ -495,8 +499,11 @@ async function syncInstalledKernels({ routingConfig, body = {}, refreshCluster = refreshClusterSlotPayloads, + targetIds, + effect = (_id, callback) => callback(), }) { - const all = listVms(project) + const allowed = targetIds ? new Set(targetIds) : null + const all = listVms(project).filter((vm) => !allowed || allowed.has(vm.id)) const wanted = Array.isArray(body.ids) ? new Set(body.ids.map(String)) : null const selected = wanted ? all.filter((vm) => wanted.has(vm.id)) : all // Local slots bind the new binaries. Cluster slots bake the same kernel, CLIs @@ -518,7 +525,7 @@ async function syncInstalledKernels({ if (resolveInferenceEngine(vm, routingConfig) !== 'rust') continue writeKernelConfig(project, vm, { routing: routingConfig, timezone: vm.timezone }) const exec = slotExec(project, vm) - item.kernel = await restartRustKernel(exec).catch((e) => ({ + item.kernel = await effect(item.id, () => restartRustKernel(exec)).catch((e) => ({ ok: false, error: String(e?.message || e).slice(0, 200), })) @@ -536,7 +543,7 @@ async function syncInstalledKernels({ if (traits) { for (const vm of localVms) { if (isCodexVm(vm)) continue - const gone = await destroySlot(vm) + const gone = await effect(vm.id, () => destroySlot(vm)) if (!gone.ok) { const item = (local.items || []).find((row) => row.id === vm.id) if (item) { @@ -547,7 +554,9 @@ async function syncInstalledKernels({ continue } const rebuilt = recreateVmFiles(project, vm, { mode: 'traits' }) - const boot = await startSlotReady(rebuilt.vm, project, { recreate: true, routing: routingConfig }) + const boot = await effect(vm.id, () => + startSlotReady(rebuilt.vm, project, { recreate: true, routing: routingConfig }), + ) const item = (local.items || []).find((row) => row.id === vm.id) || { id: vm.id, ok: true, @@ -584,6 +593,7 @@ async function syncInstalledKernels({ : await refreshCluster(project, clusterVms, { routing: routingConfig, recreate: true }) : null const failed = (local.items || []).filter((item) => !item.ok) + for (const item of cluster?.items || []) await effect(item.id, async () => item) local.ok = failed.length === 0 local.ok_count = (local.items || []).length - failed.length local.failed_count = failed.length @@ -646,6 +656,7 @@ export function createPanelHandler(ctx) { try { result = await callback() } catch (error) { + if (artifactCallbacks.getStore()) artifactCallbacks.getStore().uncertain = true await operation?.markUncertain?.() await markCurrentProxyOperationUncertain(getDb()) throw error @@ -656,17 +667,20 @@ export function createPanelHandler(ctx) { result?.report?.ok === false || (result?.ok !== true && result?.notStarted !== true && result?.vmOperationNotStarted !== true) ) { + if (artifactCallbacks.getStore()) artifactCallbacks.getStore().uncertain = true await operation?.markUncertain?.() await markCurrentProxyOperationUncertain(getDb()) } return result } async function mutateProxyVms(ids, callback) { - if (!isKubernetesMode() && !ctx.proxyOperations) return mutateVms(ids, callback) + if (!isKubernetesMode() && !ctx.proxyOperations) + return mutateVms(typeof ids === 'function' ? await ids() : ids, callback) return (ctx.proxyOperations || getSharedProxyOperations()).run(async () => { await ctx.proxyPool.refresh?.() const bindings = await getDb().prepare('SELECT bound_vm_id FROM proxies WHERE bound_vm_id IS NOT NULL').all() - const targets = [...new Set([...ids, ...bindings.flatMap((proxy) => parseBoundVmIds(proxy.bound_vm_id))])] + const requested = typeof ids === 'function' ? await ids() : ids + const targets = [...new Set([...requested, ...bindings.flatMap((proxy) => parseBoundVmIds(proxy.bound_vm_id))])] let entered = false try { const value = await mutateVms(targets, async (handles) => { @@ -680,6 +694,40 @@ export function createPanelHandler(ctx) { } }) } + async function mutateVmArtifacts(callback) { + if (!isKubernetesMode()) return callback(new Map()) + const result = await backgroundTasksFor(ctx.backgroundTasks).run( + { taskKey: 'vm_artifact_mutation', executionId: randomUUID() }, + async () => { + const state = { uncertain: false } + return artifactCallbacks.run(state, async () => { + let entered = false + try { + const value = await mutateProxyVms( + async () => { + const rows = await getDb().prepare('SELECT id FROM vms').all() + return [...new Set([...rows.map((vm) => vm.id), ...listVms(cfg.paths.project).map((vm) => vm.id)])] + }, + async (handles) => { + entered = true + return callback(handles) + }, + ) + return { terminal: !state.uncertain, value } + } catch (error) { + if (!entered) error.taskNotStarted = true + throw error + } + }) + }, + ) + if (!result.ran) + throw Object.assign(new Error('VM artifact operation is active'), { + code: 'vm_artifact_operation_active', + status: 409, + }) + return result.value + } function proxyEffect(handles, callback) { return vmEffect( { @@ -2415,172 +2463,194 @@ export function createPanelHandler(ctx) { ) } if (req.method === 'POST' && p === '/api/panel/dataplane') { - const body = await readBody(req, 8 * 1024).catch(() => ({})) - const parsed = parseKernelDataplanePatch(body.dataplane) - if (!parsed.ok || !parsed.value) { - return json(res, 400, { - ok: false, - error: { code: 'invalid_dataplane', message: parsed.error || 'dataplane must be wrap, cc, or crag' }, - }) - } - const targets = parseSlotPolicyTargets({ - ids: body.ids, - all: body.all === true || !Array.isArray(body.ids), - }) - if (!targets.ok) { - return json(res, 400, { ok: false, error: { message: targets.error } }) - } - const previous = structuredClone(ctx.routingConfig) - try { - if (targets.all) { - await persistRoutingPatch({ - inference: { - ...(ctx.routingConfig.inference || {}), - dataplane: parsed.value, - }, + return mutateVmArtifacts(async (handles) => { + const body = await readBody(req, 8 * 1024).catch(() => ({})) + const parsed = parseKernelDataplanePatch(body.dataplane) + if (!parsed.ok || !parsed.value) { + return json(res, 400, { + ok: false, + error: { code: 'invalid_dataplane', message: parsed.error || 'dataplane must be wrap, cc, or crag' }, }) - } else { - for (const id of targets.ids) { - persistSlotEnginePolicy(cfg.paths.project, id, { dataplane: parsed.value }) + } + const targets = parseSlotPolicyTargets({ + ids: body.ids, + all: body.all === true || !Array.isArray(body.ids), + }) + if (!targets.ok) { + return json(res, 400, { ok: false, error: { message: targets.error } }) + } + const previous = structuredClone(ctx.routingConfig) + try { + if (targets.all) { + await persistRoutingPatch({ + inference: { + ...(ctx.routingConfig.inference || {}), + dataplane: parsed.value, + }, + }) + } else { + for (const id of targets.ids) { + persistSlotEnginePolicy(cfg.paths.project, id, { dataplane: parsed.value }) + } } + } catch (error) { + ctx.routingConfig = previous + await proxyEffect(handles, async () => ({ ok: false })) + return json(res, 503, { + ok: false, + error: { code: 'dataplane_persist_failed', message: String(error?.message || error) }, + }) } - } catch (error) { - ctx.routingConfig = previous - return json(res, 503, { - ok: false, - error: { code: 'dataplane_persist_failed', message: String(error?.message || error) }, + const report = await (ctx.vmMutationOps?.syncInstalledKernels || syncInstalledKernels)({ + project: cfg.paths.project, + targetIds: isKubernetesMode() ? [...handles.keys()] : undefined, + effect: (id, callback) => vmEffect(handles.get(id), callback), + routingConfig: ctx.routingConfig, + body: { + ids: targets.all ? undefined : targets.ids, + restart: body.restart !== false, + }, }) - } - const report = await syncInstalledKernels({ - project: cfg.paths.project, - routingConfig: ctx.routingConfig, - body: { - ids: targets.all ? undefined : targets.ids, - restart: body.restart !== false, - }, + for (const item of report.items || []) await vmEffect(handles.get(item.id), async () => item) + return json( + res, + report.ok ? 200 : 503, + panel.ok({ + dataplane: parsed.value, + ...report, + }), + ) }) - return json( - res, - report.ok ? 200 : 503, - panel.ok({ - dataplane: parsed.value, - ...report, - }), - ) } if (req.method === 'POST' && p === '/api/panel/wrap-cli/crag-kernel') { - const raw = await readRawBody(req, 32 * 1024 * 1024) - const written = replaceCragKernelBinary(cfg.paths.project, raw) - if (!written.ok) { - return json(res, 400, { ok: false, error: { code: written.code, message: written.error } }) - } - return json(res, 200, panel.ok(written)) + return mutateVmArtifacts(async (handles) => { + const raw = await readRawBody(req, 32 * 1024 * 1024) + const written = replaceCragKernelBinary(cfg.paths.project, raw) + if (!written.ok) { + return json(res, 400, { ok: false, error: { code: written.code, message: written.error } }) + } + return json(res, 200, panel.ok(written)) + }) } if (req.method === 'POST' && p === '/api/panel/wrap-cli/make') { - const body = await readBody(req, 8 * 1024).catch(() => ({})) - const glibcVm = String(body.glibc_vm || body.glibcVm || '').trim() - let glibcFromDir = '' - if (glibcVm) { - const candidate = path.join(wrapCliHomeDir(cfg.paths.project, glibcVm), 'glibc239') - if (fs.existsSync(candidate)) glibcFromDir = candidate - } - const made = makeWrapSample(cfg.paths.project, { glibcFromDir }) - if (!made.ok) { - return json(res, 400, { ok: false, error: { code: made.code, message: made.error } }) - } - return json(res, 200, panel.ok(made)) + return mutateVmArtifacts(async (handles) => { + const body = await readBody(req, 8 * 1024).catch(() => ({})) + const glibcVm = String(body.glibc_vm || body.glibcVm || '').trim() + let glibcFromDir = '' + if (glibcVm) { + const candidate = path.join(wrapCliHomeDir(cfg.paths.project, glibcVm), 'glibc239') + if (fs.existsSync(candidate)) glibcFromDir = candidate + } + const made = makeWrapSample(cfg.paths.project, { glibcFromDir }) + if (!made.ok) { + return json(res, 400, { ok: false, error: { code: made.code, message: made.error } }) + } + return json(res, 200, panel.ok(made)) + }) } if (req.method === 'POST' && p === '/api/panel/wrap-cli/kernel/release') { - const body = await readBody(req, 8 * 1024).catch(() => ({})) - const downloaded = await downloadReleaseKernel({ - tag: body.tag, - fetchImpl: ctx.fetchImpl, - }) - if (!downloaded.ok) { - return json(res, kernelReleaseHttpStatus(downloaded.code), { - ok: false, - error: { code: downloaded.code, message: downloaded.error }, + return mutateVmArtifacts(async (handles) => { + const body = await readBody(req, 8 * 1024).catch(() => ({})) + const downloaded = await downloadReleaseKernel({ + tag: body.tag, + fetchImpl: ctx.fetchImpl, }) - } - const replaced = replaceKernelBinary(cfg.paths.project, downloaded.bytes, { - source: 'github', - release_tag: downloaded.tag, - }) - if (!replaced.ok) { - return json(res, 400, { ok: false, error: { code: replaced.code, message: replaced.error } }) - } - const cliNode = replaceCliNodeBinary(cfg.paths.project, downloaded.cliNode?.bytes) - if (!cliNode.ok) { - return json(res, 400, { ok: false, error: { code: cliNode.code, message: cliNode.error } }) - } - const ccNode = replaceCcNodeBinary(cfg.paths.project, downloaded.ccNode?.bytes) - if (!ccNode.ok) { - return json(res, 400, { ok: false, error: { code: ccNode.code, message: ccNode.error } }) - } - const cragWritten = replaceCragKernelBinary(cfg.paths.project, downloaded.crag?.bytes) - if (!cragWritten.ok) { - return json(res, 400, { ok: false, error: { code: cragWritten.code, message: cragWritten.error } }) - } - const report = await syncInstalledKernels({ - project: cfg.paths.project, - routingConfig: ctx.routingConfig, - body, - }) - const release = { - tag: downloaded.tag, - version: downloaded.version, - asset: downloaded.asset, - size: downloaded.size, - cli_node: downloaded.cliNode?.asset, - cli_node_size: cliNode.size, - cc_node: downloaded.ccNode?.asset, - cc_node_size: ccNode.size, - crag: downloaded.crag?.asset || null, - crag_size: cragWritten.size || 0, - crag_skipped: false, - } - if (!report.ok) { - return json(res, 400, { - ok: false, - error: { - code: 'kernel_sync_failed', - message: `已写入 ${downloaded.tag}，槽同步 ${report.ok_count}/${report.total} 失败`, - }, - data: { release, kernel: replaced, sync: report }, + if (!downloaded.ok) { + return json(res, kernelReleaseHttpStatus(downloaded.code), { + ok: false, + error: { code: downloaded.code, message: downloaded.error }, + }) + } + const replaced = replaceKernelBinary(cfg.paths.project, downloaded.bytes, { + source: 'github', + release_tag: downloaded.tag, }) - } - return json(res, 200, panel.ok({ release, kernel: replaced, sync: report })) + if (!replaced.ok) { + return json(res, 400, { ok: false, error: { code: replaced.code, message: replaced.error } }) + } + const cliNode = replaceCliNodeBinary(cfg.paths.project, downloaded.cliNode?.bytes) + if (!cliNode.ok) { + return json(res, 400, { ok: false, error: { code: cliNode.code, message: cliNode.error } }) + } + const ccNode = replaceCcNodeBinary(cfg.paths.project, downloaded.ccNode?.bytes) + if (!ccNode.ok) { + return json(res, 400, { ok: false, error: { code: ccNode.code, message: ccNode.error } }) + } + const cragWritten = replaceCragKernelBinary(cfg.paths.project, downloaded.crag?.bytes) + if (!cragWritten.ok) { + return json(res, 400, { ok: false, error: { code: cragWritten.code, message: cragWritten.error } }) + } + const report = await (ctx.vmMutationOps?.syncInstalledKernels || syncInstalledKernels)({ + project: cfg.paths.project, + targetIds: isKubernetesMode() ? [...handles.keys()] : undefined, + effect: (id, callback) => vmEffect(handles.get(id), callback), + routingConfig: ctx.routingConfig, + body, + }) + for (const item of report.items || []) await vmEffect(handles.get(item.id), async () => item) + const release = { + tag: downloaded.tag, + version: downloaded.version, + asset: downloaded.asset, + size: downloaded.size, + cli_node: downloaded.cliNode?.asset, + cli_node_size: cliNode.size, + cc_node: downloaded.ccNode?.asset, + cc_node_size: ccNode.size, + crag: downloaded.crag?.asset || null, + crag_size: cragWritten.size || 0, + crag_skipped: false, + } + if (!report.ok) { + return json(res, 400, { + ok: false, + error: { + code: 'kernel_sync_failed', + message: `已写入 ${downloaded.tag}，槽同步 ${report.ok_count}/${report.total} 失败`, + }, + data: { release, kernel: replaced, sync: report }, + }) + } + return json(res, 200, panel.ok({ release, kernel: replaced, sync: report })) + }) } if (req.method === 'POST' && p === '/api/panel/wrap-cli/kernel') { - let buf - try { - buf = await readRawBody(req, cfg.limits?.max_body_bytes || 32 * 1024 * 1024) - } catch (error) { - const status = error?.status || 400 - const body = error?.body || { error: { message: String(error?.message || error) } } - return json(res, status, { ok: false, ...body }) - } - if (!Buffer.isBuffer(buf)) buf = Buffer.from(buf || []) - if (!buf.length) { - return json(res, 400, { ok: false, error: { code: 'kernel_empty', message: 'kernel binary required' } }) - } - const replaced = replaceKernelBinary(cfg.paths.project, buf) - if (!replaced.ok) { - return json(res, 400, { ok: false, error: { code: replaced.code, message: replaced.error } }) - } - return json(res, 200, panel.ok(replaced)) + return mutateVmArtifacts(async (handles) => { + let buf + try { + buf = await readRawBody(req, cfg.limits?.max_body_bytes || 32 * 1024 * 1024) + } catch (error) { + const status = error?.status || 400 + const body = error?.body || { error: { message: String(error?.message || error) } } + return json(res, status, { ok: false, ...body }) + } + if (!Buffer.isBuffer(buf)) buf = Buffer.from(buf || []) + if (!buf.length) { + return json(res, 400, { ok: false, error: { code: 'kernel_empty', message: 'kernel binary required' } }) + } + const replaced = replaceKernelBinary(cfg.paths.project, buf) + if (!replaced.ok) { + return json(res, 400, { ok: false, error: { code: replaced.code, message: replaced.error } }) + } + return json(res, 200, panel.ok(replaced)) + }) } if (req.method === 'POST' && p === '/api/panel/wrap-cli/sync') { - const body = await readBody(req, 32 * 1024).catch(() => ({})) - const report = await syncInstalledKernels({ - project: cfg.paths.project, - routingConfig: ctx.routingConfig, - body, - refreshCluster, + return mutateVmArtifacts(async (handles) => { + const body = await readBody(req, 32 * 1024).catch(() => ({})) + const report = await (ctx.vmMutationOps?.syncInstalledKernels || syncInstalledKernels)({ + project: cfg.paths.project, + targetIds: isKubernetesMode() ? [...handles.keys()] : undefined, + effect: (id, callback) => vmEffect(handles.get(id), callback), + routingConfig: ctx.routingConfig, + body, + refreshCluster, + }) + for (const item of report.items || []) await vmEffect(handles.get(item.id), async () => item) + return json(res, report.ok ? 200 : 400, panel.ok(report)) }) - return json(res, report.ok ? 200 : 400, panel.ok(report)) } if (req.method === 'POST' && p === '/api/panel/vms/slot-policy') { const body = await readBody(req, 32 * 1024).catch(() => ({})) @@ -3939,13 +4009,16 @@ export function createPanelHandler(ctx) { } if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/wrap-cli\/promote$/.test(p)) { - const id = p.split('/')[4] - const vm = getVm(cfg.paths.project, id) - if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - if (hostUnsupported(res, vm, 'wrap_cli')) return - const captured = captureWrapSample(cfg.paths.project, vm) - if (!captured.ok) return json(res, 400, { ok: false, error: { code: captured.code, message: captured.error } }) - return json(res, 200, panel.ok(captured)) + return mutateVmArtifacts(async () => { + const id = p.split('/')[4] + const vm = getVm(cfg.paths.project, id) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + if (hostUnsupported(res, vm, 'wrap_cli')) return + const captured = captureWrapSample(cfg.paths.project, vm) + if (!captured.ok) + return json(res, 400, { ok: false, error: { code: captured.code, message: captured.error } }) + return json(res, 200, panel.ok(captured)) + }) } if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/wrap-cli\/repair$/.test(p)) { const id = p.split('/')[4] @@ -6363,6 +6436,7 @@ export function createPanelHandler(ctx) { 'proxy_operation_active', 'proxy_operation_inactive', 'vm_mutation_claim_required', + 'vm_artifact_operation_active', ].includes(error.code) ) { state.response = [res, error.status || 409, { ok: false, error: { code: error.code, message: error.message } }] diff --git a/test/unit/panel-vm-artifacts.test.mjs b/test/unit/panel-vm-artifacts.test.mjs index 0c2e8913b..410610617 100644 --- a/test/unit/panel-vm-artifacts.test.mjs +++ b/test/unit/panel-vm-artifacts.test.mjs @@ -107,9 +107,73 @@ test('an idle artifact upload completes its durable operation and releases VM ow const row = await f.peer .prepare('SELECT finished_at, unknown_at FROM background_task_runs WHERE task_key = $1') .get(TASK_KEY) + assert.ok(row) assert.notEqual(row?.finished_at, null) assert.equal(row?.unknown_at, null) const next = await f.other.tryAcquire({ vmId: 'vm-a' }) assert.equal(next.ok, true) await next.release() }) + +test('an overall successful artifact sync retains each explicit unknown target and the global claim', async (t) => { + const f = await fixture(t) + const h = f.handler( + {}, + { + vmMutationOps: { + syncInstalledKernels: async () => ({ ok: true, items: [{ id: 'vm-a', ok: true, vmOperationUncertain: true }] }), + }, + }, + ) + await h.request('/api/panel/wrap-cli/sync') + assert.equal(h.response.status, 200) + assert.equal((await f.other.tryAcquire({ vmId: 'vm-a' })).ok, false) + const row = await f.peer + .prepare('SELECT finished_at, unknown_at FROM background_task_runs WHERE task_key = $1') + .get(TASK_KEY) + assert.equal(row.finished_at, null) + assert.notEqual(row.unknown_at, null) +}) + +test('an empty fleet artifact download owns its original promise and rejects a concurrent writer', async (t) => { + const f = await fixture(t, { empty: true }) + let finish + let enter + const pending = new Promise((resolve) => { + finish = resolve + }) + const entered = new Promise((resolve) => { + enter = resolve + }) + const h = f.handler( + { tag: 'v1.2.3' }, + { + fetchImpl: async () => { + enter() + await pending + throw new Error('synthetic download ended') + }, + }, + ) + const request = h.request('/api/panel/wrap-cli/kernel/release') + try { + await entered + const concurrent = f.handler() + await concurrent.request('/api/panel/wrap-cli/kernel') + assert.equal(concurrent.response.status, 409) + assert.equal(fs.existsSync(path.join(f.root, 'share')), false) + assert.equal(f.tasks.pending.size, 1) + } finally { + finish() + await request + } + assert.equal(h.response.status, 502) + assert.equal( + ( + await f.peer + .prepare('SELECT count(*) AS n FROM background_task_runs WHERE task_key = $1 AND finished_at IS NULL') + .get(TASK_KEY) + ).n, + 0, + ) +}) From 713f6cae6d98c95d4a81c94caf110df338625ae8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:06:44 +0800 Subject: [PATCH 279/473] test(slotgate): require captured fences and durable PG node completion --- test/fixtures/fake-slotgate.mjs | 102 +++++++++++++++++++++++ test/unit/slotgate-client.test.mjs | 108 +++++++++++++++++++++++++ test/unit/slotgate-pg-binding.test.mjs | 67 +++++++++++++++ 3 files changed, 277 insertions(+) create mode 100644 test/fixtures/fake-slotgate.mjs create mode 100644 test/unit/slotgate-client.test.mjs create mode 100644 test/unit/slotgate-pg-binding.test.mjs diff --git a/test/fixtures/fake-slotgate.mjs b/test/fixtures/fake-slotgate.mjs new file mode 100644 index 000000000..d3a932803 --- /dev/null +++ b/test/fixtures/fake-slotgate.mjs @@ -0,0 +1,102 @@ +import { execFileSync } from 'node:child_process' +import { X509Certificate } from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import tls from 'node:tls' + +export function syntheticFence(generation = '1') { + return { + generation, + incarnation: 'synthetic-incarnation', + executor: { + kind: 'container', + node_boot_id: 'synthetic-boot', + container_id: 'a'.repeat(64), + docker_daemon_id: 'synthetic-engine', + started_at: '2026-10-21T00:00:00.000000001Z', + }, + } +} + +export const assistant = { + id: 'synthetic-message', type: 'message', role: 'assistant', + content: [{ type: 'text', text: 'synthetic answer' }], stop_reason: 'end_turn', +} + +// A local protocol double, not a node/Docker or provider acceptance test. +export async function fakeSlotgate(t, options = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-ksg2-')) + const keyPath = path.join(dir, 'key.pem') + const certPath = path.join(dir, 'cert.pem') + execFileSync('openssl', [ + 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', keyPath, + '-out', certPath, '-days', '1', '-subj', '/CN=synthetic-slotgate', + ], { stdio: 'ignore' }) + const cert = fs.readFileSync(certPath) + const sockets = new Set() + const commands = new Map() + const requests = [] + const token = 'synthetic-local-slotgate-token' + const server = tls.createServer({ key: fs.readFileSync(keyPath), cert, minVersion: 'TLSv1.3' }, (socket) => { + sockets.add(socket) + socket.on('error', () => {}) + socket.once('close', () => sockets.delete(socket)) + let buffer = Buffer.alloc(0) + let command + socket.on('data', (chunk) => { + buffer = Buffer.concat([buffer, chunk]) + if (!command) { + const first = buffer.indexOf(10) + const second = buffer.indexOf(10, first + 1) + if (first < 0 || second < 0) return + if (buffer.subarray(0, first).toString() !== `KSG2 ${token}`) return socket.end('ERR auth\n') + command = JSON.parse(buffer.subarray(first + 1, second).toString(), (key, value, context) => + key === 'generation' && typeof value === 'number' ? context.source : value) + buffer = buffer.subarray(second + 1) + requests.push(command) + if (command.op === 'prepare') { + commands.set(command.command_id, { ticket: `ticket-${command.command_id}`, command }) + return socket.end(JSON.stringify({ ok: true, ticket: `ticket-${command.command_id}` }) + '\n') + } + if (command.op === 'command') { + const held = commands.get(command.command_id) + if (!held || held.ticket !== command.ticket) return socket.end('{"ok":false,"error":"ticket_invalid"}\n') + const result = typeof options.status === 'function' ? options.status(command, held) : options.status + return socket.end(JSON.stringify(result || { + ok: true, + command: { id: command.command_id, fence: command.fence, status: 'completed', dispatched: true }, + }) + '\n') + } + if (command.op !== 'forward') return socket.end('ERR bad_request\n') + } + const end = buffer.indexOf('\r\n\r\n') + if (end < 0) return + const headers = buffer.subarray(0, end).toString() + const length = Number(/content-length: (\d+)/i.exec(headers)?.[1] || 0) + if (buffer.length < end + 4 + length) return + socket.removeAllListeners('data') + const envelope = JSON.parse(buffer.subarray(end + 4, end + 4 + length).toString()) + const held = commands.get(command.command_id) + if (!held || held.ticket !== command.ticket) return socket.end('ERR ticket_invalid\n') + held.envelope = envelope + options.onForward?.(command, envelope) + const payload = options.body || JSON.stringify(assistant) + if (options.truncated) return socket.end(`OK\nHTTP/1.1 200 OK\r\nContent-Length: ${Buffer.byteLength(payload) + 10}\r\n\r\n${payload}`) + socket.end(`OK\nHTTP/1.1 ${options.statusCode || 200} Result\r\nContent-Type: ${options.sse ? 'text/event-stream' : 'application/json'}\r\nContent-Length: ${Buffer.byteLength(payload)}\r\nConnection: close\r\n\r\n${payload}`) + }) + }) + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve) }) + t.after(async () => { + for (const socket of sockets) socket.destroy() + await new Promise((resolve) => server.close(resolve)) + fs.rmSync(dir, { recursive: true, force: true }) + }) + const config = { + trusted: true, pgBindingReady: true, nodeId: 'node-synthetic', vmId: 'vm-synthetic', + fence: syntheticFence(), + link: { address: '127.0.0.1', port: server.address().port, token, + certSha256: new X509Certificate(cert).fingerprint256.replaceAll(':', '').toLowerCase() }, + } + return { config, commands, requests } +} diff --git a/test/unit/slotgate-client.test.mjs b/test/unit/slotgate-client.test.mjs new file mode 100644 index 000000000..87d7c8b2f --- /dev/null +++ b/test/unit/slotgate-client.test.mjs @@ -0,0 +1,108 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { SlotgateClient } from '../../src/lib/transport/slotgate-client.mjs' +import { callGoWorker, streamGoWorker } from '../../src/lib/transport/go-worker-client.mjs' +import { dispatchCallInference } from '../../src/lib/transport/kernel-router.mjs' +import { assistant, fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' + +function execFor(config) { + return { vmId: config.vmId, slotgate: config, vm: { id: config.vmId, claude: {}, runtime: { kernel_socket: '/must-not-fallback.sock' } } } +} +function acknowledged(execution) { return { bound: true, requestId: execution.requestId, nodeBinding: execution.nodeBinding } } +function call(config, patch = {}) { + return callGoWorker({ exec: execFor(config), envelope: { body: {} }, onExecutionStart: async (value) => acknowledged(value), ...patch }) +} + +test('KSG2 snapshots the original Fence and preserves uint64 generation without rounding', async (t) => { + const fx = await fakeSlotgate(t) + fx.config.fence = syntheticFence('18446744073709551615') + const client = new SlotgateClient(fx.config) + fx.config.fence.generation = '2' + fx.config.fence.executor.container_id = 'b'.repeat(64) + const binding = client.nodeBinding + assert.equal(binding.fence.generation, '18446744073709551615') + const handle = await client.prepare('captured-job') + const status = await client.completion(handle) + assert.equal(status.fence.generation, '18446744073709551615') + assert.equal(fx.requests[0].fence.executor.container_id, 'a'.repeat(64)) +}) + +test('KSG2 requires explicit trust, PG readiness and exact immutable identity fields', async (t) => { + const fx = await fakeSlotgate(t) + for (const patch of [ + { trusted: false }, { pgBindingReady: false }, + { fence: { ...syntheticFence(), generation: Number.MAX_SAFE_INTEGER + 1 } }, + { fence: { ...syntheticFence(), generation: '01' } }, + { fence: { ...syntheticFence(), extra: true } }, + { fence: { ...syntheticFence(), executor: { ...syntheticFence().executor, pid: 123 } } }, + ]) assert.throws(() => new SlotgateClient({ ...fx.config, ...patch }), { code: 'slotgate_capability_invalid' }) + assert.equal(fx.requests.length, 0) +}) + +test('complete HTTP body remains uncertain until original-ticket node status is durably completed', async (t) => { + const fx = await fakeSlotgate(t) + let start + const result = await call(fx.config, { requestId: 'normal-job', onExecutionStart: async (value) => { + start = structuredClone(value) + assert.equal(fx.requests.length, 0, 'PG binding precedes node prepare and forward') + return acknowledged(value) + } }) + assert.equal(result.ok, true) + assert.equal(result.execution.ended, true) + assert.deepEqual(result.execution.nodeBinding, start.nodeBinding) + assert.equal(result.execution.nodeCompletion.status, 'completed') + assert.deepEqual(fx.requests.map((value) => value.op), ['prepare', 'forward', 'command']) + assert.equal(fx.commands.get('normal-job').envelope.request_id, 'normal-job') + assert.deepEqual(fx.requests[2].fence, fx.requests[0].fence) + assert.equal(fx.requests[2].ticket, fx.requests[1].ticket) +}) + +for (const mode of ['pending', 'unknown', 'mismatch', 'unavailable']) { + test(`HTTP success with ${mode} node proof retains execution and does not resend`, async (t) => { + const fx = await fakeSlotgate(t, { status: (command) => { + if (mode === 'unavailable') return { ok: false, error: 'authority_unavailable' } + return { ok: true, command: { id: command.command_id, fence: mode === 'mismatch' ? syntheticFence('2') : command.fence, + status: mode === 'pending' || mode === 'unknown' ? mode : 'completed', dispatched: true } } + } }) + const result = await call(fx.config) + assert.equal(result.execution.ended, false) + assert.equal(result.execution.nodeCompletion, undefined) + assert.equal(fx.requests.filter((value) => value.op === 'forward').length, 1) + }) +} + +test('a failed or malformed PG binding acknowledgement sends no node or fallback request', async (t) => { + const fx = await fakeSlotgate(t) + const error = new Error('synthetic bind failure') + await assert.rejects(call(fx.config, { onExecutionStart: async () => { throw error } }), (value) => value === error) + await assert.rejects(call(fx.config, { onExecutionStart: async () => ({ bound: true }) }), { code: 'slotgate_binding_unconfirmed' }) + assert.equal(fx.requests.length, 0) +}) + +test('truncated body never queries a successful completion or releases execution', async (t) => { + const fx = await fakeSlotgate(t, { truncated: true }) + const result = await call(fx.config) + assert.equal(result.execution.ended, false) + assert.equal(result.transportError, true) + assert.deepEqual(fx.requests.map((value) => value.op), ['prepare', 'forward']) +}) + +test('normal SSE requires the same durable node status as JSON', async (t) => { + const body = `event: message_start\ndata: ${JSON.stringify({ type: 'message_start', message: { ...assistant, content: [], stop_reason: null } })}\n\nevent: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}\n\nevent: message_stop\ndata: {"type":"message_stop"}\n\n` + const fx = await fakeSlotgate(t, { body, sse: true, status: { ok: false, error: 'authority_unavailable' } }) + const result = await streamGoWorker({ exec: execFor(fx.config), envelope: { body: {} }, onExecutionStart: async (value) => acknowledged(value) }) + assert.equal(result.execution.ended, false) + assert.deepEqual(fx.requests.map((value) => value.op), ['prepare', 'forward', 'command']) +}) + +test('router preserves durable PG bind acknowledgement for the fenced worker client', async (t) => { + const fx = await fakeSlotgate(t) + let completed + const result = await dispatchCallInference({ exec: execFor(fx.config), envelope: { body: {} }, + ensureRust: async () => ({ ok: true }), + onExecutionStart: async (value) => acknowledged(value), onExecutionEnd: async (value) => { completed = value }, + }) + assert.equal(result.execution.ended, true) + assert.equal(completed.nodeCompletion.status, 'completed') + assert.equal(fx.requests.filter((value) => value.op === 'forward').length, 1) +}) diff --git a/test/unit/slotgate-pg-binding.test.mjs b/test/unit/slotgate-pg-binding.test.mjs new file mode 100644 index 000000000..9de6b8337 --- /dev/null +++ b/test/unit/slotgate-pg-binding.test.mjs @@ -0,0 +1,67 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { createDatabase, getDb } from '../../src/lib/db/database.mjs' +import { callGoWorker } from '../../src/lib/transport/go-worker-client.mjs' +import { fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' + +const candidate = { accountId: 'synthetic-account', vmId: 'vm-synthetic', accountLimit: 1, vmLimit: 1, rpmLimit: 0, fableLimit: 0 } + +async function fixture(t, options) { + const gate = await fakeSlotgate(t, options) + const db = getDb() + const otherDb = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: db.schema }) + const first = new ExecutionAdmissionsRepo(db, { ownerId: 'synthetic-controller-a' }) + const second = new ExecutionAdmissionsRepo(otherDb, { ownerId: 'synthetic-controller-b' }) + const lease = await first.tryReserve(candidate) + assert.equal(lease.ok, true) + return { ...gate, db, second, lease, async run() { + const result = await callGoWorker({ exec: { vmId: candidate.vmId, slotgate: gate.config }, + requestId: 'synthetic-fenced-job', envelope: { body: {} }, onExecutionStart: lease.bindExecution }) + await lease.completeExecution(result.execution) + return result + } } +} + +test('two PG connections observe the original node binding before forwarding and completion before release', async (t) => { + const fx = await fixture(t) + const result = await fx.run() + const row = await fx.second.db.prepare('SELECT * FROM execution_jobs WHERE request_id = $1').get('synthetic-fenced-job') + assert.deepEqual(row.node_binding, result.execution.nodeBinding) + assert.deepEqual(row.node_completion, result.execution.nodeCompletion) + assert.equal(row.node_binding.fence.generation, '1') + assert.ok(row.ended_at) + assert.equal((await fx.lease.release()).released, true) + assert.equal((await fx.second.tryReserve(candidate)).ok, true) +}) + +test('an HTTP success without durable node status retains shared PG capacity', async (t) => { + const fx = await fixture(t, { status: { ok: false, error: 'authority_unavailable' } }) + const result = await fx.run() + assert.equal(result.execution.ended, false) + assert.equal((await fx.lease.release()).released, false) + assert.equal((await fx.second.tryReserve(candidate)).code, 'concurrency_limit') + const job = await fx.second.db.prepare('SELECT * FROM execution_jobs WHERE request_id = $1').get('synthetic-fenced-job') + assert.equal(job.ended_at, null) + assert.equal(job.node_completion, null) +}) + +test('PG completion refuses a local EOF or a different generation for a node-bound job', async (t) => { + const fx = await fixture(t) + const nodeBinding = { protocol: 'KSG2', nodeId: fx.config.nodeId, vmId: candidate.vmId, kind: 'kernel', fence: syntheticFence() } + await fx.lease.bindExecution({ requestId: 'synthetic-job', nodeBinding }) + await assert.rejects(fx.lease.completeExecution({ requestId: 'synthetic-job', started: true, ended: true, reason: 'eof' })) + await assert.rejects(fx.lease.completeExecution({ requestId: 'synthetic-job', started: true, ended: true, reason: 'node_completed', + nodeCompletion: { id: 'synthetic-job', fence: syntheticFence('2'), status: 'completed', dispatched: true } })) + assert.equal((await fx.lease.release()).released, false) + assert.equal((await fx.second.db.prepare('SELECT ended_at FROM execution_jobs WHERE request_id=$1').get('synthetic-job')).ended_at, null) +}) + +test('PG binding failure causes zero node prepare or forward operations', async (t) => { + const fx = await fixture(t) + await fx.db.exec(`CREATE FUNCTION reject_synthetic_binding() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic node binding failure' USING ERRCODE='P0001'; END; $$; + CREATE TRIGGER reject_synthetic_binding BEFORE INSERT ON execution_jobs FOR EACH ROW EXECUTE FUNCTION reject_synthetic_binding()`) + await assert.rejects(fx.run(), /synthetic node binding failure/) + assert.equal(fx.requests.length, 0) +}) From bf7615f8be9adf4f569c2a7d802be514a4c58e64 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:07:47 +0800 Subject: [PATCH 280/473] feat: coordinate circuit failures and half-open probes with shared admissions --- src/lib/admin/panel-api.mjs | 44 ++++++ .../015_execution_circuits.sql | 11 ++ .../db/repos/execution-admissions-repo.mjs | 21 ++- src/lib/db/repos/execution-circuits-repo.mjs | 128 ++++++++++++++++++ src/lib/pool/failover-runner.mjs | 7 +- src/lib/pool/pool-scheduler.mjs | 14 +- src/lib/pool/shared-execution-admissions.mjs | 8 ++ test/unit/backup-service.test.mjs | 7 +- test/unit/database.test.mjs | 7 +- test/unit/panel-api-proxy.test.mjs | 35 +++++ test/unit/shared-unit-circuit.test.mjs | 103 ++++++++++++++ 11 files changed, 368 insertions(+), 17 deletions(-) create mode 100644 src/lib/db/postgres-migrations/015_execution_circuits.sql create mode 100644 src/lib/db/repos/execution-circuits-repo.mjs diff --git a/src/lib/admin/panel-api.mjs b/src/lib/admin/panel-api.mjs index 4c9b71537..225be9b24 100644 --- a/src/lib/admin/panel-api.mjs +++ b/src/lib/admin/panel-api.mjs @@ -46,6 +46,9 @@ import { inferClaudeTier } from '../pool/claude-tier.mjs' import { listQuotaFromHeaders, isOfficialWindowLimited } from '../pool/quota-window.mjs' import { hardBlockOf } from '../pool/rate-limit-service.mjs' import { unitCircuit } from '../pool/unit-circuit.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { getDb } from '../db/database.mjs' +import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' import { accountIdOf, normalizePoolRouting } from '../pool/pool-scheduler.mjs' import { codexMaxSessions } from '../pool/codex-slot-pool.mjs' import { resolveCredentialScheduleLevel } from '../pool/credential-weight.mjs' @@ -360,6 +363,7 @@ export async function clearVmCooldown({ cfg, accountQuota, stickyRouter = null, /** Same key as PoolScheduler: full vm record (listVms summaries drop `claude`), slot identity first. */ function circuitViewFor(vm, projectRoot) { + if (isKubernetesMode()) return null try { const full = (projectRoot && vm?.id && getVm(projectRoot, vm.id)) || vm const accountId = accountIdOf(full, projectRoot || null) @@ -369,6 +373,28 @@ function circuitViewFor(vm, projectRoot) { } } +function sharedCircuitConfig(routingConfig = {}) { + const pool = normalizePoolRouting(routingConfig.pool) + return { failureThreshold: pool.circuit_failure_threshold, openMs: pool.circuit_open_ms } +} + +async function stampSharedCircuitViews(vms, projectRoot, routingConfig) { + if (!isKubernetesMode()) return + const circuits = new ExecutionAdmissionsRepo(getDb()).circuits + const config = sharedCircuitConfig(routingConfig) + const views = new Map() + for (const vm of vms) { + if (isCodexVm(vm)) continue + const accountId = accountIdOf(getVm(projectRoot, vm.id) || vm, projectRoot) + if (!accountId) continue + if (!views.has(accountId)) { + const { state, failures, threshold, open_until, open_ms } = await circuits.inspect(accountId, config) + views.set(accountId, { account_id: accountId, state, failures, threshold, open_until, open_ms }) + } + vm.circuit = views.get(accountId) + } +} + /** POST /vms/:id/circuit/reset — operator closes a tripped Claude unit circuit. */ export function resetVmCircuit({ cfg, poolScheduler = null, id } = {}) { const vm = getVm(cfg?.paths?.project, id) @@ -383,6 +409,20 @@ export function resetVmCircuit({ cfg, poolScheduler = null, id } = {}) { ) } const accountId = accountIdOf(vm, cfg.paths.project) + if (isKubernetesMode()) { + return (async () => { + if (accountId) { + const circuits = new ExecutionAdmissionsRepo(getDb()).circuits + await circuits.reset(accountId, poolScheduler?.circuitConfig?.() || sharedCircuitConfig()) + } + const item = { ...vm } + await stampSharedCircuitViews([item], cfg.paths.project, { + pool: poolScheduler?.config || {}, + }) + poolScheduler?.notifyCapacity?.(accountId) + return ok({ id: vm.id, circuit: item.circuit || null }) + })() + } if (accountId) unitCircuit.reset(accountId) try { poolScheduler?.notifyCapacity?.(accountId) @@ -607,6 +647,7 @@ export async function buildDashboard({ }), ) const snap = accountQuota.snapshot() + await stampSharedCircuitViews(vms, cfg.paths.project, routingConfig) const accounts = liveAccounts(snap.accounts, listed) const peak5 = Math.max(0, ...accounts.map((a) => Number(a.unified?.['5h']?.utilization || 0)), 0) const peak7 = Math.max(0, ...accounts.map((a) => Number(a.unified?.['7d']?.utilization || 0)), 0) @@ -736,6 +777,7 @@ export async function buildVmList({ role, }), ) + await stampSharedCircuitViews(vms, cfg.paths.project, routingConfig) const accounts = (() => { try { return accountQuota?.snapshot?.().accounts || [] @@ -803,6 +845,7 @@ export async function buildVmDetail({ clusterNodes, role, }) + await stampSharedCircuitViews([summary], cfg.paths.project, routingConfig) const acc = findAccount(accountQuota, summary) const billing = await (async () => { try { @@ -1494,6 +1537,7 @@ export async function snapshotAccountPool({ projectRoot: cfg.paths.project, }), ) + await stampSharedCircuitViews(vms, cfg.paths.project, routingConfig) const accounts = (() => { try { return accountQuota?.snapshot?.().accounts || [] diff --git a/src/lib/db/postgres-migrations/015_execution_circuits.sql b/src/lib/db/postgres-migrations/015_execution_circuits.sql new file mode 100644 index 000000000..8d71f877c --- /dev/null +++ b/src/lib/db/postgres-migrations/015_execution_circuits.sql @@ -0,0 +1,11 @@ +CREATE TABLE execution_circuits ( + account_id text COLLATE "C" PRIMARY KEY, + failures integer NOT NULL DEFAULT 0 CHECK (failures >= 0), + open_until timestamptz, + revision bigint NOT NULL DEFAULT 0 CHECK (revision >= 0), + probe_admission_id text COLLATE "C" +); + +ALTER TABLE execution_admissions ADD COLUMN circuit_revision bigint; +ALTER TABLE execution_admissions ADD COLUMN circuit_outcome text + CHECK (circuit_outcome IN ('failure', 'success', 'neutral')); diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 4e8a16f65..4e03274e5 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -2,6 +2,7 @@ import crypto from 'node:crypto' import { withTransaction } from '../database.mjs' import { physicalContainerName, physicalVmAliases } from '../../vm/vm-physical-target.mjs' import { readWatchdogState } from '../../transport/kernel-watchdog-state.mjs' +import { ExecutionCircuitsRepo } from './execution-circuits-repo.mjs' function invalid(field) { return Object.assign(new Error(`Invalid execution admission ${field}`), { code: 'invalid_execution_admission' }) @@ -31,6 +32,7 @@ export class ExecutionAdmissionsRepo { this.db = db this.ownerId = identifier(ownerId, 'ownerId') this.routingRevisionCheck = assertRoutingRevision + this.circuits = new ExecutionCircuitsRepo(this) } async assertRoutingRevision(revision) { @@ -142,10 +144,13 @@ export class ExecutionAdmissionsRepo { }) } const admissionId = crypto.randomUUID() + const circuit = + input.circuit && !input.pinned ? await this.circuits.admit(accountId, admissionId, now, input.circuit) : null + if (circuit && !circuit.ok) return circuit const record = await this.db - .prepare(`INSERT INTO execution_admissions (id, owner_id, request_id, account_id, vm_id, model_family, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING *`) - .get(admissionId, ownerId, requestId, accountId, vmId, family, now) + .prepare(`INSERT INTO execution_admissions (id, owner_id, request_id, account_id, vm_id, model_family, created_at, circuit_revision) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *`) + .get(admissionId, ownerId, requestId, accountId, vmId, family, now, circuit?.revision ?? null) const handle = { admissionId, ownerId } return { ok: true, @@ -159,6 +164,15 @@ export class ExecutionAdmissionsRepo { markUncertain: () => this.markUncertain(handle), bindExecution: (execution) => this.bindExecution(handle, execution), completeExecution: (execution) => this.completeExecution(handle, execution), + noteCircuitResult: async (outcome) => { + if (!input.circuit || input.pinned) return { updated: false } + try { + return await this.circuits.noteResult(handle, outcome, input.circuit) + } catch (error) { + await this.markUncertain(handle) + throw error + } + }, } }) } @@ -251,6 +265,7 @@ export class ExecutionAdmissionsRepo { WHERE id = $1 AND owner_id = $2 AND released_at IS NULL AND uncertain_at IS NULL AND NOT EXISTS (SELECT 1 FROM execution_jobs j WHERE j.admission_id = execution_admissions.id AND j.ended_at IS NULL)`) .run(handle.admissionId, handle.ownerId) + if (result.changes > 0) await this.circuits.releaseProbe(handle.admissionId) return { released: result.changes > 0 } }) } diff --git a/src/lib/db/repos/execution-circuits-repo.mjs b/src/lib/db/repos/execution-circuits-repo.mjs new file mode 100644 index 000000000..e2a6f7462 --- /dev/null +++ b/src/lib/db/repos/execution-circuits-repo.mjs @@ -0,0 +1,128 @@ +import { withTransaction } from '../database.mjs' + +function configOf(config) { + if ( + !Number.isSafeInteger(config?.failureThreshold) || + config.failureThreshold < 1 || + !Number.isSafeInteger(config?.openMs) || + config.openMs < 1 + ) { + throw Object.assign(new Error('Invalid execution circuit configuration'), { code: 'invalid_execution_circuit' }) + } + return config +} + +function view(row, now, config) { + const state = row?.open_until ? (row.open_until > now ? 'open' : 'half_open') : 'closed' + return { + state, + failures: row?.failures || 0, + threshold: config.failureThreshold, + open_until: state === 'open' ? row.open_until.getTime() : null, + open_ms: config.openMs, + probeHeld: !!row?.probe_admission_id, + } +} + +/** Circuit writes share the account admission lock; probes belong to durable admissions. */ +export class ExecutionCircuitsRepo { + constructor(executions) { + this.executions = executions + this.db = executions.db + } + + async inspect(accountId, config) { + configOf(config) + const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() + const row = await this.db.prepare('SELECT * FROM execution_circuits WHERE account_id = $1').get(accountId) + const current = view(row, now, config) + if (current.state === 'open') return { ...current, ok: false, reason: 'circuit_open', until: current.open_until } + if (current.probeHeld) + return { ...current, ok: false, reason: 'circuit_probe', until: now.getTime() + config.openMs } + return { ...current, ok: true } + } + + async admit(accountId, admissionId, now, config) { + configOf(config) + await this.db + .prepare('INSERT INTO execution_circuits (account_id) VALUES ($1) ON CONFLICT DO NOTHING') + .run(accountId) + const row = await this.db + .prepare('SELECT * FROM execution_circuits WHERE account_id = $1 FOR UPDATE') + .get(accountId) + if (row.open_until && row.open_until > now) + return { ok: false, code: 'circuit_open', retry_after_ms: Math.max(1, row.open_until - now) } + if (row.probe_admission_id) return { ok: false, code: 'circuit_probe' } + if (row.open_until) { + await this.db + .prepare('UPDATE execution_circuits SET probe_admission_id = $2 WHERE account_id = $1') + .run(accountId, admissionId) + } + return { ok: true, revision: row.revision } + } + + async noteResult(handle, outcome, config) { + configOf(config) + if (!['failure', 'success', 'neutral'].includes(outcome)) + throw Object.assign(new Error('Invalid execution circuit outcome'), { code: 'invalid_execution_circuit' }) + return withTransaction(this.db, async () => { + const before = await this.db + .prepare('SELECT account_id FROM execution_admissions WHERE id = $1 AND owner_id = $2') + .get(handle.admissionId, handle.ownerId) + if (!before) return { updated: false } + await this.executions.lockResources({ accountId: before.account_id }) + const admission = await this.executions._lockAdmission(handle) + if (!admission || admission.released_at || admission.circuit_revision == null || admission.circuit_outcome) + return { updated: false } + await this.db + .prepare('UPDATE execution_admissions SET circuit_outcome = $2 WHERE id = $1') + .run(handle.admissionId, outcome) + const row = await this.db + .prepare('SELECT * FROM execution_circuits WHERE account_id = $1 FOR UPDATE') + .get(admission.account_id) + if (!row || row.revision !== admission.circuit_revision || admission.uncertain_at) return { updated: false } + // An older closed-state request cannot finish a newer half-open probe. + if (row.probe_admission_id && row.probe_admission_id !== admission.id) return { updated: false } + if (outcome === 'failure') { + const failures = row.failures + 1 + if (row.probe_admission_id || failures >= config.failureThreshold) { + await this.db + .prepare(`UPDATE execution_circuits SET failures = $2, + open_until = clock_timestamp() + $3::integer * interval '1 millisecond', + revision = revision + 1, probe_admission_id = NULL WHERE account_id = $1`) + .run(admission.account_id, config.failureThreshold, config.openMs) + } else { + await this.db + .prepare('UPDATE execution_circuits SET failures = $2 WHERE account_id = $1') + .run(admission.account_id, failures) + } + } else if (outcome === 'success') { + await this.db + .prepare(`UPDATE execution_circuits SET failures = 0, open_until = NULL, + probe_admission_id = NULL, revision = revision + CASE WHEN open_until IS NULL THEN 0 ELSE 1 END + WHERE account_id = $1`) + .run(admission.account_id) + } + return { updated: true } + }) + } + + async releaseProbe(admissionId) { + await this.db + .prepare('UPDATE execution_circuits SET probe_admission_id = NULL WHERE probe_admission_id = $1') + .run(admissionId) + } + + async reset(accountId, config) { + configOf(config) + return withTransaction(this.db, async () => { + await this.executions.lockResources({ accountId }) + await this.db + .prepare(`INSERT INTO execution_circuits (account_id, revision) VALUES ($1, 1) + ON CONFLICT (account_id) DO UPDATE SET failures = 0, open_until = NULL, + probe_admission_id = NULL, revision = execution_circuits.revision + 1`) + .run(accountId) + return this.inspect(accountId, config) + }) + } +} diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 2e8d18c60..4477e7cfd 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -476,6 +476,9 @@ export class FailoverRunner { } noteUnitHealth(selected, policy, result) { + if (this.scheduler?.shared) { + return selected.noteCircuitResult?.(policy?.circuit ? 'failure' : verifiedSuccess(result) ? 'success' : 'neutral') + } const circuit = this.scheduler?.unitCircuit if (!circuit || !selected?.accountId) return if (policy?.circuit) circuit.recordFailure(selected.accountId) @@ -1022,7 +1025,7 @@ export class FailoverRunner { lastResult = result lastPolicy = policy - this.noteUnitHealth(selected, policy, result) + await this.noteUnitHealth(selected, policy, result) await notifyProxyFailure(this.onProxyFailure, selected, policy) if (policy.reason === 'fable_plan_denied' && typeof this.onFablePlanDenied === 'function') { try { @@ -1214,7 +1217,7 @@ export class FailoverRunner { ) lastResult = result lastPolicy = policy - this.noteUnitHealth(selected, policy, result) + await this.noteUnitHealth(selected, policy, result) await notifyProxyFailure(this.onProxyFailure, selected, policy) await this.attemptsRepo?.complete?.(requestId, attemptNo, { upstreamStatus: 0, diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 02b370f22..8e51cbd6b 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -1127,7 +1127,9 @@ export class PoolScheduler extends EventEmitter { if (hardBlock) return { ok: false, reason: hardBlock.reason, until: hardBlock.until } let circuitProbeUntil = null if (!pinned) { - const circuit = this.unitCircuit?.inspect?.(accountId, now) + const circuit = this.shared + ? await this.executionAdmissions.inspectCircuit?.(accountId, this.circuitConfig()) + : this.unitCircuit?.inspect?.(accountId, now) if (circuit?.reason === 'circuit_open') return { ok: false, reason: circuit.reason, until: circuit.until } if (circuit?.reason === 'circuit_probe') circuitProbeUntil = circuit.until } @@ -1691,6 +1693,10 @@ export class PoolScheduler extends EventEmitter { } } + circuitConfig() { + return { failureThreshold: this.config.circuit_failure_threshold, openMs: this.config.circuit_open_ms } + } + async reserveShared(candidate, options) { if (!options.skipQuota) { const gate = this.accountQuota?.canAccept?.(candidate.accountId, { @@ -1700,8 +1706,6 @@ export class PoolScheduler extends EventEmitter { if (gate && !gate.ok && !['concurrency_limit', 'rpm_limit', 'quota_band_inflight'].includes(gate.reason)) return null } - const circuitHold = options.pinned ? null : this.unitCircuit?.admit?.(candidate.accountId) - if (circuitHold && !circuitHold.ok) return null const limits = this.executionLimits(candidate) let lease try { @@ -1710,10 +1714,10 @@ export class PoolScheduler extends EventEmitter { vmId: candidate.vmId, ...limits, routingRevision: candidate.routingRevision, + circuit: this.circuitConfig(), ...options, }) } catch (error) { - if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) if (error?.code === 'routing_config_stale' && this.routingState) { await this.routingState.refresh() return null @@ -1721,7 +1725,6 @@ export class PoolScheduler extends EventEmitter { throw error } if (!lease.ok) { - if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) candidate.busy = true candidate.waitReason = lease.code if (lease.retry_after_ms) candidate.availableAt = Date.now() + lease.retry_after_ms @@ -1770,7 +1773,6 @@ export class PoolScheduler extends EventEmitter { else this.inflight.delete(candidate.accountId) this.bumpFamily(candidate.accountId, limits.family, -1) this.bumpVmInflight(candidate.vmId, -1) - if (circuitHold?.probe) this.unitCircuit?.releaseProbe?.(candidate.accountId) this.planner.pump(candidate.vmId) this.notifyCapacity(candidate.accountId) return result diff --git a/src/lib/pool/shared-execution-admissions.mjs b/src/lib/pool/shared-execution-admissions.mjs index 596ee633c..02ebe6c02 100644 --- a/src/lib/pool/shared-execution-admissions.mjs +++ b/src/lib/pool/shared-execution-admissions.mjs @@ -49,6 +49,14 @@ export class SharedExecutionAdmissions { return this.executions.getCounts({ accountId, vmId }) } + inspectCircuit(accountId, config) { + return this.executions.circuits.inspect(accountId, config) + } + + resetCircuit(accountId, config) { + return this.executions.circuits.reset(accountId, config) + } + async tryReserve(input) { const { seatKey } = input if (seatKey == null) return this.executions.tryReserve(input) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 6a5c992bc..b7fb9c719 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,7 +79,7 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; @@ -102,6 +102,7 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'api_key_admissions', 'gateway_rate_buckets', 'execution_admissions', + 'execution_circuits', 'execution_seats', 'execution_jobs', 'session_turns', @@ -112,13 +113,13 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 14) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 15) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets; + `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index 518376e24..5892f9be2 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 41) + assert.equal(rows.length, 42) for (const name of [ 'settings', 'users', @@ -49,6 +49,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'api_key_admissions', 'gateway_rate_buckets', 'execution_admissions', + 'execution_circuits', 'execution_seats', 'execution_jobs', 'session_turns', @@ -116,7 +117,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 14) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 15) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -132,7 +133,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 14) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 15) fs.rmSync(dir, { recursive: true, force: true }) }) diff --git a/test/unit/panel-api-proxy.test.mjs b/test/unit/panel-api-proxy.test.mjs index f02bb2cca..9fa633700 100644 --- a/test/unit/panel-api-proxy.test.mjs +++ b/test/unit/panel-api-proxy.test.mjs @@ -322,6 +322,41 @@ test('VM list exposes the Claude unit circuit and reset closes it', async (t) => assert.equal(resetVmCircuit({ cfg, id: 'nope' }).status, 404) }) +test('Kubernetes VM list and reset use the shared circuit authority', async (t) => { + const { resetVmCircuit } = await import('../../src/lib/admin/panel-api.mjs') + const { getDb } = await import('../../src/lib/db/database.mjs') + const { SharedExecutionAdmissions } = await import('../../src/lib/pool/shared-execution-admissions.mjs') + const project = tmpDir() + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(() => { + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + fs.rmSync(project, { recursive: true, force: true }) + }) + writeVm(project, { id: 'vm-shared-circuit', claude: { account_uuid: 'shared-circuit-account' } }) + const admissions = new SharedExecutionAdmissions(getDb()) + const circuit = { failureThreshold: 1, openMs: 1000 } + const lease = await admissions.tryReserve({ + accountId: 'shared-circuit-account', + vmId: 'vm-shared-circuit', + accountLimit: 4, + vmLimit: 4, + fableLimit: 0, + rpmLimit: 0, + circuit, + }) + await lease.noteCircuitResult('failure') + await lease.release() + const cfg = { paths: { project } } + const listed = await buildVmList({ cfg, accountQuota: fakeQuota() }) + assert.equal(listed.data.items[0].circuit.state, 'open') + assert.equal(listed.data.items[0].circuit.account_id, 'shared-circuit-account') + const reset = await resetVmCircuit({ cfg, id: 'vm-shared-circuit' }) + assert.equal(reset.data.circuit.state, 'closed') + assert.equal((await admissions.inspectCircuit('shared-circuit-account', circuit)).failures, 0) +}) + test('local proxy node labels are present for admin and absent for tenant', async (t) => { const project = tmpDir() const dataDir = path.join(project, 'data') diff --git a/test/unit/shared-unit-circuit.test.mjs b/test/unit/shared-unit-circuit.test.mjs index b56977b35..739319018 100644 --- a/test/unit/shared-unit-circuit.test.mjs +++ b/test/unit/shared-unit-circuit.test.mjs @@ -16,6 +16,18 @@ const candidate = { maxConcurrency: 8, } +const success = { + ok: true, + terminalState: 'verified', + body: { + type: 'message', + id: 'msg-test', + role: 'assistant', + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + }, +} + async function replicas(threshold = 1) { const firstDb = getDb() const secondDb = await createRepositoryTestDatabase({ @@ -83,3 +95,94 @@ test('failure streaks combine across control planes before the threshold', () => await next?.release() } })) + +async function openCircuit(peer) { + const lease = await peer.pool.reserve(candidate) + await peer.runner.noteUnitHealth({ ...candidate, ...lease }, { circuit: true }, { ok: false }) + await lease.release() + await peer.db.exec("UPDATE execution_circuits SET open_until = clock_timestamp() - interval '1 second'") +} + +test('an expired shared circuit admits exactly one probe and unknown work never expires', () => + kubernetes(async () => { + const peers = await replicas() + await openCircuit(peers[0]) + const leases = await Promise.all(Array.from({ length: 12 }, (_, index) => peers[index % 2].pool.reserve(candidate))) + assert.equal(leases.filter(Boolean).length, 1) + const probe = leases.find(Boolean) + await probe.markUncertain() + await peers[0].db.exec("UPDATE execution_admissions SET created_at = clock_timestamp() - interval '1 year'") + assert.notEqual((await probe.release()).released, true) + assert.equal(await peers[1].pool.reserve(candidate), null) + assert.equal( + (await peers[1].pool.executionAdmissions.inspectCircuit(candidate.accountId, peers[1].pool.circuitConfig())) + .reason, + 'circuit_probe', + ) + })) + +test('only the current half-open probe can close the shared circuit', () => + kubernetes(async () => { + const [first, second] = await replicas() + const older = await first.pool.reserve(candidate) + await openCircuit(second) + const probe = await second.pool.reserve(candidate) + assert.ok(probe) + await first.runner.noteUnitHealth({ ...candidate, ...older }, {}, success) + assert.equal(await first.pool.reserve(candidate), null, 'old success cannot complete the current probe') + await second.runner.noteUnitHealth({ ...candidate, ...probe }, {}, success) + assert.equal( + (await first.pool.executionAdmissions.inspectCircuit(candidate.accountId, first.pool.circuitConfig())).state, + 'closed', + ) + await older.release() + await probe.release() + const next = await first.pool.reserve(candidate) + assert.ok(next) + await next.release() + })) + +test('a half-open probe without submitted jobs is returned only by its own release', () => + kubernetes(async () => { + const [first, second] = await replicas() + await openCircuit(first) + const probe = await first.pool.reserve(candidate) + const otherOwner = { ...probe, ownerId: 'different-controller' } + assert.equal((await second.pool.executionAdmissions.executions.release(otherOwner)).released, false) + assert.equal(await second.pool.reserve(candidate), null) + assert.equal((await probe.release()).released, true) + const retry = await second.pool.reserve(candidate) + assert.ok(retry) + await retry.release() + })) + +test('duplicate outcomes count once and an operator reset invalidates old outcomes', () => + kubernetes(async () => { + const [first, second] = await replicas(3) + const lease = await first.pool.reserve(candidate) + await lease.noteCircuitResult('failure') + await lease.noteCircuitResult('failure') + let state = await second.pool.executionAdmissions.inspectCircuit(candidate.accountId, second.pool.circuitConfig()) + assert.equal(state.failures, 1) + const older = await second.pool.reserve(candidate) + await first.pool.executionAdmissions.resetCircuit(candidate.accountId, first.pool.circuitConfig()) + await older.noteCircuitResult('failure') + state = await first.pool.executionAdmissions.inspectCircuit(candidate.accountId, first.pool.circuitConfig()) + assert.equal(state.failures, 0) + await lease.release() + await older.release() + })) + +test('a failed circuit write preserves its admission and probe', () => + kubernetes(async () => { + const [first, second] = await replicas() + await openCircuit(first) + const probe = await first.pool.reserve(candidate) + await first.db.exec(`CREATE FUNCTION reject_circuit_write() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic circuit write failure'; END $$; + CREATE TRIGGER reject_circuit_write BEFORE UPDATE ON execution_circuits + FOR EACH ROW EXECUTE FUNCTION reject_circuit_write()`) + await assert.rejects(probe.noteCircuitResult('success'), { code: 'P0001' }) + assert.equal((await probe.release()).released, false) + assert.equal(await second.pool.reserve(candidate), null) + })) From 66a580de17670a954638fbfb46fee25740adab78 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:09:12 +0800 Subject: [PATCH 281/473] fix: return durable execution binding acknowledgement to transports --- src/lib/protocol/handle-protocol.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 40c0d4a74..175a0eb34 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -513,7 +513,7 @@ export function createHandleProtocol(deps) { return { onExecutionStart: async (execution) => { await observeExecution(req, admission, { ...execution, started: true, ended: false }) - await candidate.bindExecution?.(execution) + return await candidate.bindExecution?.(execution) }, onExecutionEnd: async (execution) => { await candidate.completeExecution?.(execution) From 4371c581effbb5f5e19bd7a9952fe1475e7e3fb5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:13:18 +0800 Subject: [PATCH 282/473] fix: serialize observed credential refresh attempts across controllers --- src/lib/core/shared-background-tasks.mjs | 5 +- .../016_shared_refresh_observations.sql | 7 ++ .../db/repos/background-task-runs-repo.mjs | 25 ++++++- src/lib/vm/shared-vm-operations.mjs | 67 ++++++++++++++++--- 4 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 src/lib/db/postgres-migrations/016_shared_refresh_observations.sql diff --git a/src/lib/core/shared-background-tasks.mjs b/src/lib/core/shared-background-tasks.mjs index 1586bda0f..605a12f9a 100644 --- a/src/lib/core/shared-background-tasks.mjs +++ b/src/lib/core/shared-background-tasks.mjs @@ -105,7 +105,10 @@ export class SharedBackgroundTasks { } run(plan, callback) { - const pending = this._run(plan, callback) + return this.track(this._run(plan, callback)) + } + + track(pending) { this.pending.add(pending) pending.then( () => this.pending.delete(pending), diff --git a/src/lib/db/postgres-migrations/016_shared_refresh_observations.sql b/src/lib/db/postgres-migrations/016_shared_refresh_observations.sql new file mode 100644 index 000000000..9d48490e5 --- /dev/null +++ b/src/lib/db/postgres-migrations/016_shared_refresh_observations.sql @@ -0,0 +1,7 @@ +-- Stable ordering for observing a refresh attempt across controller processes. +-- The identity sequence belongs to the existing table and follows schema backups. +ALTER TABLE background_task_runs + ADD COLUMN claim_sequence bigint GENERATED ALWAYS AS IDENTITY; + +CREATE INDEX background_task_runs_latest + ON background_task_runs (task_key, claim_sequence DESC); diff --git a/src/lib/db/repos/background-task-runs-repo.mjs b/src/lib/db/repos/background-task-runs-repo.mjs index 8911530af..c390bc92f 100644 --- a/src/lib/db/repos/background-task-runs-repo.mjs +++ b/src/lib/db/repos/background-task-runs-repo.mjs @@ -18,9 +18,27 @@ export class BackgroundTaskRunsRepo { return (await this.db.prepare('SELECT extract(epoch FROM clock_timestamp()) * 1000 AS now_ms').get()).now_ms } - async tryClaim({ taskKey, executionId, intervalMs, ownerId }) { + async latest(taskKey) { + identifier(taskKey, 512) + const row = await this.db + .prepare(`SELECT execution_id,owner_id,finished_at,unknown_at,shared_result + FROM background_task_runs WHERE task_key=$1 ORDER BY claim_sequence DESC LIMIT 1`) + .get(taskKey) + return row + ? { + executionId: row.execution_id, + ownerId: row.owner_id, + finishedAt: row.finished_at, + unknownAt: row.unknown_at, + sharedResult: row.shared_result, + } + : null + } + + async tryClaim({ taskKey, executionId, intervalMs, ownerId, ifPreviousExecutionId }) { identifier(taskKey, 512) identifier(ownerId, 200) + if (ifPreviousExecutionId != null) identifier(ifPreviousExecutionId, 1024) if (executionId == null) { if (!Number.isSafeInteger(intervalMs) || intervalMs <= 0) throw invalidPlan() } else { @@ -37,6 +55,11 @@ export class BackgroundTaskRunsRepo { `vm2api:background:${this.db.schema}`, taskKey, ]) + if (ifPreviousExecutionId !== undefined) { + const latest = await this.latest(taskKey) + if ((latest?.executionId ?? null) !== ifPreviousExecutionId) + return { ok: false, code: 'background_task_changed', latest } + } const active = await this.db .prepare('SELECT execution_id FROM background_task_runs WHERE task_key = $1 AND finished_at IS NULL') .get(taskKey) diff --git a/src/lib/vm/shared-vm-operations.mjs b/src/lib/vm/shared-vm-operations.mjs index 2805decf8..1d2064021 100644 --- a/src/lib/vm/shared-vm-operations.mjs +++ b/src/lib/vm/shared-vm-operations.mjs @@ -45,21 +45,55 @@ export class SharedVmOperations { this.executions = new ExecutionAdmissionsRepo(db, { ownerId: this.ownerId }) } - async tryAcquire({ vmId, kind = 'mutation' } = {}) { + async _executionException({ vmId, accountId, admission }) { + if (!admission) return { ok: true, admissionId: null } + if ( + !admission.db || + admission.db.schema !== this.db.schema || + admission.db.connectionString !== this.db.connectionString || + admission.vmId !== vmId || + admission.accountId !== accountId || + !admission.admissionId || + !admission.ownerId + ) + return { ok: false, code: 'refresh_admission_invalid' } + const row = await this.db + .prepare(`SELECT * FROM execution_admissions WHERE id=$1 AND owner_id=$2 FOR UPDATE`) + .get(admission.admissionId, admission.ownerId) + if (!row || row.vm_id !== vmId || row.account_id !== accountId || row.released_at || row.uncertain_at) + return { ok: false, code: 'refresh_admission_invalid' } + const jobs = await this.db + .prepare(`SELECT count(*) AS total, + count(*) FILTER (WHERE ended_at IS NULL) AS pending FROM execution_jobs WHERE admission_id=$1`) + .get(row.id) + if (!jobs.total || jobs.pending) return { ok: false, code: 'refresh_execution_unfinished' } + return { ok: true, admissionId: row.id } + } + + async _otherExecution(vmId, accountId, admissionId) { + return this.db + .prepare(`SELECT id FROM execution_admissions + WHERE released_at IS NULL AND (vm_id=ANY($1::text[]) OR ($2::text IS NOT NULL AND account_id=$2)) + AND ($3::text IS NULL OR id<>$3) LIMIT 1`) + .get(physicalVmAliases(vmId), accountId ?? null, admissionId) + } + + async tryAcquire({ vmId, kind = 'mutation', accountId = null, admission = null } = {}) { validateOperationVmId(vmId) identifier(kind) + if (accountId != null) identifier(accountId) // The claim must commit independently before its owner can perform remote effects. if (this.db.isTransaction) throw operationError('vm_operation_transaction_active', 500) return withTransaction(this.db, async () => { // Identical namespace and VM resource to ExecutionAdmissionsRepo.tryReserve. - await this.executions.lockVm(vmId) + await this.executions.lockResources({ accountId, vmIds: [vmId] }) const active = await this.db .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') .get(physicalVmAliases(vmId)) if (active) return { ok: false, code: 'vm_operation_active' } - const execution = await this.db - .prepare('SELECT id FROM execution_admissions WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') - .get(physicalVmAliases(vmId)) + const exception = await this._executionException({ vmId, accountId, admission }) + if (!exception.ok) return exception + const execution = await this._otherExecution(vmId, accountId, exception.admissionId) if (execution) return { ok: false, code: 'vm_execution_active' } const operationId = crypto.randomUUID() await this.db @@ -135,7 +169,7 @@ export class SharedVmOperations { return { released: result.changes > 0 } } - async run(vmId, callback, { kind = 'mutation' } = {}) { + async run(vmId, callback, { kind = 'mutation', accountId = null, admission = null } = {}) { validateOperationVmId(vmId) const parents = callbacks.getStore() || [] const nested = parents.findLast( @@ -145,8 +179,25 @@ export class SharedVmOperations { context.db.schema === this.db.schema && context.db.connectionString === this.db.connectionString, ) - if (nested) return callback(nested.operation) - const operation = await this.tryAcquire({ vmId, kind }) + if (nested) { + if (nested.operation.uncertain) throw operationError('vm_operation_inactive') + const valid = await withTransaction(this.db, async () => { + await this.executions.lockResources({ accountId, vmIds: [vmId] }) + const current = await this.db + .prepare(`SELECT id FROM vm_operations + WHERE id=$1 AND owner_id=$2 AND vm_id=$3 AND released_at IS NULL AND uncertain_at IS NULL`) + .get(nested.operation.operationId, nested.operation.ownerId, vmId) + if (!current) return { ok: false, code: 'vm_operation_inactive' } + const exception = await this._executionException({ vmId, accountId, admission }) + if (!exception.ok) return exception + return (await this._otherExecution(vmId, accountId, exception.admissionId)) + ? { ok: false, code: 'vm_execution_active' } + : { ok: true } + }) + if (!valid.ok) throw operationError(valid.code) + return callback(nested.operation) + } + const operation = await this.tryAcquire({ vmId, kind, accountId, admission }) if (!operation.ok) throw operationError(operation.code) const context = { db: this.db, vmId, operation, active: true } let result From c8b2d9de7fcd14fed1a4642f0f0c67e65669d8d2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:14:37 +0800 Subject: [PATCH 283/473] feat: coordinate credential refresh and publication through PostgreSQL claims --- src/lib/oauth/shared-credential-refresh.mjs | 172 ++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 src/lib/oauth/shared-credential-refresh.mjs diff --git a/src/lib/oauth/shared-credential-refresh.mjs b/src/lib/oauth/shared-credential-refresh.mjs new file mode 100644 index 000000000..af1b7d290 --- /dev/null +++ b/src/lib/oauth/shared-credential-refresh.mjs @@ -0,0 +1,172 @@ +import { createHash, randomUUID } from 'node:crypto' +import { getDb, getActivityCoordinator, withTransaction } from '../db/database.mjs' +import { getSharedBackgroundTasks } from '../core/shared-background-tasks.mjs' +import { SharedVmOperations, currentVmOperationIds } from '../vm/shared-vm-operations.mjs' + +const services = new WeakMap() + +function failure(code) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code, status: 409 }) +} + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical) + if (value && typeof value === 'object') + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonical(value[key])]), + ) + return value +} + +/** An opaque observation of a credential; tokens are never written to task results. */ +export function credentialGeneration(credential) { + if (!credential || typeof credential !== 'object') throw failure('credential_generation_required') + return createHash('sha256') + .update(JSON.stringify(canonical(credential))) + .digest('hex') +} + +function checkCurrent(current) { + if (typeof current?.generation !== 'string' || !current.generation || !current.credential) + throw failure('credential_generation_required') + return current +} + +/** A refresh claim remains occupied until publication and its terminal SQL are confirmed. */ +export class SharedCredentialRefresh { + constructor(db, { ownerId } = {}) { + this.db = db + this.tasks = getSharedBackgroundTasks(db) + this.mutations = new SharedVmOperations(db, ownerId ? { ownerId } : undefined) + } + + run(input, callbacks) { + return this.tasks.track(this._activity(input, callbacks)) + } + + async _activity(input, callbacks) { + if (typeof input?.expectedGeneration !== 'string' || !input.expectedGeneration) + throw failure('credential_generation_required') + if (!['claude', 'codex'].includes(input.platform) || !input.vmId || !input.accountId) + throw failure('invalid_credential_refresh') + const activity = await getActivityCoordinator(this.db)?.admit({ kind: 'credential_refresh' }) + const state = { unknown: false } + try { + const invoke = () => this._run(input, callbacks, state) + return await (activity ? activity.run(invoke) : invoke()) + } finally { + if (state.unknown) await activity?.markUncertain('credential_refresh_unknown') + else await activity?.complete({ ended: true, reason: 'credential_refresh_completed' }) + } + } + + async _joined(taskKey, latest, { readCurrent }) { + // Waiting never changes ownership or treats elapsed time as execution completion. + for (let count = 0; latest && !latest.finishedAt && !latest.unknownAt && count < 200; count++) { + await new Promise((resolve) => setTimeout(resolve, 25)) + latest = await this.tasks.repo.latest(taskKey) + } + if (!latest?.finishedAt) return { finished: false, unknown: true, shared: true } + const current = checkCurrent(await readCurrent()) + const meta = latest.sharedResult + if (meta?.ok && current.generation !== meta.generation) throw failure('credential_projection_stale') + return { + finished: true, + unknown: false, + shared: true, + current, + value: { ok: meta?.ok === true, ...(meta?.code ? { error: meta.code } : {}) }, + } + } + + async _run(input, callbacks, state) { + const { platform, vmId, accountId, expectedGeneration, admission } = input + const { readCurrent, refresh, publish } = callbacks + const taskKey = `oauth-refresh:${platform}:${accountId}` + const observed = await this.tasks.repo.latest(taskKey) + if (observed && !observed.finishedAt) return this._joined(taskKey, observed, callbacks) + const current = checkCurrent(await readCurrent()) + if (current.generation !== expectedGeneration) + return { finished: true, unknown: false, shared: true, current, value: { ok: true } } + if (observed?.sharedResult?.ok && observed.sharedResult.inputGeneration === expectedGeneration) + throw failure('credential_projection_stale') + const claim = await this.tasks.tryClaim({ + taskKey, + executionId: randomUUID(), + ifPreviousExecutionId: observed?.executionId ?? null, + }) + if (!claim.ok) return this._joined(taskKey, await this.tasks.repo.latest(taskKey), callbacks) + let acquired = false + let metadata + try { + const inherited = currentVmOperationIds(this.db) + const result = await claim.run(() => + this.mutations.run( + vmId, + async (operation) => { + acquired = true + const fresh = checkCurrent(await readCurrent()) + let outcome = { finished: true, value: { ok: true } } + let next = fresh + if (fresh.generation === expectedGeneration) { + outcome = await refresh(fresh) + if (outcome?.finished !== true) { + state.unknown = true + await claim.markUnknown() + return { + finished: false, + unknown: true, + shared: false, + value: outcome?.value, + vmOperationUncertain: true, + } + } + if (outcome.value?.ok === true) next = checkCurrent(await publish(fresh, outcome.value)) + } + metadata = { + ok: outcome.value?.ok === true, + inputGeneration: expectedGeneration, + generation: next.generation, + // Only an explicit machine code can cross the shared-result boundary. + ...(typeof outcome.value?.error === 'string' && /^[a-z][a-z0-9_]{0,100}$/.test(outcome.value.error) + ? { code: outcome.value.error } + : {}), + } + await withTransaction(this.db, async () => { + if (!(await this.tasks.complete(claim, { sharedResult: metadata })).completed) + throw failure('credential_refresh_completion_unconfirmed') + if (!inherited.has(operation.operationId) && !(await this.mutations.release(operation)).released) + throw failure('vm_operation_release_unconfirmed') + }) + return { finished: true, unknown: false, shared: false, current: next, value: outcome.value } + }, + { kind: 'credential_refresh', accountId, admission }, + ), + ) + if (result.unknown) return result + if (!(await claim.complete({ sharedResult: metadata })).completed) + throw failure('credential_refresh_completion_unconfirmed') + return result + } catch (error) { + if (acquired) { + state.unknown = true + await claim.markUnknown() + } else { + // No mutation callback ran; the claim has caused no remote or credential effect. + await claim.complete({ sharedResult: { ok: false, code: 'refresh_not_started' } }) + } + throw error + } + } +} + +export function getSharedCredentialRefresh(db = getDb()) { + let service = services.get(db) + if (!service) { + service = new SharedCredentialRefresh(db) + services.set(db, service) + } + return service +} From f80d5ba577a482098f7498aaf0b92ad785eeebc8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:15:12 +0800 Subject: [PATCH 284/473] test: allow a new refresh after a successful unchanged credential --- test/unit/shared-credential-refresh.test.mjs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/unit/shared-credential-refresh.test.mjs b/test/unit/shared-credential-refresh.test.mjs index 1e0d807c2..7a2240bcc 100644 --- a/test/unit/shared-credential-refresh.test.mjs +++ b/test/unit/shared-credential-refresh.test.mjs @@ -139,6 +139,22 @@ test('a later caller may retry a definitively ended failure with the same creden ) }) +test('a later caller may retry after a successful no-op that kept the same credential generation', async (t) => { + const f = await fixture(t) + let calls = 0 + const callbacks = { + readCurrent: f.readCurrent, + publish: async () => f.readCurrent(), + refresh: async () => { + calls++ + return { finished: true, value: { ok: true, refreshed: false } } + }, + } + await f.first.run(input(), callbacks) + await f.second.run(input(), callbacks) + assert.equal(calls, 2) +}) + test('missing expected generation or unreadable current credential never falls back to a raw refresh', async (t) => { const f = await fixture(t) let calls = 0 From 40d480985ad230da29fa99bda1a482f6315d8934 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:15:54 +0800 Subject: [PATCH 285/473] feat(slotgate): bind captured node fences before jobs and require durable completion --- src/lib/core/node-execution-proof.mjs | 108 ++++++ .../017_execution_node_bindings.sql | 5 + .../db/repos/execution-admissions-repo.mjs | 43 ++- src/lib/transport/go-worker-client.mjs | 75 +++- src/lib/transport/kernel-router.mjs | 3 +- src/lib/transport/slotgate-client.mjs | 321 ++++++++++++++++++ test/fixtures/fake-slotgate.mjs | 4 +- test/fixtures/real-slotgate.mjs | 51 +++ test/fixtures/slotgate-client-node.go | 85 +++++ test/unit/slotgate-client.test.mjs | 150 ++++++-- test/unit/slotgate-pg-binding.test.mjs | 146 +++++++- 11 files changed, 934 insertions(+), 57 deletions(-) create mode 100644 src/lib/core/node-execution-proof.mjs create mode 100644 src/lib/db/postgres-migrations/017_execution_node_bindings.sql create mode 100644 src/lib/transport/slotgate-client.mjs create mode 100644 test/fixtures/real-slotgate.mjs create mode 100644 test/fixtures/slotgate-client-node.go diff --git a/src/lib/core/node-execution-proof.mjs b/src/lib/core/node-execution-proof.mjs new file mode 100644 index 000000000..c20b8632b --- /dev/null +++ b/src/lib/core/node-execution-proof.mjs @@ -0,0 +1,108 @@ +import { isDeepStrictEqual } from 'node:util' + +function invalid() { + throw Object.assign(new Error('invalid captured node execution identity'), { code: 'invalid_node_execution' }) +} + +function shape(value, keys) { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + ![Object.prototype, null].includes(Object.getPrototypeOf(value)) || + Object.keys(value).length !== keys.length || + keys.some((key) => !Object.hasOwn(value, key)) + ) + invalid() +} + +function identifier(value) { + if ( + typeof value !== 'string' || + !value || + value.trim() !== value || + value.length > 256 || + /[\x00-\x1f\x7f]/.test(value) + ) + invalid() + return value +} + +function generation(value) { + if (typeof value === 'number') { + if (!Number.isSafeInteger(value)) invalid() + value = String(value) + } else if (typeof value === 'bigint') value = String(value) + if (typeof value !== 'string' || !/^[1-9][0-9]{0,19}$/.test(value) || BigInt(value) > 18446744073709551615n) invalid() + return value +} + +export function normalizeNodeFence(value) { + shape(value, ['generation', 'incarnation', 'executor']) + const executor = value.executor + shape(executor, ['kind', 'node_boot_id', 'container_id', 'docker_daemon_id', 'started_at']) + if ( + executor.kind !== 'container' || + typeof executor.container_id !== 'string' || + !/^[a-f0-9]{64}$/.test(executor.container_id) || + typeof executor.started_at !== 'string' || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/.test(executor.started_at) || + !Number.isFinite(Date.parse(executor.started_at)) + ) + invalid() + const calendar = Date.parse(executor.started_at.slice(0, 19) + 'Z') + if (!Number.isFinite(calendar) || new Date(calendar).toISOString().slice(0, 19) !== executor.started_at.slice(0, 19)) + invalid() + return { + generation: generation(value.generation), + incarnation: identifier(value.incarnation), + executor: { + kind: 'container', + node_boot_id: identifier(executor.node_boot_id), + container_id: executor.container_id, + docker_daemon_id: identifier(executor.docker_daemon_id), + started_at: executor.started_at, + }, + } +} + +export function normalizeNodeBinding(value) { + shape(value, ['protocol', 'nodeId', 'vmId', 'kind', 'fence']) + if ( + value.protocol !== 'KSG2' || + value.kind !== 'kernel' || + typeof value.vmId !== 'string' || + !/^[a-zA-Z0-9_-]{1,128}$/.test(value.vmId) + ) + invalid() + return { + protocol: 'KSG2', + nodeId: identifier(value.nodeId), + vmId: value.vmId, + kind: 'kernel', + fence: normalizeNodeFence(value.fence), + } +} + +export function sameNodeBinding(left, right) { + return isDeepStrictEqual(normalizeNodeBinding(left), normalizeNodeBinding(right)) +} + +export function normalizeNodeCommand(value, { requestId, nodeBinding }) { + shape(value, ['id', 'fence', 'status', 'dispatched']) + if ( + value.id !== requestId || + !['pending', 'unknown', 'completed', 'terminated'].includes(value.status) || + typeof value.dispatched !== 'boolean' + ) + invalid() + const fence = normalizeNodeFence(value.fence) + if (!isDeepStrictEqual(fence, normalizeNodeBinding(nodeBinding).fence)) invalid() + return { id: identifier(value.id), fence, status: value.status, dispatched: value.dispatched } +} + +export function normalizeNodeCompletion(value, binding) { + const command = normalizeNodeCommand(value, binding) + if (command.status !== 'completed' || command.dispatched !== true) invalid() + return command +} diff --git a/src/lib/db/postgres-migrations/017_execution_node_bindings.sql b/src/lib/db/postgres-migrations/017_execution_node_bindings.sql new file mode 100644 index 000000000..ba22b4a06 --- /dev/null +++ b/src/lib/db/postgres-migrations/017_execution_node_bindings.sql @@ -0,0 +1,5 @@ +-- An immutable captured node identity is attached before sending a KSG2 job. +-- Only authenticated node-persisted completion can end these jobs; neither +-- controller EOF nor a later generation may replace the original identity. +ALTER TABLE execution_jobs ADD COLUMN node_binding jsonb; +ALTER TABLE execution_jobs ADD COLUMN node_completion jsonb; diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 4e03274e5..3f4a52ca8 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -3,6 +3,7 @@ import { withTransaction } from '../database.mjs' import { physicalContainerName, physicalVmAliases } from '../../vm/vm-physical-target.mjs' import { readWatchdogState } from '../../transport/kernel-watchdog-state.mjs' import { ExecutionCircuitsRepo } from './execution-circuits-repo.mjs' +import { normalizeNodeBinding, normalizeNodeCompletion, sameNodeBinding } from '../../core/node-execution-proof.mjs' function invalid(field) { return Object.assign(new Error(`Invalid execution admission ${field}`), { code: 'invalid_execution_admission' }) @@ -192,6 +193,7 @@ export class ExecutionAdmissionsRepo { } async bindExecution(handle, execution) { + const nodeBinding = execution?.nodeBinding === undefined ? null : normalizeNodeBinding(execution.nodeBinding) return withTransaction(this.db, async () => { const admission = await this._lockAdmission(handle) if (!admission || admission.released_at || admission.uncertain_at) { @@ -200,16 +202,22 @@ export class ExecutionAdmissionsRepo { if (typeof execution?.requestId !== 'string' || !execution.requestId.trim()) { throw jobError('invalid_execution_job') } + if (nodeBinding && nodeBinding.vmId !== admission.vm_id) throw jobError('execution_node_binding_mismatch') const pending = await this.db .prepare('SELECT request_id FROM execution_jobs WHERE admission_id = $1 AND ended_at IS NULL LIMIT 1') .get(handle.admissionId) if (pending) throw jobError('execution_job_pending') const row = await this.db - .prepare(`INSERT INTO execution_jobs (request_id, admission_id) VALUES ($1, $2) + .prepare(`INSERT INTO execution_jobs (request_id, admission_id, node_binding) VALUES ($1, $2, $3::jsonb) ON CONFLICT (request_id) DO NOTHING RETURNING request_id`) - .get(execution.requestId, handle.admissionId) + .get(execution.requestId, handle.admissionId, nodeBinding ? JSON.stringify(nodeBinding) : null) if (!row) throw jobError('execution_job_exists') - return { bound: true, admissionId: handle.admissionId, requestId: execution.requestId } + return { + bound: true, + admissionId: handle.admissionId, + requestId: execution.requestId, + ...(nodeBinding ? { nodeBinding } : {}), + } }) } @@ -236,13 +244,38 @@ export class ExecutionAdmissionsRepo { await this.markUncertain(handle) return { error: 'execution_job_mismatch' } } + let nodeCompletion = null + if (job.node_binding) { + try { + if (!sameNodeBinding(execution.nodeBinding, job.node_binding)) + throw jobError('execution_node_binding_mismatch') + if (execution.ended) + nodeCompletion = normalizeNodeCompletion(execution.nodeCompletion, { + requestId: execution.requestId, + nodeBinding: job.node_binding, + }) + } catch { + await this.markUncertain(handle) + return { error: 'execution_node_unconfirmed' } + } + } else if (execution.nodeBinding !== undefined || execution.nodeCompletion !== undefined) { + await this.markUncertain(handle) + return { error: 'execution_node_binding_mismatch' } + } if (job.ended_at) return { ended: true, uncertain: !!admission.uncertain_at, requestId: execution.requestId } await this.db .prepare(`UPDATE execution_jobs SET ended_at = CASE WHEN $1 THEN clock_timestamp() ELSE ended_at END, - started = COALESCE(started, false) OR $2, reason = $3 + started = COALESCE(started, false) OR $2, reason = $3, node_completion = $6::jsonb WHERE request_id = $4 AND admission_id = $5 AND ended_at IS NULL`) - .run(execution.ended, execution.started, execution.reason, execution.requestId, handle.admissionId) + .run( + execution.ended, + execution.started, + execution.reason, + execution.requestId, + handle.admissionId, + nodeCompletion ? JSON.stringify(nodeCompletion) : null, + ) if (!execution.ended) await this.markUncertain(handle) return { ended: execution.ended, diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index 4529b5f12..6869247a4 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -33,6 +33,7 @@ import { isWrapConnectionError, } from '../core/errors.mjs' import { extraHeadersFromLimitError } from '../pool/quota-window.mjs' +import { SlotgateClient } from './slotgate-client.mjs' const MAX_BODY = 64 * 1024 * 1024 const credentialRefreshTail = new Map() @@ -81,6 +82,12 @@ function workerRequest( exec, { method = 'GET', requestPath, body = null, signal, timeoutMs = 180000, timeoutMode = 'overall', headers = {} } = {}, ) { + // Ancillary legacy health/credential/cancel routes have no fenced adapter. + // An explicitly fenced executor must never reach the old UDS/SSH relay. + if (exec?.slotgate != null) + return Promise.reject( + Object.assign(new Error('fenced worker route unavailable'), { code: 'slotgate_route_unavailable' }), + ) return new Promise((resolve, reject) => { const { socketPath } = workerPaths(exec) if (!socketPath) { @@ -358,6 +365,15 @@ function kernelExecutionUnconfirmed(body, httpStatus = 0) { return !rejected(status) && !rejected(httpStatus) } +async function completeFencedExecution(client, handle, execution) { + try { + execution.nodeCompletion = await client.completion(handle) + Object.assign(execution, { ended: true, reason: 'node_completed' }) + } catch { + Object.assign(execution, { ended: false, reason: 'node_completion_unconfirmed' }) + } +} + function completeHttpExecution(execution, body, httpStatus) { const uncertain = kernelExecutionUnconfirmed(body, httpStatus) Object.assign(execution, { @@ -590,6 +606,8 @@ export async function callGoWorker({ execution, } } + const fenced = exec?.slotgate == null ? null : new SlotgateClient(exec.slotgate) + let startError let stopCancel = () => {} try { if (signal?.aborted) throw Object.assign(new Error('The operation was aborted'), { code: 'ABORT_ERR' }) @@ -609,20 +627,35 @@ export async function callGoWorker({ requestContext, }) // The caller can durably bind this ID before any bytes reach the kernel. - if (onExecutionStart) await onExecutionStart({ requestId }) + if (fenced) { + execution.nodeBinding = fenced.nodeBinding + try { + if (typeof onExecutionStart !== 'function') + throw Object.assign(new Error('durable node binding required'), { code: 'slotgate_binding_unconfirmed' }) + const acknowledgement = await onExecutionStart({ requestId, nodeBinding: fenced.nodeBinding }) + fenced.assertBinding(acknowledgement, requestId) + } catch (error) { + startError = error + throw error + } + Object.assign(execution, { ended: false, reason: 'node_prepared_unknown' }) + } else if (onExecutionStart) await onExecutionStart({ requestId }) if (signal?.aborted) throw Object.assign(new Error('The operation was aborted'), { code: 'ABORT_ERR' }) - stopCancel = cancelOnAbort(exec, signal, requestId) + if (!fenced) stopCancel = cancelOnAbort(exec, signal, requestId) + const handle = fenced ? await fenced.prepare(requestId) : null Object.assign(execution, { started: true, ended: false, reason: 'in_progress' }) - const response = await workerRequest(exec, { + const request = { method: 'POST', requestPath, body: { ...prepared.envelope, request_id: requestId }, signal, timeoutMs, - }) + } + const response = await (fenced ? fenced.forward(handle, request) : workerRequest(exec, request)) const data = await readAll(response) const parsed = parseJson(data) - completeHttpExecution(execution, parsed, response.statusCode) + if (fenced) await completeFencedExecution(fenced, handle, execution) + else completeHttpExecution(execution, parsed, response.statusCode) const headers = mergeRateLimitHeaders(publicHeaders(response.headers)) const ok = response.statusCode >= 200 && response.statusCode < 300 && parsed?.type !== 'error' if (ok) rememberThreadTurn(parsed?.id, prepared.thread) @@ -641,6 +674,7 @@ export async function callGoWorker({ execution, }) } catch (error) { + if (startError) throw startError if (!execution.ended) { execution.reason = signal?.aborted ? 'client_aborted' @@ -777,6 +811,8 @@ export async function streamGoWorker({ execution, } } + const fenced = exec?.slotgate == null ? null : new SlotgateClient(exec.slotgate) + let startError let committed = false // Hop-relative TTFT. firstTokenAt is epoch ms so request logs subtract ingress t0. const startedAt = Date.now() @@ -800,11 +836,24 @@ export async function streamGoWorker({ cliHop, requestContext, }) - if (onExecutionStart) await onExecutionStart({ requestId }) + if (fenced) { + execution.nodeBinding = fenced.nodeBinding + try { + if (typeof onExecutionStart !== 'function') + throw Object.assign(new Error('durable node binding required'), { code: 'slotgate_binding_unconfirmed' }) + const acknowledgement = await onExecutionStart({ requestId, nodeBinding: fenced.nodeBinding }) + fenced.assertBinding(acknowledgement, requestId) + } catch (error) { + startError = error + throw error + } + Object.assign(execution, { ended: false, reason: 'node_prepared_unknown' }) + } else if (onExecutionStart) await onExecutionStart({ requestId }) if (signal?.aborted) throw Object.assign(new Error('The operation was aborted'), { code: 'ABORT_ERR' }) - stopCancel = cancelOnAbort(exec, signal, requestId) + if (!fenced) stopCancel = cancelOnAbort(exec, signal, requestId) + const handle = fenced ? await fenced.prepare(requestId) : null Object.assign(execution, { started: true, ended: false, reason: 'in_progress' }) - const response = await workerRequest(exec, { + const request = { method: 'POST', requestPath, body: { ...prepared.envelope, request_id: requestId }, @@ -812,12 +861,14 @@ export async function streamGoWorker({ timeoutMs, timeoutMode: 'first-byte', headers: { te: 'trailers' }, - }) + } + const response = await (fenced ? fenced.forward(handle, request) : workerRequest(exec, request)) const headers = mergeRateLimitHeaders(publicHeaders(response.headers)) if ((response.statusCode || 0) < 200 || (response.statusCode || 0) >= 300) { const data = await readAll(response, 1024 * 1024) const parsed = parseJson(data) - completeHttpExecution(execution, parsed, response.statusCode) + if (fenced) await completeFencedExecution(fenced, handle, execution) + else completeHttpExecution(execution, parsed, response.statusCode) return restoreKernelErrorStatus({ ok: false, status: response.statusCode || 0, @@ -944,7 +995,8 @@ export async function streamGoWorker({ const event = observeSseEvent(takeSseEvent()) if (isDownstreamCommitEvent(event)) await flushCommit() } - completeHttpExecution(execution, unconfirmedKernelError || lastError, response.statusCode) + if (fenced) await completeFencedExecution(fenced, handle, execution) + else completeHttpExecution(execution, unconfirmedKernelError || lastError, response.statusCode) const trailers = mergeRateLimitHeaders(publicHeaders(response.trailers)) const meta = streamMetaFromHeaders({ ...headers, ...trailers }) const assembled = assembler.message @@ -978,6 +1030,7 @@ export async function streamGoWorker({ if (idleTimer) clearInterval(idleTimer) } } catch (error) { + if (startError) throw startError if (!execution.ended) { execution.reason = signal?.aborted ? 'client_aborted' diff --git a/src/lib/transport/kernel-router.mjs b/src/lib/transport/kernel-router.mjs index f40ba9e5f..f356565df 100644 --- a/src/lib/transport/kernel-router.mjs +++ b/src/lib/transport/kernel-router.mjs @@ -281,8 +281,9 @@ async function runHop({ mode, opts }) { typeof opts.onExecutionStart === 'function' ? async (execution) => { try { - await opts.onExecutionStart(execution) + const acknowledgement = await opts.onExecutionStart(execution) bound = true + return acknowledgement } catch (error) { startError = error throw error diff --git a/src/lib/transport/slotgate-client.mjs b/src/lib/transport/slotgate-client.mjs new file mode 100644 index 000000000..b08df56ea --- /dev/null +++ b/src/lib/transport/slotgate-client.mjs @@ -0,0 +1,321 @@ +import http from 'node:http' +import net from 'node:net' +import tls from 'node:tls' +import { Duplex } from 'node:stream' +import { setTimeout as delay } from 'node:timers/promises' +import { + normalizeNodeBinding, + normalizeNodeCommand, + normalizeNodeCompletion, + sameNodeBinding, +} from '../core/node-execution-proof.mjs' + +function failure(code) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} +function freeze(value) { + for (const item of Object.values(value)) if (item && typeof item === 'object') freeze(item) + return Object.freeze(value) +} +function wire(value) { + return JSON.stringify(value, (key, item) => (key === 'generation' ? JSON.rawJSON(item) : item)) +} +function parse(value) { + return JSON.parse(value, (key, item, context) => + key === 'generation' && typeof item === 'number' ? context.source : item, + ) +} + +// The gate reads the full request before replying OK. This Duplex sends HTTP +// immediately, while consuming only the bounded gate prefix before HTTP parses. +class ForwardSocket extends Duplex { + constructor(socket) { + super() + this.target = socket + let prefix = Buffer.alloc(0) + let accepted = false + socket.on('data', (chunk) => { + if (!accepted) { + prefix = Buffer.concat([prefix, chunk]) + const newline = prefix.indexOf(10) + if (newline < 0) { + if (prefix.length > 128) this.destroy(failure('slotgate_bad_reply')) + return + } + if (newline > 128 || prefix.subarray(0, newline).toString() !== 'OK') { + this.destroy(failure('slotgate_forward_rejected')) + return + } + accepted = true + chunk = prefix.subarray(newline + 1) + prefix = null + } + if (chunk.length && !this.push(chunk)) socket.pause() + }) + socket.once('end', () => { + if (!accepted) this.destroy(failure('slotgate_bad_reply')) + else this.push(null) + }) + socket.once('error', (error) => this.destroy(error)) + socket.once('close', () => { + if (!socket.readableEnded) this.destroy(failure('slotgate_disconnected')) + }) + } + _read() { + this.target.resume() + } + _write(chunk, encoding, callback) { + this.target.write(chunk, encoding, callback) + } + _final(callback) { + this.target.end(callback) + } + _destroy(error, callback) { + this.target.destroy() + callback(error) + } + setTimeout(...args) { + this.target.setTimeout(...args) + return this + } + setNoDelay(...args) { + this.target.setNoDelay(...args) + return this + } + setKeepAlive(...args) { + this.target.setKeepAlive(...args) + return this + } + ref() { + this.target.ref() + return this + } + unref() { + this.target.unref() + return this + } +} + +/** Explicit, trusted configuration only. No KSG1, UDS or SSH fallback exists. */ +export class SlotgateClient { + constructor(config) { + try { + if (config?.trusted !== true || config.pgBindingReady !== true) throw failure('slotgate_capability_invalid') + Object.defineProperty(this, 'nodeBinding', { + value: freeze( + normalizeNodeBinding({ + protocol: 'KSG2', + nodeId: config.nodeId, + vmId: config.vmId, + kind: 'kernel', + fence: config.fence, + }), + ), + enumerable: true, + }) + const link = config.link + if ( + !link || + !net.isIP(link.address) || + !Number.isInteger(link.port) || + link.port < 1 || + link.port > 65535 || + typeof link.token !== 'string' || + !/^[\x21-\x7e]{1,256}$/.test(link.token) || + !/^[a-f0-9]{64}$/.test(link.certSha256) + ) + throw failure('slotgate_capability_invalid') + this.link = freeze({ address: link.address, port: link.port, token: link.token, certSha256: link.certSha256 }) + } catch { + throw failure('slotgate_capability_invalid') + } + this.handles = new WeakSet() + } + + assertBinding(acknowledgement, requestId) { + try { + if ( + acknowledgement?.bound !== true || + acknowledgement.requestId !== requestId || + !sameNodeBinding(acknowledgement.nodeBinding, this.nodeBinding) + ) + throw failure('slotgate_binding_unconfirmed') + } catch { + throw failure('slotgate_binding_unconfirmed') + } + } + + _command(op, requestId, ticket) { + const value = { + op, + vm_id: this.nodeBinding.vmId, + fence: this.nodeBinding.fence, + command_id: requestId, + kind: this.nodeBinding.kind, + } + if (ticket) value.ticket = ticket + return value + } + + _connect(timeoutMs) { + return new Promise((resolve, reject) => { + let settled = false + const socket = tls.connect({ + host: this.link.address, + port: this.link.port, + rejectUnauthorized: false, + minVersion: 'TLSv1.3', + allowHalfOpen: true, + }) + const fail = (error) => { + if (settled) return + settled = true + clearTimeout(timer) + socket.destroy() + reject(error) + } + const timer = setTimeout(() => fail(failure('slotgate_timeout')), timeoutMs) + socket.once('error', fail) + socket.once('close', () => fail(failure('slotgate_disconnected'))) + socket.once('secureConnect', () => { + const pin = socket.getPeerCertificate()?.fingerprint256?.replaceAll(':', '').toLowerCase() + if (pin !== this.link.certSha256) return fail(failure('slotgate_cert_mismatch')) + settled = true + clearTimeout(timer) + socket.off('error', fail) + // Install an error observer until the protocol consumer attaches. + socket.on('error', () => {}) + resolve(socket) + }) + }) + } + + async _rpc(command, timeoutMs = 3000) { + const socket = await this._connect(timeoutMs) + return new Promise((resolve, reject) => { + let settled = false + let buffer = Buffer.alloc(0) + const done = (error, value) => { + if (settled) return + settled = true + clearTimeout(timer) + socket.removeAllListeners('data') + socket.destroy() + if (error) reject(error) + else resolve(value) + } + const timer = setTimeout(() => done(failure('slotgate_timeout')), timeoutMs) + socket.once('error', (error) => done(error)) + socket.once('end', () => done(failure('slotgate_bad_reply'))) + socket.once('close', () => done(failure('slotgate_disconnected'))) + socket.on('data', (chunk) => { + buffer = Buffer.concat([buffer, chunk]) + const newline = buffer.indexOf(10) + if (buffer.length > 8192) return done(failure('slotgate_bad_reply')) + if (newline < 0) return + try { + const reply = parse(buffer.subarray(0, newline).toString()) + if (reply?.ok !== true) return done(failure('slotgate_proof_unavailable')) + done(null, reply) + } catch (error) { + done(error) + } + }) + socket.write(`KSG2 ${this.link.token}\n${wire(command)}\n`) + }) + } + + async prepare(requestId) { + if ( + typeof requestId !== 'string' || + !requestId || + requestId.trim() !== requestId || + requestId.length > 256 || + /[\x00-\x1f\x7f]/.test(requestId) + ) + throw failure('slotgate_request_invalid') + const reply = await this._rpc(this._command('prepare', requestId)) + if (typeof reply.ticket !== 'string' || !/^[\x21-\x7e]{1,512}$/.test(reply.ticket)) + throw failure('slotgate_bad_reply') + const handle = freeze({ requestId, ticket: reply.ticket }) + this.handles.add(handle) + return handle + } + + async completion(handle) { + if (!this.handles.has(handle)) throw failure('slotgate_ticket_invalid') + const deadline = Date.now() + 1000 + for (;;) { + const reply = await this._rpc(this._command('command', handle.requestId, handle.ticket)) + const command = normalizeNodeCommand(reply.command, { + requestId: handle.requestId, + nodeBinding: this.nodeBinding, + }) + // A body can arrive just before the node's completion fsync. Retry only + // this read; a mismatch, unknown or unavailable proof ends the wait. + if (command.status !== 'pending' || !command.dispatched || Date.now() >= deadline) + return normalizeNodeCompletion(command, { requestId: handle.requestId, nodeBinding: this.nodeBinding }) + await delay(50) + } + } + + forward( + handle, + { method = 'POST', requestPath, body, headers, signal, timeoutMs = 180000, timeoutMode = 'overall' }, + ) { + if ( + !this.handles.has(handle) || + method !== 'POST' || + requestPath !== '/internal/v1/messages' || + body?.request_id !== handle.requestId + ) + return Promise.reject(failure('slotgate_request_invalid')) + return new Promise((resolve, reject) => { + const payload = Buffer.from(JSON.stringify(body)) + if (payload.length > 15 * 1024 * 1024) return reject(failure('slotgate_request_invalid')) + const agent = new http.Agent({ keepAlive: false }) + agent.createConnection = (_options, callback) => { + this._connect(Math.min(3000, timeoutMs)).then((socket) => { + if (request.destroyed) { + socket.destroy() + return + } + const forwarded = new ForwardSocket(socket) + socket.write(`KSG2 ${this.link.token}\n${wire(this._command('forward', handle.requestId, handle.ticket))}\n`) + callback(null, forwarded) + }, callback) + } + let timer + const request = http.request( + { + host: 'slotgate', + path: requestPath, + method, + agent, + signal, + headers: { + ...headers, + 'content-type': 'application/json', + 'content-length': payload.length, + connection: 'close', + }, + }, + (response) => { + if (timeoutMode === 'first-byte') clearTimeout(timer) + response.once('close', () => { + clearTimeout(timer) + agent.destroy() + }) + resolve(response) + }, + ) + timer = setTimeout(() => request.destroy(failure('worker_timeout')), Math.max(1, timeoutMs)) + request.once('error', (error) => { + clearTimeout(timer) + agent.destroy() + reject(error) + }) + request.end(payload) + }) + } +} diff --git a/test/fixtures/fake-slotgate.mjs b/test/fixtures/fake-slotgate.mjs index d3a932803..24f145af1 100644 --- a/test/fixtures/fake-slotgate.mjs +++ b/test/fixtures/fake-slotgate.mjs @@ -23,6 +23,7 @@ export const assistant = { id: 'synthetic-message', type: 'message', role: 'assistant', content: [{ type: 'text', text: 'synthetic answer' }], stop_reason: 'end_turn', } +const wire = (value) => JSON.stringify(value, (key, item) => key === 'generation' ? JSON.rawJSON(String(item)) : item) // A local protocol double, not a node/Docker or provider acceptance test. export async function fakeSlotgate(t, options = {}) { @@ -63,7 +64,7 @@ export async function fakeSlotgate(t, options = {}) { const held = commands.get(command.command_id) if (!held || held.ticket !== command.ticket) return socket.end('{"ok":false,"error":"ticket_invalid"}\n') const result = typeof options.status === 'function' ? options.status(command, held) : options.status - return socket.end(JSON.stringify(result || { + return socket.end(wire(result || { ok: true, command: { id: command.command_id, fence: command.fence, status: 'completed', dispatched: true }, }) + '\n') @@ -81,6 +82,7 @@ export async function fakeSlotgate(t, options = {}) { if (!held || held.ticket !== command.ticket) return socket.end('ERR ticket_invalid\n') held.envelope = envelope options.onForward?.(command, envelope) + if (options.hold) return const payload = options.body || JSON.stringify(assistant) if (options.truncated) return socket.end(`OK\nHTTP/1.1 200 OK\r\nContent-Length: ${Buffer.byteLength(payload) + 10}\r\n\r\n${payload}`) socket.end(`OK\nHTTP/1.1 ${options.statusCode || 200} Result\r\nContent-Type: ${options.sse ? 'text/event-stream' : 'application/json'}\r\nContent-Length: ${Buffer.byteLength(payload)}\r\nConnection: close\r\n\r\n${payload}`) diff --git a/test/fixtures/real-slotgate.mjs b/test/fixtures/real-slotgate.mjs new file mode 100644 index 000000000..e09262383 --- /dev/null +++ b/test/fixtures/real-slotgate.mjs @@ -0,0 +1,51 @@ +import { spawn, execFileSync } from 'node:child_process' +import { once } from 'node:events' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +export async function realSlotgate(t) { + // Keep the incarnation Unix path below the platform sockaddr_un limit. + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kg2-')) + const repository = fileURLToPath(new URL('../../', import.meta.url)) + const worker = path.join(root, 'worker') + fs.cpSync(path.join(repository, 'worker'), worker, { recursive: true }) + const cmd = path.join(worker, 'cmd', 'slotgate-client-fixture') + fs.mkdirSync(cmd, { recursive: true }) + fs.copyFileSync(fileURLToPath(new URL('./slotgate-client-node.go', import.meta.url)), path.join(cmd, 'main.go')) + const binary = path.join(root, 'node-fixture') + let process + let exited + t.after(async () => { + try { + if (process && process.exitCode == null) { + process.kill('SIGTERM') + const timer = setTimeout(() => process.kill('SIGKILL'), 3000) + try { await exited } finally { clearTimeout(timer) } + } + } finally { fs.rmSync(root, { recursive: true, force: true }) } + }) + execFileSync('go', ['build', '-buildvcs=false', '-o', binary, './cmd/slotgate-client-fixture'], { + cwd: worker, env: { ...globalThis.process.env, GOCACHE: '/tmp/vm2api-648-go-cache', GOPROXY: 'off' }, stdio: 'pipe', timeout: 60000, + }) + const runtime = path.join(root, 'runtime') + fs.mkdirSync(runtime, { mode: 0o700 }) + process = spawn(binary, [runtime], { stdio: ['ignore', 'pipe', 'pipe'] }) + exited = once(process, 'exit') + return new Promise((resolve, reject) => { + let buffer = '' + let stderr = '' + const timer = setTimeout(() => reject(new Error('synthetic slotgate fixture startup timeout')), 5000) + process.once('error', (error) => { clearTimeout(timer); reject(error) }) + process.once('exit', (code) => { clearTimeout(timer); if (code) reject(new Error(`synthetic slotgate fixture exit ${code}: ${stderr}`)) }) + process.stderr.on('data', (chunk) => { stderr = (stderr + chunk).slice(-8192) }) + process.stdout.on('data', (chunk) => { + buffer += chunk + const newline = buffer.indexOf('\n') + if (newline < 0) return + clearTimeout(timer) + try { resolve(JSON.parse(buffer.slice(0, newline))) } catch (error) { reject(error) } + }) + }) +} diff --git a/test/fixtures/slotgate-client-node.go b/test/fixtures/slotgate-client-node.go new file mode 100644 index 000000000..26ea08b71 --- /dev/null +++ b/test/fixtures/slotgate-client-node.go @@ -0,0 +1,85 @@ +// Local interoperability fixture. Synthetic state is installed by trusted test +// code; no Docker daemon or provider is contacted and no container is launched. +package main + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + + "github.com/dofastted/kin-gateway/worker/internal/slotgate" +) + +func check(err error) { + if err != nil { + panic(err) + } +} + +func main() { + root := os.Args[1] + journal := filepath.Join(root, "authority") + authority, err := slotgate.OpenAuthority(journal, nil) + check(err) + state, err := authority.CreateSlot("vm-synthetic") + check(err) + launch, err := authority.BeginLaunch("vm-synthetic", state.Fence, "synthetic-bootstrap") + check(err) + check(authority.StartLaunch(launch)) + state, err = authority.BindExecutor(launch, slotgate.ExecutorIdentity{ + Kind: "container", NodeBootID: "synthetic-boot", ContainerID: strings.Repeat("a", 64), + DockerDaemonID: "synthetic-engine", StartedAt: "2026-10-21T00:00:00.000000001Z", + }) + check(err) + check(authority.Close()) + run := filepath.Join(root, "vms", "vm-synthetic", "incarnations", state.Fence.Incarnation, "run") + check(os.MkdirAll(run, 0700)) + unix, err := net.Listen("unix", filepath.Join(run, "kernel.sock")) + check(err) + kernel := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var envelope struct { + Body struct { + Mode string `json:"mode"` + } `json:"body"` + } + check(json.NewDecoder(r.Body).Decode(&envelope)) + w.Header().Set("Content-Type", "application/json") + if envelope.Body.Mode == "unknown" { + w.WriteHeader(502) + fmt.Fprint(w, `{"type":"error","error":{"code":"provider_error","message":"client disconnected"}}`) + return + } + fmt.Fprint(w, `{"id":"synthetic-message","type":"message","role":"assistant","content":[{"type":"text","text":"synthetic answer"}],"stop_reason":"end_turn"}`) + })} + go kernel.Serve(unix) + defer kernel.Close() + const token = "synthetic-local-node-gate-token-000000" + server, err := slotgate.New(slotgate.Config{Listen: "127.0.0.1:0", Root: root, Token: token, + TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: journal, + DockerSocket: filepath.Join(root, "never-opened-docker.sock")}) + check(err) + listener, err := net.Listen("tcp", "127.0.0.1:0") + check(err) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + done := make(chan error, 1) + go func() { done <- server.ServeListener(ctx, tls.NewListener(listener, server.TLSConfig())) }() + cert := server.TLSConfig().Certificates[0].Certificate[0] + pin := sha256.Sum256(cert) + check(json.NewEncoder(os.Stdout).Encode(map[string]any{ + "trusted": true, "pgBindingReady": true, "nodeId": "node-synthetic", "vmId": "vm-synthetic", "fence": state.Fence, + "link": map[string]any{"address": "127.0.0.1", "port": listener.Addr().(*net.TCPAddr).Port, "token": token, "certSha256": hex.EncodeToString(pin[:])}, + })) + <-ctx.Done() + check(<-done) +} diff --git a/test/unit/slotgate-client.test.mjs b/test/unit/slotgate-client.test.mjs index 87d7c8b2f..bafee5e88 100644 --- a/test/unit/slotgate-client.test.mjs +++ b/test/unit/slotgate-client.test.mjs @@ -1,16 +1,27 @@ import test from 'node:test' import assert from 'node:assert/strict' import { SlotgateClient } from '../../src/lib/transport/slotgate-client.mjs' -import { callGoWorker, streamGoWorker } from '../../src/lib/transport/go-worker-client.mjs' +import { callGoWorker, streamGoWorker, workerHealth } from '../../src/lib/transport/go-worker-client.mjs' import { dispatchCallInference } from '../../src/lib/transport/kernel-router.mjs' import { assistant, fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' function execFor(config) { - return { vmId: config.vmId, slotgate: config, vm: { id: config.vmId, claude: {}, runtime: { kernel_socket: '/must-not-fallback.sock' } } } + return { + vmId: config.vmId, + slotgate: config, + vm: { id: config.vmId, claude: {}, runtime: { kernel_socket: '/must-not-fallback.sock' } }, + } +} +function acknowledged(execution) { + return { bound: true, requestId: execution.requestId, nodeBinding: execution.nodeBinding } } -function acknowledged(execution) { return { bound: true, requestId: execution.requestId, nodeBinding: execution.nodeBinding } } function call(config, patch = {}) { - return callGoWorker({ exec: execFor(config), envelope: { body: {} }, onExecutionStart: async (value) => acknowledged(value), ...patch }) + return callGoWorker({ + exec: execFor(config), + envelope: { body: {} }, + onExecutionStart: async (value) => acknowledged(value), + ...patch, + }) } test('KSG2 snapshots the original Fence and preserves uint64 generation without rounding', async (t) => { @@ -30,28 +41,36 @@ test('KSG2 snapshots the original Fence and preserves uint64 generation without test('KSG2 requires explicit trust, PG readiness and exact immutable identity fields', async (t) => { const fx = await fakeSlotgate(t) for (const patch of [ - { trusted: false }, { pgBindingReady: false }, + { trusted: false }, + { pgBindingReady: false }, { fence: { ...syntheticFence(), generation: Number.MAX_SAFE_INTEGER + 1 } }, { fence: { ...syntheticFence(), generation: '01' } }, { fence: { ...syntheticFence(), extra: true } }, { fence: { ...syntheticFence(), executor: { ...syntheticFence().executor, pid: 123 } } }, - ]) assert.throws(() => new SlotgateClient({ ...fx.config, ...patch }), { code: 'slotgate_capability_invalid' }) + ]) + assert.throws(() => new SlotgateClient({ ...fx.config, ...patch }), { code: 'slotgate_capability_invalid' }) assert.equal(fx.requests.length, 0) }) test('complete HTTP body remains uncertain until original-ticket node status is durably completed', async (t) => { const fx = await fakeSlotgate(t) let start - const result = await call(fx.config, { requestId: 'normal-job', onExecutionStart: async (value) => { - start = structuredClone(value) - assert.equal(fx.requests.length, 0, 'PG binding precedes node prepare and forward') - return acknowledged(value) - } }) + const result = await call(fx.config, { + requestId: 'normal-job', + onExecutionStart: async (value) => { + start = structuredClone(value) + assert.equal(fx.requests.length, 0, 'PG binding precedes node prepare and forward') + return acknowledged(value) + }, + }) assert.equal(result.ok, true) assert.equal(result.execution.ended, true) assert.deepEqual(result.execution.nodeBinding, start.nodeBinding) assert.equal(result.execution.nodeCompletion.status, 'completed') - assert.deepEqual(fx.requests.map((value) => value.op), ['prepare', 'forward', 'command']) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['prepare', 'forward', 'command'], + ) assert.equal(fx.commands.get('normal-job').envelope.request_id, 'normal-job') assert.deepEqual(fx.requests[2].fence, fx.requests[0].fence) assert.equal(fx.requests[2].ticket, fx.requests[1].ticket) @@ -59,11 +78,20 @@ test('complete HTTP body remains uncertain until original-ticket node status is for (const mode of ['pending', 'unknown', 'mismatch', 'unavailable']) { test(`HTTP success with ${mode} node proof retains execution and does not resend`, async (t) => { - const fx = await fakeSlotgate(t, { status: (command) => { - if (mode === 'unavailable') return { ok: false, error: 'authority_unavailable' } - return { ok: true, command: { id: command.command_id, fence: mode === 'mismatch' ? syntheticFence('2') : command.fence, - status: mode === 'pending' || mode === 'unknown' ? mode : 'completed', dispatched: true } } - } }) + const fx = await fakeSlotgate(t, { + status: (command) => { + if (mode === 'unavailable') return { ok: false, error: 'authority_unavailable' } + return { + ok: true, + command: { + id: command.command_id, + fence: mode === 'mismatch' ? syntheticFence('2') : command.fence, + status: mode === 'pending' || mode === 'unknown' ? mode : 'completed', + dispatched: true, + }, + } + }, + }) const result = await call(fx.config) assert.equal(result.execution.ended, false) assert.equal(result.execution.nodeCompletion, undefined) @@ -74,8 +102,17 @@ for (const mode of ['pending', 'unknown', 'mismatch', 'unavailable']) { test('a failed or malformed PG binding acknowledgement sends no node or fallback request', async (t) => { const fx = await fakeSlotgate(t) const error = new Error('synthetic bind failure') - await assert.rejects(call(fx.config, { onExecutionStart: async () => { throw error } }), (value) => value === error) - await assert.rejects(call(fx.config, { onExecutionStart: async () => ({ bound: true }) }), { code: 'slotgate_binding_unconfirmed' }) + await assert.rejects( + call(fx.config, { + onExecutionStart: async () => { + throw error + }, + }), + (value) => value === error, + ) + await assert.rejects(call(fx.config, { onExecutionStart: async () => ({ bound: true }) }), { + code: 'slotgate_binding_unconfirmed', + }) assert.equal(fx.requests.length, 0) }) @@ -84,23 +121,88 @@ test('truncated body never queries a successful completion or releases execution const result = await call(fx.config) assert.equal(result.execution.ended, false) assert.equal(result.transportError, true) - assert.deepEqual(fx.requests.map((value) => value.op), ['prepare', 'forward']) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['prepare', 'forward'], + ) }) test('normal SSE requires the same durable node status as JSON', async (t) => { const body = `event: message_start\ndata: ${JSON.stringify({ type: 'message_start', message: { ...assistant, content: [], stop_reason: null } })}\n\nevent: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}\n\nevent: message_stop\ndata: {"type":"message_stop"}\n\n` const fx = await fakeSlotgate(t, { body, sse: true, status: { ok: false, error: 'authority_unavailable' } }) - const result = await streamGoWorker({ exec: execFor(fx.config), envelope: { body: {} }, onExecutionStart: async (value) => acknowledged(value) }) + const result = await streamGoWorker({ + exec: execFor(fx.config), + envelope: { body: {} }, + onExecutionStart: async (value) => acknowledged(value), + }) + assert.equal(result.execution.ended, false) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['prepare', 'forward', 'command'], + ) +}) + +test('a pending node fsync is awaited using only the original ticket', async (t) => { + let reads = 0 + const fx = await fakeSlotgate(t, { + status: (command) => ({ + ok: true, + command: { + id: command.command_id, + fence: command.fence, + status: ++reads === 1 ? 'pending' : 'completed', + dispatched: true, + }, + }), + }) + const result = await call(fx.config) + assert.equal(result.execution.ended, true) + assert.equal(reads, 2) + assert.equal(fx.requests.filter((value) => value.op === 'forward').length, 1) + assert.equal(new Set(fx.requests.filter((value) => value.ticket).map((value) => value.ticket)).size, 1) +}) + +test('certificate mismatch sends no token or job and never opens the legacy UDS', async (t) => { + const fx = await fakeSlotgate(t) + fx.config.link.certSha256 = '0'.repeat(64) + const result = await call(fx.config) assert.equal(result.execution.ended, false) - assert.deepEqual(fx.requests.map((value) => value.op), ['prepare', 'forward', 'command']) + assert.equal(result.transportError, true) + assert.equal(fx.requests.length, 0) +}) + +test('fenced cancellation retains the original execution without an unfenced cancel request', async (t) => { + const controller = new AbortController() + const fx = await fakeSlotgate(t, { hold: true, onForward: () => controller.abort() }) + const result = await call(fx.config, { signal: controller.signal }) + assert.equal(result.execution.ended, false) + assert.equal(result.execution.reason, 'client_aborted') + assert.deepEqual( + fx.requests.map((value) => value.op), + ['prepare', 'forward'], + ) + const health = await workerHealth(execFor(fx.config)) + assert.equal(health.code, 'slotgate_route_unavailable') + assert.equal(fx.requests.length, 2) }) test('router preserves durable PG bind acknowledgement for the fenced worker client', async (t) => { + const previous = process.env.KIN_KERNEL_BIN + process.env.KIN_KERNEL_BIN = '/bin/true' + t.after(() => { + if (previous == null) delete process.env.KIN_KERNEL_BIN + else process.env.KIN_KERNEL_BIN = previous + }) const fx = await fakeSlotgate(t) let completed - const result = await dispatchCallInference({ exec: execFor(fx.config), envelope: { body: {} }, + const result = await dispatchCallInference({ + exec: execFor(fx.config), + envelope: { body: {} }, ensureRust: async () => ({ ok: true }), - onExecutionStart: async (value) => acknowledged(value), onExecutionEnd: async (value) => { completed = value }, + onExecutionStart: async (value) => acknowledged(value), + onExecutionEnd: async (value) => { + completed = value + }, }) assert.equal(result.execution.ended, true) assert.equal(completed.nodeCompletion.status, 'completed') diff --git a/test/unit/slotgate-pg-binding.test.mjs b/test/unit/slotgate-pg-binding.test.mjs index 9de6b8337..0601c3ce6 100644 --- a/test/unit/slotgate-pg-binding.test.mjs +++ b/test/unit/slotgate-pg-binding.test.mjs @@ -4,8 +4,16 @@ import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admiss import { createDatabase, getDb } from '../../src/lib/db/database.mjs' import { callGoWorker } from '../../src/lib/transport/go-worker-client.mjs' import { fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' +import { realSlotgate } from '../fixtures/real-slotgate.mjs' -const candidate = { accountId: 'synthetic-account', vmId: 'vm-synthetic', accountLimit: 1, vmLimit: 1, rpmLimit: 0, fableLimit: 0 } +const candidate = { + accountId: 'synthetic-account', + vmId: 'vm-synthetic', + accountLimit: 1, + vmLimit: 1, + rpmLimit: 0, + fableLimit: 0, +} async function fixture(t, options) { const gate = await fakeSlotgate(t, options) @@ -15,18 +23,30 @@ async function fixture(t, options) { const second = new ExecutionAdmissionsRepo(otherDb, { ownerId: 'synthetic-controller-b' }) const lease = await first.tryReserve(candidate) assert.equal(lease.ok, true) - return { ...gate, db, second, lease, async run() { - const result = await callGoWorker({ exec: { vmId: candidate.vmId, slotgate: gate.config }, - requestId: 'synthetic-fenced-job', envelope: { body: {} }, onExecutionStart: lease.bindExecution }) - await lease.completeExecution(result.execution) - return result - } } + return { + ...gate, + db, + second, + lease, + async run() { + const result = await callGoWorker({ + exec: { vmId: candidate.vmId, slotgate: gate.config }, + requestId: 'synthetic-fenced-job', + envelope: { body: {} }, + onExecutionStart: lease.bindExecution, + }) + await lease.completeExecution(result.execution) + return result + }, + } } test('two PG connections observe the original node binding before forwarding and completion before release', async (t) => { const fx = await fixture(t) const result = await fx.run() - const row = await fx.second.db.prepare('SELECT * FROM execution_jobs WHERE request_id = $1').get('synthetic-fenced-job') + const row = await fx.second.db + .prepare('SELECT * FROM execution_jobs WHERE request_id = $1') + .get('synthetic-fenced-job') assert.deepEqual(row.node_binding, result.execution.nodeBinding) assert.deepEqual(row.node_completion, result.execution.nodeCompletion) assert.equal(row.node_binding.fence.generation, '1') @@ -41,20 +61,70 @@ test('an HTTP success without durable node status retains shared PG capacity', a assert.equal(result.execution.ended, false) assert.equal((await fx.lease.release()).released, false) assert.equal((await fx.second.tryReserve(candidate)).code, 'concurrency_limit') - const job = await fx.second.db.prepare('SELECT * FROM execution_jobs WHERE request_id = $1').get('synthetic-fenced-job') + const job = await fx.second.db + .prepare('SELECT * FROM execution_jobs WHERE request_id = $1') + .get('synthetic-fenced-job') assert.equal(job.ended_at, null) assert.equal(job.node_completion, null) }) test('PG completion refuses a local EOF or a different generation for a node-bound job', async (t) => { const fx = await fixture(t) - const nodeBinding = { protocol: 'KSG2', nodeId: fx.config.nodeId, vmId: candidate.vmId, kind: 'kernel', fence: syntheticFence() } + const nodeBinding = { + protocol: 'KSG2', + nodeId: fx.config.nodeId, + vmId: candidate.vmId, + kind: 'kernel', + fence: syntheticFence(), + } await fx.lease.bindExecution({ requestId: 'synthetic-job', nodeBinding }) - await assert.rejects(fx.lease.completeExecution({ requestId: 'synthetic-job', started: true, ended: true, reason: 'eof' })) - await assert.rejects(fx.lease.completeExecution({ requestId: 'synthetic-job', started: true, ended: true, reason: 'node_completed', - nodeCompletion: { id: 'synthetic-job', fence: syntheticFence('2'), status: 'completed', dispatched: true } })) + await assert.rejects( + fx.lease.completeExecution({ requestId: 'synthetic-job', nodeBinding, started: true, ended: true, reason: 'eof' }), + ) + await assert.rejects( + fx.lease.completeExecution({ + requestId: 'synthetic-job', + nodeBinding, + started: true, + ended: true, + reason: 'node_completed', + nodeCompletion: { id: 'synthetic-job', fence: syntheticFence('2'), status: 'completed', dispatched: true }, + }), + ) assert.equal((await fx.lease.release()).released, false) - assert.equal((await fx.second.db.prepare('SELECT ended_at FROM execution_jobs WHERE request_id=$1').get('synthetic-job')).ended_at, null) + assert.equal( + (await fx.second.db.prepare('SELECT ended_at FROM execution_jobs WHERE request_id=$1').get('synthetic-job')) + .ended_at, + null, + ) +}) + +test('the actual Go KSG2 node persists normal completion but retains an internal error across two PG connections', { + skip: process.platform !== 'linux', +}, async (t) => { + const config = await realSlotgate(t) + const db = getDb() + const otherDb = await createDatabase({ connectionString: process.env.VM2API_TEST_DATABASE_URL, schema: db.schema }) + const first = new ExecutionAdmissionsRepo(db, { ownerId: 'synthetic-controller-a' }) + const second = new ExecutionAdmissionsRepo(otherDb, { ownerId: 'synthetic-controller-b' }) + for (const mode of ['normal', 'unknown']) { + const lease = await first.tryReserve(candidate) + assert.equal(lease.ok, true) + const result = await callGoWorker({ + exec: { vmId: config.vmId, slotgate: config }, + envelope: { body: { mode } }, + onExecutionStart: lease.bindExecution, + }) + await lease.completeExecution(result.execution) + assert.equal(result.execution.ended, mode === 'normal') + assert.equal((await lease.release()).released, mode === 'normal') + } + assert.equal((await second.tryReserve(candidate)).code, 'concurrency_limit') + const rows = await otherDb.prepare('SELECT * FROM execution_jobs ORDER BY created_at').all() + assert.ok(rows[0].ended_at) + assert.equal(rows[0].node_completion.status, 'completed') + assert.equal(rows[1].ended_at, null) + assert.equal(rows[1].node_completion, null) }) test('PG binding failure causes zero node prepare or forward operations', async (t) => { @@ -62,6 +132,52 @@ test('PG binding failure causes zero node prepare or forward operations', async await fx.db.exec(`CREATE FUNCTION reject_synthetic_binding() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic node binding failure' USING ERRCODE='P0001'; END; $$; CREATE TRIGGER reject_synthetic_binding BEFORE INSERT ON execution_jobs FOR EACH ROW EXECUTE FUNCTION reject_synthetic_binding()`) - await assert.rejects(fx.run(), /synthetic node binding failure/) + await assert.rejects(fx.run(), { code: 'P0001' }) assert.equal(fx.requests.length, 0) }) + +test('PG completion failure keeps the original job pending after node completion without a resend', async (t) => { + const fx = await fixture(t) + await fx.db.exec(`CREATE FUNCTION reject_synthetic_completion() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic completion failure' USING ERRCODE='P0001'; END; $$; + CREATE TRIGGER reject_synthetic_completion BEFORE UPDATE OF ended_at ON execution_jobs FOR EACH ROW EXECUTE FUNCTION reject_synthetic_completion()`) + await assert.rejects(fx.run(), { code: 'P0001' }) + assert.equal(fx.requests.filter((value) => value.op === 'forward').length, 1) + assert.equal((await fx.lease.release()).released, false) + assert.equal((await fx.second.tryReserve(candidate)).code, 'concurrency_limit') + assert.equal((await fx.second.db.prepare('SELECT ended_at FROM execution_jobs').get()).ended_at, null) +}) + +test('the persisted node binding cannot be changed by later mutation or an extraneous identity field', async (t) => { + const fx = await fixture(t) + const binding = { + protocol: 'KSG2', + nodeId: fx.config.nodeId, + vmId: candidate.vmId, + kind: 'kernel', + fence: syntheticFence('18446744073709551615'), + } + const pending = fx.lease.bindExecution({ requestId: 'original-capture', nodeBinding: binding }) + binding.fence.generation = '2' + const ack = await pending + assert.equal(ack.nodeBinding.fence.generation, '18446744073709551615') + const row = await fx.second.db.prepare('SELECT node_binding FROM execution_jobs').get() + assert.equal(row.node_binding.fence.generation, '18446744073709551615') + await assert.rejects( + fx.lease.completeExecution({ + requestId: 'original-capture', + started: true, + ended: true, + reason: 'node_completed', + nodeBinding: ack.nodeBinding, + nodeCompletion: { + id: 'original-capture', + fence: { ...ack.nodeBinding.fence, extra: true }, + status: 'completed', + dispatched: true, + }, + }), + { code: 'execution_node_unconfirmed' }, + ) + assert.equal((await fx.lease.release()).released, false) +}) From 1b52a9dd7fbe00b62203a431781f599802fc53e5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:16:11 +0800 Subject: [PATCH 286/473] docs: plan node-owned VM management and file publication RPCs --- .../2026-10-21-node-vm-management-rpc.md | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 docs/plans/2026-10-21-node-vm-management-rpc.md diff --git a/docs/plans/2026-10-21-node-vm-management-rpc.md b/docs/plans/2026-10-21-node-vm-management-rpc.md new file mode 100644 index 000000000..b44a18f77 --- /dev/null +++ b/docs/plans/2026-10-21-node-vm-management-rpc.md @@ -0,0 +1,56 @@ +# 节点持有 VM 管理权限的下一批接口 + +本设计只规划替换 Kubernetes 控制面的 SSH、Docker 和 SFTP 管理路径。它不启用 HA，不修改节点配置，不创建 VM，也不要求修改闭源 kernel。执行侧复用开源 slotgate 和现有 payload。当前单活动控制面保护继续保留。 + +## 当前可复用的状态 + +- `worker/internal/slotgate/authority.go:285` 的 `CreateSlot` 只供可信节点组件调用，分配 `VACANT` generation/incarnation；KSG2 尚无 allocate RPC。已有 VM 不会被自动认领。 +- Authority journal 使用临时文件 fsync、rename、父目录 fsync，持久化失败令该 daemon 拒绝后续授权；重开时 pending 变成 unknown/revoking。command ID 保留，不能改用新 fence 重试旧 ID。 +- `docker_launch.go:63` 只启动普通 Claude Docker executor。节点 `DockerLaunchPolicy` 固定 image ID、network ID、UID/GID。客户端不能传 Docker spec、路径、环境变量或命令。启动记录完整 container/daemon/boot 身份，inspect 后才绑定。 +- `docker_launch.go:216` 的私有树必须预先存在；没有上传或发布接口。config 文件按单文件只读挂载。原子替换宿主机该文件不保证现有单文件 bind mount 看到新 inode，因此第一批 publish 仅覆盖启动前，不声称支持现有进程热更新。 +- `ksg2.go:112` 当前只提供 state、prepare/forward、command、revoke、advance、launch。客户端不能提交 bind、complete 或 ended。现有正常 Claude 消息终态和只读 command 查询可复用，其他 worker 管理响应不能因此自动成为 node completed。 +- `docker_verifier.go:25` 的 verifier 会核对原 daemon、boot、container ID、StartedAt，停止并删除该容器，再验证同 daemon 下 404。`advance` 是有副作用的结束确认，不是普通状态读取；它还拒绝未知 mutation。没有 executor 的初始 VACANT 状态不能直接套用这条推进路径。 + +## 第一批：allocate、启动前 publish、现有 launch + +只覆盖新的普通 Claude Docker 槽位；KVM、macOS native、控制面 Codex 子进程、交互 PTY 和 legacy executor 不回退到原始命令。 + +| 提议接口 | 输入约束 | 节点持久结果 | +| --- | --- | --- | +| allocate | 规范 VM ID、全局 VM operation ID、独立 allocation ID；不接受宿主机路径 | 初始 fence 和 allocation receipt；同 ID/同意图可查询，同 ID/不同内容冲突 | +| publish-initial | captured fence、command ID、预期 manifest revision、固定文件枚举、受限内容 | manifest digest/revision 与 completed；没有 secret bytes | +| launch | 继续使用已有 captured fence + command ID | 现有可信 executor identity | + +PG VM claim 必须先独立提交。新逻辑身份、现有物理别名、node placement 和每个 node command 的绑定须在实际 dispatch 前持久化。node 不能只相信客户端提交的 operation ID；需验证服务端绑定的 action/fence/node/owner/未撤销状态。最小实现可给 node 一个受限的 PG 只读授权视图，避免把发行管理授权的密钥交给所有 runtime Pod。授权查询失败即拒绝新操作，不释放已执行操作。 + +allocate 应在 journal 保存 allocation ID，解决响应丢失后的同一分配查询。不能对已存在 VM 重新调用 CreateSlot 后用 State 的新值替换原 captured fence。尚未绑定 executor 的取消/删除需要单独证明未 dispatch 的规则，不伪造 termination proof。 + +publish 从节点私有 staging 开始；不向最终路径流式写入。禁止 absolute/relative path、任意文件名、symlink/hardlink、客户端 UID/mode/chown。节点生成目录权限、internal.token 和固定 credential link。kernel config 从节点 preset 与有界参数生成，不直接接收可改变启动程序或 host 路径的 JSON。 + +发布前核对原 fence、VACANT、无未知操作及预期 manifest revision；节点在最终 rename 的发送边界再次校验，持久记录 completed 后才允许 launch。stage、rename、fsync 或响应不明保留 pending，不自动重传。新 incarnation 使用独立目录，晚到的旧 publish 不能改新目录。上传大小/文件数沿用明确的协议上限，不增加用户配置。 + +现有 `LaunchPolicy.network_id` 是每节点固定策略，尚不覆盖每 VM 动态 SOCKS 绑定；第一批不能据此宣称现有所有 egress 路径已迁移。镜像仍由节点管理员批准，不开放任意 pull/build/spec 上传。 + +## 第二批：停止、重建、删除与受限管理 + +- stop/restart 使用 captured executor，沿用 revoke 后 verifier 的不可变 ID 删除确认；restart 在新 incarnation 发布并重新 launch，不直接 Engine restart 旧 ID。 +- 为 stop 增加持久 operation receipt，保证响应丢失后可查询原操作。不能把 stop 当普通 pending mutation 再调用现有 advance，否则会被自己的 pending mutation 阻止；只允许可信停止适配器结束它自己的 exact intent，其它未知 mutation 继续阻止推进。 +- destroy 先取得上述结束 receipt，再删除该 VM 的受控私有数据。保留逻辑身份、旧 generation 与 command tombstone；不提供任意 rm 路径，也不因客户端报成功删除 PG 记录。 +- 下一组 typed RPC 可分别提供 credential import/refresh、seed/config publish、telemetry restart、identity read 和 approved payload revision 切换。每个动作独立列明开始、结束和 unknown 规则，不开放 shell/exec/env 的通用替代接口。 +- 活动 credential 发布在固定目录内做原子替换，校验权威 credential revision/预期 generation，拒绝晚完成的旧导入或刷新。节点为真实凭据持有者；Pod 本地镜像不能覆盖已经轮换的节点凭据。导出必须走已有用户权限与专用受保护响应，不进入 journal、日志或 SharedBG sharedResult。 + +调用方优先替换 `slotHost` 的 start/reload/stop/destroy、syncRun/onImport/afterRefresh，再替换 `remote-slot.mjs` 和 `remote-slot-files.mjs` 的直接 Engine/SFTP。保留 standalone 路径；已声明 fenced 的节点遇到不支持动作直接拒绝，禁止退回 KSG1、SSH、UDS relay 或 raw Docker。 + +## 权限迁移与未覆盖的异步写入 + +现有 `placement.nodeSession:95` 返回 SSH client 和 sshDocker；`cluster-manager:798` 仍可向 Engine 发送请求。`remote-slot-files:56/106` 经 SFTP 写 run/seed、credential 与二进制；远端凭据在运行后权威。Pod 即使持有 PG claim，仍可能发送绕过 fence 的旧 SSH/SFTP 命令。 + +另有需要先补的当前调用链：`rust-kernel-supervisor.writeKernelConfig:782` 调 `queueSyncRun`；`remote-slot-files.queueSlotPush:223` 只在本 Pod 串行、捕获错误后仅记录日志。现有管理 flush 不等待这些原始 Promise。这意味着局部文件更新测试不能证明远端写入已结束。迁移前应把本操作产生的 push Promise 显式返回并 await，传播错误；不能只加一个吞错的全局 flush。节点 publish 再以 fence/revision 拒绝跨 Pod 的迟到写入。 + +权限迁移分为：先以 mock/临时 Unix socket 验证新接口；再由另行授权的节点初始化流程安装服务、批准 payload/image/network、建立 TLS pin；然后迁移调用方并拒绝 legacy fallback；最后撤销 runtime Pod 的 SSH 私钥、docker-group/root、SFTP 写入、Docker socket、替换 gate/journal 的权限。需终止已经建立的 SSH/SFTP/forward/终端会话，不能只撤销新连接认证。runtime 与初始化/升级凭据分离。没有完成这一步，node fence 不能阻断仍有任意节点权限的旧 owner。 + +## 验收范围 + +先失败规格分别证明：双连接 allocate 唯一；响应丢失不重复创建；旧 fence publish 无写入；路径/链接/权限拒绝；rename 与 revoke 竞态不改新 incarnation；stage/Engine/journal 故障保留；原 container ID stop/remove 确认；unknown mutation 阻止 generation 推进；原始 push Promise 未完成时 claim 不释放；fenced 节点不走 SSH fallback。测试只用合成 Engine、临时目录与专用 PG。 + +两个不同节点上的迁移验证、旧会话撤权、节点崩溃恢复、Codex/native/KVM/交互 PTY 归属和全部管理生产调用链，均不由本设计或现有合成测试证明。 From 900b9125fff05e0d218cfac5d69bc87cc0721fd6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:16:29 +0800 Subject: [PATCH 287/473] docs: correct remote publication helper references --- docs/plans/2026-10-21-node-vm-management-rpc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plans/2026-10-21-node-vm-management-rpc.md b/docs/plans/2026-10-21-node-vm-management-rpc.md index b44a18f77..bee1aac6d 100644 --- a/docs/plans/2026-10-21-node-vm-management-rpc.md +++ b/docs/plans/2026-10-21-node-vm-management-rpc.md @@ -43,7 +43,7 @@ publish 从节点私有 staging 开始；不向最终路径流式写入。禁止 ## 权限迁移与未覆盖的异步写入 -现有 `placement.nodeSession:95` 返回 SSH client 和 sshDocker；`cluster-manager:798` 仍可向 Engine 发送请求。`remote-slot-files:56/106` 经 SFTP 写 run/seed、credential 与二进制；远端凭据在运行后权威。Pod 即使持有 PG claim，仍可能发送绕过 fence 的旧 SSH/SFTP 命令。 +现有 `placement.nodeSession:95` 返回 SSH client 和 sshDocker；`cluster-manager:798` 仍可向 Engine 发送请求。`remote-slot-files.mjs` 经 `remote-fs.mjs:56/106` 的受限文件操作写 run/seed、credential 与二进制；远端凭据在运行后权威。Pod 即使持有 PG claim，仍可能发送绕过 fence 的旧 SSH/SFTP 命令。 另有需要先补的当前调用链：`rust-kernel-supervisor.writeKernelConfig:782` 调 `queueSyncRun`；`remote-slot-files.queueSlotPush:223` 只在本 Pod 串行、捕获错误后仅记录日志。现有管理 flush 不等待这些原始 Promise。这意味着局部文件更新测试不能证明远端写入已结束。迁移前应把本操作产生的 push Promise 显式返回并 await，传播错误；不能只加一个吞错的全局 flush。节点 publish 再以 fence/revision 拒绝跨 Pod 的迟到写入。 From ef529d2e95dafe5093de1bbc1602c072ec4a7a47 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:16:56 +0800 Subject: [PATCH 288/473] test: distinguish finished credential refresh attempts from uncertain transport --- test/unit/credential-refresh-proof.test.mjs | 65 +++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 test/unit/credential-refresh-proof.test.mjs diff --git a/test/unit/credential-refresh-proof.test.mjs b/test/unit/credential-refresh-proof.test.mjs new file mode 100644 index 000000000..1e6666ef3 --- /dev/null +++ b/test/unit/credential-refresh-proof.test.mjs @@ -0,0 +1,65 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { runSlotOauth } from '../../src/lib/transport/slot-oauth.mjs' +import { refreshCodexAccessToken } from '../../src/lib/protocol/codex-models.mjs' + +const exec = { vmId: 'vm-refresh-proof', vm: {} } +const worker = (payload, extra = {}) => async () => ({ + code: 0, + stdout: JSON.stringify(payload), + stderr: '', + timed_out: false, + ...extra, +}) + +test('a complete successful worker command proves its credential attempt finished', async () => { + const result = await runSlotOauth(exec, 'refresh', { + runDocker: worker({ ok: true, status: 200, body: { refreshed: true } }), + }) + assert.equal(result.attemptFinished, true) +}) + +test('worker timeout, nonzero exit and reported upstream truncation cannot prove completion', async () => { + for (const [payload, extra] of [ + [{ ok: true, status: 200 }, { timed_out: true }], + [{ ok: true, status: 200 }, { code: 1 }], + [{ ok: false, status: 401, body: { error: { code: 'upstream_transport_error' } } }, {}], + ]) { + const result = await runSlotOauth(exec, 'refresh', { runDocker: worker(payload, extra) }) + assert.equal(result.attemptFinished, false) + } +}) + +test('a complete token rejection proves the refresh attempt finished', async () => { + const result = await refreshCodexAccessToken({ + refreshToken: 'SYNTHETIC_RT', + fetchImpl: async () => ({ status: 401, json: async () => ({ error: 'invalid_grant' }) }), + }) + assert.equal(result.attemptFinished, true) +}) + +test('a rejected response with a truncated token body is uncertain', async () => { + const result = await refreshCodexAccessToken({ + refreshToken: 'SYNTHETIC_RT', + fetchImpl: async () => ({ + status: 401, + json: async () => { + throw new Error('synthetic premature close') + }, + }), + }) + assert.equal(result.attemptFinished, false) +}) + +test('a successful token response missing its credential is uncertain', async () => { + const result = await refreshCodexAccessToken({ + refreshToken: 'SYNTHETIC_RT', + fetchImpl: async () => ({ status: 200, json: async () => ({}) }), + }) + assert.equal(result.attemptFinished, false) +}) + +test('a missing refresh token proves nothing was submitted', async () => { + const result = await refreshCodexAccessToken({}) + assert.equal(result.attemptFinished, true) +}) From db860347072bf0883270711bc4baa709c179c248 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:16:56 +0800 Subject: [PATCH 289/473] fix: allow refresh after a successful unchanged credential --- src/lib/oauth/shared-credential-refresh.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/oauth/shared-credential-refresh.mjs b/src/lib/oauth/shared-credential-refresh.mjs index af1b7d290..b775ce384 100644 --- a/src/lib/oauth/shared-credential-refresh.mjs +++ b/src/lib/oauth/shared-credential-refresh.mjs @@ -90,7 +90,11 @@ export class SharedCredentialRefresh { const current = checkCurrent(await readCurrent()) if (current.generation !== expectedGeneration) return { finished: true, unknown: false, shared: true, current, value: { ok: true } } - if (observed?.sharedResult?.ok && observed.sharedResult.inputGeneration === expectedGeneration) + if ( + observed?.sharedResult?.ok && + observed.sharedResult.inputGeneration === expectedGeneration && + observed.sharedResult.generation !== expectedGeneration + ) throw failure('credential_projection_stale') const claim = await this.tasks.tryClaim({ taskKey, From 492795cf91cacb2640898f8e2129848a9556512e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:18:00 +0800 Subject: [PATCH 290/473] test(slotgate): require existing kernel authentication headers on KSG2 --- test/fixtures/fake-slotgate.mjs | 1 + test/fixtures/real-slotgate.mjs | 4 +++- test/fixtures/slotgate-client-node.go | 5 +++++ test/unit/slotgate-client.test.mjs | 17 +++++++++++++++++ test/unit/slotgate-pg-binding.test.mjs | 2 +- 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/test/fixtures/fake-slotgate.mjs b/test/fixtures/fake-slotgate.mjs index 24f145af1..300bf7093 100644 --- a/test/fixtures/fake-slotgate.mjs +++ b/test/fixtures/fake-slotgate.mjs @@ -81,6 +81,7 @@ export async function fakeSlotgate(t, options = {}) { const held = commands.get(command.command_id) if (!held || held.ticket !== command.ticket) return socket.end('ERR ticket_invalid\n') held.envelope = envelope + held.headers = headers options.onForward?.(command, envelope) if (options.hold) return const payload = options.body || JSON.stringify(assistant) diff --git a/test/fixtures/real-slotgate.mjs b/test/fixtures/real-slotgate.mjs index e09262383..7c3bd9733 100644 --- a/test/fixtures/real-slotgate.mjs +++ b/test/fixtures/real-slotgate.mjs @@ -31,6 +31,8 @@ export async function realSlotgate(t) { }) const runtime = path.join(root, 'runtime') fs.mkdirSync(runtime, { mode: 0o700 }) + const kernelTokenFile = path.join(runtime, 'internal.token') + fs.writeFileSync(kernelTokenFile, 'synthetic-kernel-internal-token\n', { mode: 0o600 }) process = spawn(binary, [runtime], { stdio: ['ignore', 'pipe', 'pipe'] }) exited = once(process, 'exit') return new Promise((resolve, reject) => { @@ -45,7 +47,7 @@ export async function realSlotgate(t) { const newline = buffer.indexOf('\n') if (newline < 0) return clearTimeout(timer) - try { resolve(JSON.parse(buffer.slice(0, newline))) } catch (error) { reject(error) } + try { resolve({ ...JSON.parse(buffer.slice(0, newline)), kernelTokenFile }) } catch (error) { reject(error) } }) }) } diff --git a/test/fixtures/slotgate-client-node.go b/test/fixtures/slotgate-client-node.go index 26ea08b71..25ebd10f6 100644 --- a/test/fixtures/slotgate-client-node.go +++ b/test/fixtures/slotgate-client-node.go @@ -47,6 +47,11 @@ func main() { unix, err := net.Listen("unix", filepath.Join(run, "kernel.sock")) check(err) kernel := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Kin-Internal-Token") != "synthetic-kernel-internal-token" || r.Header.Get("X-Internal-Token") != "synthetic-kernel-internal-token" { + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprint(w, `{"type":"error","error":{"code":"internal_unauthorized"}}`) + return + } var envelope struct { Body struct { Mode string `json:"mode"` diff --git a/test/unit/slotgate-client.test.mjs b/test/unit/slotgate-client.test.mjs index bafee5e88..53dfc2647 100644 --- a/test/unit/slotgate-client.test.mjs +++ b/test/unit/slotgate-client.test.mjs @@ -1,5 +1,8 @@ import test from 'node:test' import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' import { SlotgateClient } from '../../src/lib/transport/slotgate-client.mjs' import { callGoWorker, streamGoWorker, workerHealth } from '../../src/lib/transport/go-worker-client.mjs' import { dispatchCallInference } from '../../src/lib/transport/kernel-router.mjs' @@ -76,6 +79,20 @@ test('complete HTTP body remains uncertain until original-ticket node status is assert.equal(fx.requests[2].ticket, fx.requests[1].ticket) }) +test('KSG2 forwards the existing kernel internal authentication headers', async (t) => { + const fx = await fakeSlotgate(t) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ksg2-token-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const tokenFile = path.join(root, 'internal.token') + fs.writeFileSync(tokenFile, 'synthetic-kernel-internal-token\n', { mode: 0o600 }) + const exec = execFor(fx.config) + exec.vm.runtime.worker_token_file = tokenFile + await call(fx.config, { exec, requestId: 'authenticated-job' }) + const headers = fx.commands.get('authenticated-job').headers + assert.match(headers, /x-kin-internal-token: synthetic-kernel-internal-token\r\n/i) + assert.match(headers, /x-internal-token: synthetic-kernel-internal-token\r\n/i) +}) + for (const mode of ['pending', 'unknown', 'mismatch', 'unavailable']) { test(`HTTP success with ${mode} node proof retains execution and does not resend`, async (t) => { const fx = await fakeSlotgate(t, { diff --git a/test/unit/slotgate-pg-binding.test.mjs b/test/unit/slotgate-pg-binding.test.mjs index 0601c3ce6..e0feca824 100644 --- a/test/unit/slotgate-pg-binding.test.mjs +++ b/test/unit/slotgate-pg-binding.test.mjs @@ -111,7 +111,7 @@ test('the actual Go KSG2 node persists normal completion but retains an internal const lease = await first.tryReserve(candidate) assert.equal(lease.ok, true) const result = await callGoWorker({ - exec: { vmId: config.vmId, slotgate: config }, + exec: { vmId: config.vmId, slotgate: config, vm: { runtime: { worker_token_file: config.kernelTokenFile } } }, envelope: { body: { mode } }, onExecutionStart: lease.bindExecution, }) From ef018d2d9447e9eb6cd05599e4d93339c46ea93a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:18:42 +0800 Subject: [PATCH 291/473] fix(slotgate): preserve kernel authentication on fenced forwards --- src/lib/transport/go-worker-client.mjs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index 6869247a4..1f4f43286 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -374,6 +374,17 @@ async function completeFencedExecution(client, handle, execution) { } } +function forwardFencedWorker(exec, client, handle, request) { + const token = readInternalToken(exec) + return client.forward(handle, { + ...request, + headers: { + ...request.headers, + ...(token ? { 'x-kin-internal-token': token, 'x-internal-token': token } : {}), + }, + }) +} + function completeHttpExecution(execution, body, httpStatus) { const uncertain = kernelExecutionUnconfirmed(body, httpStatus) Object.assign(execution, { @@ -651,7 +662,7 @@ export async function callGoWorker({ signal, timeoutMs, } - const response = await (fenced ? fenced.forward(handle, request) : workerRequest(exec, request)) + const response = await (fenced ? forwardFencedWorker(exec, fenced, handle, request) : workerRequest(exec, request)) const data = await readAll(response) const parsed = parseJson(data) if (fenced) await completeFencedExecution(fenced, handle, execution) @@ -862,7 +873,7 @@ export async function streamGoWorker({ timeoutMode: 'first-byte', headers: { te: 'trailers' }, } - const response = await (fenced ? fenced.forward(handle, request) : workerRequest(exec, request)) + const response = await (fenced ? forwardFencedWorker(exec, fenced, handle, request) : workerRequest(exec, request)) const headers = mergeRateLimitHeaders(publicHeaders(response.headers)) if ((response.statusCode || 0) < 200 || (response.statusCode || 0) >= 300) { const data = await readAll(response, 1024 * 1024) From 08a2d9d4910e36890b8f5038bbd781ce6035457d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:18:42 +0800 Subject: [PATCH 292/473] fix: expose definitive credential attempt completion metadata --- src/lib/protocol/codex-models.mjs | 25 +++++++++++++++++++------ src/lib/transport/slot-oauth.mjs | 7 +++++++ test/unit/slot-oauth.test.mjs | 3 +++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/lib/protocol/codex-models.mjs b/src/lib/protocol/codex-models.mjs index 17ecebc40..084cf16e6 100644 --- a/src/lib/protocol/codex-models.mjs +++ b/src/lib/protocol/codex-models.mjs @@ -193,7 +193,7 @@ export function makeProxyFetch(proxyUrl, timeoutMs = 15000, { maxMs = 30000 } = */ export async function refreshCodexAccessToken(opts = {}) { const refreshToken = String(opts.refreshToken || '').trim() - if (!refreshToken) return { ok: false, error: 'missing_refresh_token' } + if (!refreshToken) return { ok: false, error: 'missing_refresh_token', attemptFinished: true } const fetchFn = opts.fetchImpl || makeProxyFetch(opts.proxyUrl, opts.timeoutMs) const body = new URLSearchParams({ grant_type: 'refresh_token', @@ -212,16 +212,29 @@ export async function refreshCodexAccessToken(opts = {}) { body: String(body), }) const status = Number(res?.status) || 0 - const payload = await readFetchBody(res) + // Consume the actual body before accepting even a rejection as terminal. + // readFetchBody intentionally tolerates malformed catalog responses; refresh cannot. + let payload + if (typeof res?.text === 'function') { + const text = await res.text() + try { + payload = text ? JSON.parse(text) : null + } catch { + payload = null + } + } else if (typeof res?.json === 'function') payload = await res.json() + else return { ok: false, error: 'refresh_empty', status, attemptFinished: false } if (status === 401 || status === 403) { - return { ok: false, error: 'refresh_rejected', status } + return { ok: false, error: 'refresh_rejected', status, attemptFinished: true } } - if (status >= 400) return { ok: false, error: 'refresh_failed', status } + if (status >= 400) return { ok: false, error: 'refresh_failed', status, attemptFinished: true } const access = String(payload?.access_token || '').trim() - if (!access) return { ok: false, error: 'refresh_empty', status } + if (!access || status < 200 || status >= 300) + return { ok: false, error: 'refresh_empty', status, attemptFinished: false } const expiresIn = Number(payload?.expires_in) return { ok: true, + attemptFinished: true, access_token: access, refresh_token: String(payload?.refresh_token || refreshToken).trim() || refreshToken, id_token: String(payload?.id_token || '').trim(), @@ -229,7 +242,7 @@ export async function refreshCodexAccessToken(opts = {}) { } } catch (e) { const aborted = e?.name === 'AbortError' || /aborted/i.test(String(e?.message || e)) - return { ok: false, error: aborted ? 'timeout' : 'fetch_failed' } + return { ok: false, error: aborted ? 'timeout' : 'fetch_failed', attemptFinished: false } } } diff --git a/src/lib/transport/slot-oauth.mjs b/src/lib/transport/slot-oauth.mjs index 4e77c416b..66a85c4ec 100644 --- a/src/lib/transport/slot-oauth.mjs +++ b/src/lib/transport/slot-oauth.mjs @@ -17,6 +17,7 @@ function failure(code, message = undefined) { headers: {}, via: 'slot-worker', transportError: true, + attemptFinished: false, } } @@ -136,6 +137,12 @@ export async function runSlotOauth( body: payload.body && typeof payload.body === 'object' ? payload.body : {}, headers: payload.headers && typeof payload.headers === 'object' ? payload.headers : {}, via: 'slot-worker', + attemptFinished: + result?.code === 0 && + payload.body?.error?.code !== 'upstream_transport_error' && + ((payload.ok && status >= 200 && status < 300) || + status >= 400 || + ['config_failed', 'proxy_required', 'credential_required'].includes(payload.body?.error?.code)), } if (status === 0) response.transportError = true return response diff --git a/test/unit/slot-oauth.test.mjs b/test/unit/slot-oauth.test.mjs index 4c6976b02..60cf4f692 100644 --- a/test/unit/slot-oauth.test.mjs +++ b/test/unit/slot-oauth.test.mjs @@ -68,6 +68,7 @@ test('runSlotOauth builds the worker argv and sends count-tokens stdin', async ( body: { input_tokens: 12 }, headers: { 'x-test': 'ok' }, via: 'slot-worker', + attemptFinished: true, }) }) @@ -138,6 +139,7 @@ test('runSlotOauth maps timeout to a transport error', async () => { headers: {}, via: 'slot-worker', transportError: true, + attemptFinished: false, }) }) @@ -157,6 +159,7 @@ test('runSlotOauth reports invalid worker output with stderr context', async () headers: {}, via: 'slot-worker', transportError: true, + attemptFinished: false, }) }) From 6ab5a505a93ead45cb5804fe41ad6be2220f1453 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:19:10 +0800 Subject: [PATCH 293/473] test: bound refresh admission context to its exact active callback --- test/unit/shared-refresh-context.test.mjs | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test/unit/shared-refresh-context.test.mjs diff --git a/test/unit/shared-refresh-context.test.mjs b/test/unit/shared-refresh-context.test.mjs new file mode 100644 index 000000000..955270da6 --- /dev/null +++ b/test/unit/shared-refresh-context.test.mjs @@ -0,0 +1,27 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import * as refresh from '../../src/lib/oauth/shared-credential-refresh.mjs' + +const db = { schema: 'synthetic', connectionString: 'synthetic-connection' } +const admission = { admissionId: 'admission', ownerId: 'owner', vmId: 'vm', accountId: 'account' } + +test('refresh context carries only the exact database and active admission callback', async () => { + assert.equal(refresh.currentRefreshAdmission(db), null) + await refresh.withRefreshAdmission(db, admission, async () => { + assert.deepEqual(refresh.currentRefreshAdmission(db), { db, ...admission }) + assert.equal(refresh.currentRefreshAdmission({ ...db, schema: 'other' }), null) + assert.equal(refresh.currentRefreshAdmission({ ...db, connectionString: 'other' }), null) + }) + assert.equal(refresh.currentRefreshAdmission(db), null) +}) + +test('detached descendants cannot reuse a completed refresh admission scope', async () => { + let release + const wait = new Promise((resolve) => { release = resolve }) + let detached + await refresh.withRefreshAdmission(db, admission, async () => { + detached = wait.then(() => refresh.currentRefreshAdmission(db)) + }) + release() + assert.equal(await detached, null) +}) From 9481ad1ad21d40546a2836c1558a53b5cbbfa7d6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:19:46 +0800 Subject: [PATCH 294/473] test(pool): prevent binding jobs during an active refresh VM operation --- .../execution-binding-vm-operation.test.mjs | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/unit/execution-binding-vm-operation.test.mjs diff --git a/test/unit/execution-binding-vm-operation.test.mjs b/test/unit/execution-binding-vm-operation.test.mjs new file mode 100644 index 000000000..d59d34cb4 --- /dev/null +++ b/test/unit/execution-binding-vm-operation.test.mjs @@ -0,0 +1,46 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import { createDatabase, getDb, withTransaction } from '../../src/lib/db/database.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + t.after(() => peer.close()) + const first = new ExecutionAdmissionsRepo(db) + const gate = await first.tryReserve({ accountId: 'refresh-account', vmId: 'refresh-vm', + accountLimit: 1, vmLimit: 1, rpmLimit: 0, fableLimit: 0 }) + await gate.bindExecution({ requestId: 'completed-before-refresh' }) + await gate.completeExecution({ requestId: 'completed-before-refresh', started: true, ended: true, reason: 'eof' }) + return { db, peer, first, gate, operations: new SharedVmOperations(peer) } +} + +test('a held refresh VM operation rejects another job on its exact completed admission', async (t) => { + const { db, peer, gate, operations } = await fixture(t) + const operation = await operations.tryAcquire({ vmId: gate.vmId, accountId: gate.accountId, + kind: 'oauth_refresh', admission: { db, ...gate } }) + try { + assert.equal(operation.ok, true) + await assert.rejects(gate.bindExecution({ requestId: 'must-not-overlap-refresh' }), { code: 'vm_operation_active' }) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 1) + } finally { await operation.release?.() } + await gate.bindExecution({ requestId: 'allowed-after-refresh' }) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 2) +}) + +test('binding waits for an in-progress management transaction before checking its committed VM operation', async (t) => { + const { db, peer, gate, operations } = await fixture(t) + let binding + await withTransaction(peer, async () => { + await operations.executions.lockResources({ accountId: gate.accountId, vmIds: [gate.vmId] }) + await peer.prepare('INSERT INTO vm_operations (id,vm_id,owner_id,kind) VALUES ($1,$2,$3,$4)') + .run('synthetic-uncommitted-operation', gate.vmId, 'synthetic-operation-owner', 'oauth_refresh') + binding = gate.bindExecution({ requestId: 'wait-for-management-commit' }) + const observed = await Promise.race([binding.then(() => 'bound', () => 'rejected'), delay(200).then(() => 'waiting')]) + assert.equal(observed, 'waiting', 'a new job cannot pass the management transaction using only its admission row') + }) + await assert.rejects(binding, { code: 'vm_operation_active' }) + assert.equal((await db.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 1) +}) From c906f02d66488afc0ea54872898150dc31fd1ad6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:20:23 +0800 Subject: [PATCH 295/473] test: require shared claims around actual worker credential commands --- test/unit/slot-oauth-shared.test.mjs | 85 ++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 test/unit/slot-oauth-shared.test.mjs diff --git a/test/unit/slot-oauth-shared.test.mjs b/test/unit/slot-oauth-shared.test.mjs new file mode 100644 index 000000000..df3ec8e96 --- /dev/null +++ b/test/unit/slot-oauth-shared.test.mjs @@ -0,0 +1,85 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import pg from 'pg' +import { getDb, registerActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { readWorkerCredentialFile, writeWorkerCredentialFile } from '../../src/lib/oauth/oauth-credentials.mjs' +import { runSlotOauth } from '../../src/lib/transport/slot-oauth.mjs' + +async function fixture(t) { + const db = getDb() + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + registerActivityCoordinator(db, coordinator) + const bootstrap = await coordinator.admitBootstrap() + await coordinator.acknowledgeEpoch(bootstrap.epoch) + await bootstrap.complete({ ended: true }) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-slot-refresh-')) + const homeDir = path.join(root, 'vms', 'vm-01', 'cli-home') + writeWorkerCredentialFile(homeDir, { + type: 'oauth', access_token: 'SYNTHETIC_AT', refresh_token: 'SYNTHETIC_RT', expires_at: 1, + account_uuid: 'slot-refresh-account', + }) + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(async () => { + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + await coordinator.stop() + const client = new pg.Client({ connectionString: db.connectionString }) + await client.connect() + try { await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) } + finally { await client.end(); fs.rmSync(root, { recursive: true, force: true }) } + }) + const run = (callback) => coordinator.runActivity({ kind: 'fixture' }, async () => ({ ended: true, value: await callback() })) + return { db, coordinator, homeDir, exec: { vmId: 'vm-01', homeDir, vm: {} }, run } +} + +test('the actual slot OAuth refresh shares a committed attempt after bootstrap has closed', async (t) => { + const f = await fixture(t) + let calls = 0 + const runDocker = async () => { + calls++ + await new Promise((resolve) => setTimeout(resolve, 30)) + writeWorkerCredentialFile(f.homeDir, { + ...readWorkerCredentialFile(f.homeDir), access_token: 'SYNTHETIC_NEW_AT', refresh_token: 'SYNTHETIC_NEW_RT', + }) + return { code: 0, stdout: '{"ok":true,"status":200,"body":{"refreshed":true}}' } + } + const results = await Promise.all([runSlotOauth(f.exec, 'refresh', { runDocker }), runSlotOauth(f.exec, 'refresh', { runDocker })]) + assert.equal(calls, 1) + assert.ok(results.every((result) => result.ok)) + const rows = await f.run(() => f.db.query('SELECT finished_at,shared_result FROM background_task_runs')) + assert.equal(rows.rows.length, 1) + assert.ok(rows.rows[0].finished_at) + assert.equal(JSON.stringify(rows.rows).includes('SYNTHETIC_'), false) +}) + +test('a slot refresh timeout retains its shared operation and task', async (t) => { + const f = await fixture(t) + const result = await runSlotOauth(f.exec, 'refresh', { runDocker: async () => ({ timed_out: true }) }) + assert.equal(result.attemptFinished, false) + const rows = await f.run(() => f.db.query('SELECT released_at,uncertain_at FROM vm_operations')) + assert.equal(rows.rows.length, 1) + assert.equal(rows.rows[0].released_at, null) + assert.ok(rows.rows[0].uncertain_at) +}) + +test('implicit worker refresh commands cannot execute while another inference is active', async (t) => { + const f = await fixture(t) + const gate = await f.run(() => new ExecutionAdmissionsRepo(f.db).tryReserve({ + accountId: 'slot-refresh-account', vmId: 'vm-01', accountLimit: 2, vmLimit: 2, rpmLimit: 0, fableLimit: 0, + })) + let calls = 0 + try { + await assert.rejects(runSlotOauth(f.exec, 'models', { runDocker: async () => { + calls++ + return { code: 0, stdout: '{"ok":true,"status":200}' } + } }), { code: 'vm_execution_active' }) + assert.equal(calls, 0) + } finally { await f.run(() => gate.release()) } +}) From e1e5d2a9e6cea391fb96aacc6b0757fa3ad0360f Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:20:45 +0800 Subject: [PATCH 296/473] fix(pool): serialize job binding with refresh VM operations --- .../2026-10-21-node-fencing-launch-adapter.md | 10 ++++-- .../db/repos/execution-admissions-repo.mjs | 19 ++++++++++- .../execution-binding-vm-operation.test.mjs | 33 +++++++++++++++---- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/docs/plans/2026-10-21-node-fencing-launch-adapter.md b/docs/plans/2026-10-21-node-fencing-launch-adapter.md index 92a82ede1..449f60267 100644 --- a/docs/plans/2026-10-21-node-fencing-launch-adapter.md +++ b/docs/plans/2026-10-21-node-fencing-launch-adapter.md @@ -43,7 +43,13 @@ KSG2 增加 `launch` 操作。客户端只提供 `vm_id`、创建操作时捕获 正常终态仅在 `CompleteCommand` 持久成功后成为 node 的 `completed`。新增只读 `command` RPC，要求原 captured fence、command ID、目标 kind 与 ticket，并只向已认证 token 返回状态，不暴露内部 nonce。客户端不能提交完成状态。 -已转发的 HTTP body 不等于持久确认。未来 KSG2 客户端必须在释放 PG job 前取得 node 的持久 `completed`。若 node 写盘失败，状态查询会失败，不能据 HTTP 成功释放。此客户端与 PG 绑定尚未接线。 +已转发的 HTTP body 不等于持久确认。`SlotgateClient` 在完整读取响应后使用原 captured fence、command ID 与 ticket 查询 node 的持久状态。只有 `completed` 且 `dispatched=true` 才能确认执行结束。node 写盘失败、状态不可用、身份不符或 unknown 都会保留 PG job；pending 只允许短时重读原状态，不会重发执行。 + +该客户端仅在可信调用方显式注入 `exec.slotgate` 时使用。配置必须包含 `trusted=true`、`pgBindingReady=true`、node/VM ID、immutable Docker fence 以及固定证书与 token 的 TLS link。构造时按值捕获并冻结 fence。generation 在控制面与 PG 使用无损十进制字符串，在 KSG2 wire 使用 uint64 数值。未知或多余身份字段拒绝。 + +发送前的 `onExecutionStart` 必须先持久 `execution_jobs.node_binding`，并返回相同 request ID 和 node binding 的确认。绑定失败或确认不符时不会调用 node prepare/forward。每次 forward 的 command ID 与 kernel envelope 的 request ID 相同。完整响应后取得的原 node `completed` 保存到 `node_completion`；PG 不接受本地 EOF 或其他 generation 的完成声明。PG 完成记录失败仍保留 pending，不会重试执行。 + +生产 capability 来源和节点权限迁移尚未接入，现有部署没有自动启用该路径。显式 fenced executor 拒绝旧 UDS/SSH 辅助入口；health、凭据管理、初始 allocate 与文件发布尚需 typed adapter。controller 在 prepare/forward 之后崩溃仍会保留未知作业，本批没有提供自动恢复或自动删除 prepared command。 ## 容器与文件权限 @@ -67,7 +73,7 @@ Authority journal、其父目录、节点 agent 的可执行文件与配置， - 把容器生命周期和文件写权限迁移到节点服务。runtime Pod 不得继续持有 root/docker-group SSH、raw Docker socket、可写 SFTP 或替换 gate 的权限。bootstrap 凭据须与 runtime 分离。 - 迁移时撤销旧授权，并关闭已经建立的 SSH、SFTP、终端和 forwarding 会话。只修改新连接的认证方式不能阻止旧 owner 的延迟命令。 -- 将 KSG2、captured generation、PG admission/job/VM operation 和文件发布接入生产调用链。fenced 节点不得回退到 KSG1 或原始 SSH 路径。 +- 将已具备的 KSG2 客户端与 PG job 绑定接到可信 capability、captured VM operation 和文件发布生产调用链。fenced 节点不得回退到 KSG1 或原始 SSH 路径。 - 提供节点可信 allocate、文件发布、未知 launch 的有证据恢复，以及尚未覆盖的响应终态处理。恢复只能关闭同一 immutable executor/incarnation 映射的作业；不能靠 TTL、ready_slots 或 cancel ACK 释放。 - 单独处理 KVM、原生 macOS 和 Codex 进程归属。现有 Codex Docker 容器仅持有身份和终端，停止它不能证明控制面 detached Codex 子进程已经终止。 - 在两个不同节点上的控制面验证同时执行、旧命令迟到、节点服务崩溃、Engine 故障及安全接管。本批合成测试不能代替该验收。 diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 3f4a52ca8..175076600 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -195,14 +195,31 @@ export class ExecutionAdmissionsRepo { async bindExecution(handle, execution) { const nodeBinding = execution?.nodeBinding === undefined ? null : normalizeNodeBinding(execution.nodeBinding) return withTransaction(this.db, async () => { + const before = await this.db + .prepare('SELECT account_id, vm_id FROM execution_admissions WHERE id = $1 AND owner_id = $2') + .get(handle.admissionId, handle.ownerId) + if (!before) throw jobError('execution_admission_inactive') + // Refresh may hold this exact admission after its previous job ended. + // Serialize with that operation before taking the admission row lock. + await this.lockResources({ accountId: before.account_id, vmIds: [before.vm_id] }) const admission = await this._lockAdmission(handle) - if (!admission || admission.released_at || admission.uncertain_at) { + if ( + !admission || + admission.released_at || + admission.uncertain_at || + admission.account_id !== before.account_id || + admission.vm_id !== before.vm_id + ) { throw jobError('execution_admission_inactive') } if (typeof execution?.requestId !== 'string' || !execution.requestId.trim()) { throw jobError('invalid_execution_job') } if (nodeBinding && nodeBinding.vmId !== admission.vm_id) throw jobError('execution_node_binding_mismatch') + const operation = await this.db + .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') + .get(physicalVmAliases(admission.vm_id)) + if (operation) throw jobError('vm_operation_active') const pending = await this.db .prepare('SELECT request_id FROM execution_jobs WHERE admission_id = $1 AND ended_at IS NULL LIMIT 1') .get(handle.admissionId) diff --git a/test/unit/execution-binding-vm-operation.test.mjs b/test/unit/execution-binding-vm-operation.test.mjs index d59d34cb4..a72cfd92f 100644 --- a/test/unit/execution-binding-vm-operation.test.mjs +++ b/test/unit/execution-binding-vm-operation.test.mjs @@ -10,8 +10,14 @@ async function fixture(t) { const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) t.after(() => peer.close()) const first = new ExecutionAdmissionsRepo(db) - const gate = await first.tryReserve({ accountId: 'refresh-account', vmId: 'refresh-vm', - accountLimit: 1, vmLimit: 1, rpmLimit: 0, fableLimit: 0 }) + const gate = await first.tryReserve({ + accountId: 'refresh-account', + vmId: 'refresh-vm', + accountLimit: 1, + vmLimit: 1, + rpmLimit: 0, + fableLimit: 0, + }) await gate.bindExecution({ requestId: 'completed-before-refresh' }) await gate.completeExecution({ requestId: 'completed-before-refresh', started: true, ended: true, reason: 'eof' }) return { db, peer, first, gate, operations: new SharedVmOperations(peer) } @@ -19,13 +25,19 @@ async function fixture(t) { test('a held refresh VM operation rejects another job on its exact completed admission', async (t) => { const { db, peer, gate, operations } = await fixture(t) - const operation = await operations.tryAcquire({ vmId: gate.vmId, accountId: gate.accountId, - kind: 'oauth_refresh', admission: { db, ...gate } }) + const operation = await operations.tryAcquire({ + vmId: gate.vmId, + accountId: gate.accountId, + kind: 'oauth_refresh', + admission: { db, ...gate }, + }) try { assert.equal(operation.ok, true) await assert.rejects(gate.bindExecution({ requestId: 'must-not-overlap-refresh' }), { code: 'vm_operation_active' }) assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 1) - } finally { await operation.release?.() } + } finally { + await operation.release?.() + } await gate.bindExecution({ requestId: 'allowed-after-refresh' }) assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 2) }) @@ -35,10 +47,17 @@ test('binding waits for an in-progress management transaction before checking it let binding await withTransaction(peer, async () => { await operations.executions.lockResources({ accountId: gate.accountId, vmIds: [gate.vmId] }) - await peer.prepare('INSERT INTO vm_operations (id,vm_id,owner_id,kind) VALUES ($1,$2,$3,$4)') + await peer + .prepare('INSERT INTO vm_operations (id,vm_id,owner_id,kind) VALUES ($1,$2,$3,$4)') .run('synthetic-uncommitted-operation', gate.vmId, 'synthetic-operation-owner', 'oauth_refresh') binding = gate.bindExecution({ requestId: 'wait-for-management-commit' }) - const observed = await Promise.race([binding.then(() => 'bound', () => 'rejected'), delay(200).then(() => 'waiting')]) + const observed = await Promise.race([ + binding.then( + () => 'bound', + () => 'rejected', + ), + delay(200).then(() => 'waiting'), + ]) assert.equal(observed, 'waiting', 'a new job cannot pass the management transaction using only its admission row') }) await assert.rejects(binding, { code: 'vm_operation_active' }) From 36da391a78dc1e2769f733d453ed297a7564b1a0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:21:27 +0800 Subject: [PATCH 297/473] feat: guard worker OAuth commands with shared refresh and VM claims --- src/lib/oauth/shared-credential-refresh.mjs | 25 +++++- src/lib/transport/slot-oauth.mjs | 98 ++++++++++++++++++++- test/unit/credential-refresh-proof.test.mjs | 16 ++-- test/unit/shared-refresh-context.test.mjs | 4 +- test/unit/slot-oauth-shared.test.mjs | 55 +++++++++--- 5 files changed, 173 insertions(+), 25 deletions(-) diff --git a/src/lib/oauth/shared-credential-refresh.mjs b/src/lib/oauth/shared-credential-refresh.mjs index b775ce384..5a879581d 100644 --- a/src/lib/oauth/shared-credential-refresh.mjs +++ b/src/lib/oauth/shared-credential-refresh.mjs @@ -1,9 +1,28 @@ import { createHash, randomUUID } from 'node:crypto' +import { AsyncLocalStorage } from 'node:async_hooks' import { getDb, getActivityCoordinator, withTransaction } from '../db/database.mjs' import { getSharedBackgroundTasks } from '../core/shared-background-tasks.mjs' import { SharedVmOperations, currentVmOperationIds } from '../vm/shared-vm-operations.mjs' const services = new WeakMap() +const admissionContext = new AsyncLocalStorage() + +export async function withRefreshAdmission(db, admission, callback) { + if (!admission?.admissionId) return admissionContext.run(null, callback) + const context = { db, admission: { ...admission, db }, active: true } + try { + return await admissionContext.run(context, callback) + } finally { + context.active = false + } +} + +export function currentRefreshAdmission(db = getDb()) { + const context = admissionContext.getStore() + if (!context?.active || context.db.schema !== db.schema || context.db.connectionString !== db.connectionString) + return null + return context.admission +} function failure(code) { return Object.assign(new Error(code.replaceAll('_', ' ')), { code, status: 409 }) @@ -118,7 +137,6 @@ export class SharedCredentialRefresh { outcome = await refresh(fresh) if (outcome?.finished !== true) { state.unknown = true - await claim.markUnknown() return { finished: false, unknown: true, @@ -149,7 +167,10 @@ export class SharedCredentialRefresh { { kind: 'credential_refresh', accountId, admission }, ), ) - if (result.unknown) return result + if (result.unknown) { + await claim.markUnknown() + return result + } if (!(await claim.complete({ sharedResult: metadata })).completed) throw failure('credential_refresh_completion_unconfirmed') return result diff --git a/src/lib/transport/slot-oauth.mjs b/src/lib/transport/slot-oauth.mjs index 66a85c4ec..d7426f535 100644 --- a/src/lib/transport/slot-oauth.mjs +++ b/src/lib/transport/slot-oauth.mjs @@ -2,6 +2,17 @@ import { spawn } from 'node:child_process' import path from 'node:path' import { slotHost } from '../vm/slot-host.mjs' import { slotExecArgv } from '../vm/slot-exec.mjs' +import { getDb, getActivityCoordinator } from '../db/database.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { getSharedBackgroundTasks } from '../core/shared-background-tasks.mjs' +import { SharedVmOperations } from '../vm/shared-vm-operations.mjs' +import { flushVmDbSync } from '../vm/vm-db-sync.mjs' +import { readWorkerCredentialFile } from '../oauth/oauth-credentials.mjs' +import { + credentialGeneration, + currentRefreshAdmission, + getSharedCredentialRefresh, +} from '../oauth/shared-credential-refresh.mjs' const WORKER_CONFIG = '/run/kin/worker.json' const WORKER_BIN = '/usr/local/bin/kin-worker' @@ -89,7 +100,92 @@ function lastJsonLine(stdout) { } } -export async function runSlotOauth( +export function runSlotOauth(exec, op, options = {}) { + if (!isKubernetesMode() || process.env.KIN_CRS_MOCK === '1') return runSlotOauthRaw(exec, op, options) + const db = options.db || getDb() + return getSharedBackgroundTasks(db).track(runSharedSlotOauth(db, exec, op, options)) +} + +async function runSharedSlotOauth(db, exec, op, options) { + const readCurrent = async () => { + const credential = readWorkerCredentialFile(exec.homeDir) + if (!credential) throw Object.assign(new Error('credential unavailable'), { code: 'credential_required' }) + return { credential, generation: credentialGeneration(credential) } + } + const current = await readCurrent() + const admission = options.admission || currentRefreshAdmission(db) + const vmId = exec.vmId || exec.vm?.id + const accountId = + options.accountId || + admission?.accountId || + current.credential.account_uuid || + exec.vm?.claude?.account_uuid || + vmId + if (op === 'refresh') { + const outcome = await getSharedCredentialRefresh(db).run( + { + platform: 'claude', + vmId, + accountId, + expectedGeneration: options.expectedGeneration || current.generation, + admission, + }, + { + readCurrent, + refresh: async () => { + const value = await runSlotOauthRaw(exec, op, options) + return { finished: value?.attemptFinished === true, value } + }, + publish: async (_previous, value) => { + await options.publishCredential?.(value) + await flushVmDbSync() + return readCurrent() + }, + }, + ) + if (outcome.unknown) + return { + ...(outcome.value || failure('credential_refresh_unknown')), + attemptFinished: false, + credentialRefreshUnknown: true, + } + if (!outcome.shared) return outcome.value + return { + ok: outcome.value?.ok === true, + status: outcome.value?.ok ? 200 : 400, + body: outcome.value?.ok ? { refreshed: false } : { error: { code: outcome.value?.error || 'refresh_failed' } }, + headers: {}, + via: 'slot-worker', + attemptFinished: true, + } + } + const activity = await getActivityCoordinator(db)?.admit({ kind: 'slot_oauth', requestId: op }) + let entered = false + let finished = false + let result + try { + const invoke = () => + new SharedVmOperations(db).run( + vmId, + async () => { + entered = true + result = await runSlotOauthRaw(exec, op, options) + if (result?.attemptFinished === true && exec.homeDir) + await slotHost(exec.vm).afterRefresh(exec.vm, path.dirname(exec.homeDir)) + await flushVmDbSync() + finished = result?.attemptFinished === true + return { ...result, vmOperationUncertain: result?.attemptFinished !== true } + }, + { kind: `oauth_${op}`, accountId, admission }, + ) + return await (activity ? activity.run(invoke) : invoke()) + } finally { + if (entered && !finished) await activity?.markUncertain('slot_oauth_unknown') + else await activity?.complete({ ended: true, reason: entered ? 'slot_oauth_completed' : 'slot_oauth_not_started' }) + } +} + +async function runSlotOauthRaw( exec, op, { body, headers, force = false, timeoutMs = 45000, runDocker = runDockerProcess } = {}, diff --git a/test/unit/credential-refresh-proof.test.mjs b/test/unit/credential-refresh-proof.test.mjs index 1e6666ef3..3d55b2ea6 100644 --- a/test/unit/credential-refresh-proof.test.mjs +++ b/test/unit/credential-refresh-proof.test.mjs @@ -4,13 +4,15 @@ import { runSlotOauth } from '../../src/lib/transport/slot-oauth.mjs' import { refreshCodexAccessToken } from '../../src/lib/protocol/codex-models.mjs' const exec = { vmId: 'vm-refresh-proof', vm: {} } -const worker = (payload, extra = {}) => async () => ({ - code: 0, - stdout: JSON.stringify(payload), - stderr: '', - timed_out: false, - ...extra, -}) +const worker = + (payload, extra = {}) => + async () => ({ + code: 0, + stdout: JSON.stringify(payload), + stderr: '', + timed_out: false, + ...extra, + }) test('a complete successful worker command proves its credential attempt finished', async () => { const result = await runSlotOauth(exec, 'refresh', { diff --git a/test/unit/shared-refresh-context.test.mjs b/test/unit/shared-refresh-context.test.mjs index 955270da6..52421ef4c 100644 --- a/test/unit/shared-refresh-context.test.mjs +++ b/test/unit/shared-refresh-context.test.mjs @@ -17,7 +17,9 @@ test('refresh context carries only the exact database and active admission callb test('detached descendants cannot reuse a completed refresh admission scope', async () => { let release - const wait = new Promise((resolve) => { release = resolve }) + const wait = new Promise((resolve) => { + release = resolve + }) let detached await refresh.withRefreshAdmission(db, admission, async () => { detached = wait.then(() => refresh.currentRefreshAdmission(db)) diff --git a/test/unit/slot-oauth-shared.test.mjs b/test/unit/slot-oauth-shared.test.mjs index df3ec8e96..7c31607f6 100644 --- a/test/unit/slot-oauth-shared.test.mjs +++ b/test/unit/slot-oauth-shared.test.mjs @@ -21,7 +21,10 @@ async function fixture(t) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-slot-refresh-')) const homeDir = path.join(root, 'vms', 'vm-01', 'cli-home') writeWorkerCredentialFile(homeDir, { - type: 'oauth', access_token: 'SYNTHETIC_AT', refresh_token: 'SYNTHETIC_RT', expires_at: 1, + type: 'oauth', + access_token: 'SYNTHETIC_AT', + refresh_token: 'SYNTHETIC_RT', + expires_at: 1, account_uuid: 'slot-refresh-account', }) const previous = process.env.VM2API_CONTROL_PLANE_MODE @@ -32,10 +35,15 @@ async function fixture(t) { await coordinator.stop() const client = new pg.Client({ connectionString: db.connectionString }) await client.connect() - try { await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) } - finally { await client.end(); fs.rmSync(root, { recursive: true, force: true }) } + try { + await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await client.end() + fs.rmSync(root, { recursive: true, force: true }) + } }) - const run = (callback) => coordinator.runActivity({ kind: 'fixture' }, async () => ({ ended: true, value: await callback() })) + const run = (callback) => + coordinator.runActivity({ kind: 'fixture' }, async () => ({ ended: true, value: await callback() })) return { db, coordinator, homeDir, exec: { vmId: 'vm-01', homeDir, vm: {} }, run } } @@ -46,11 +54,16 @@ test('the actual slot OAuth refresh shares a committed attempt after bootstrap h calls++ await new Promise((resolve) => setTimeout(resolve, 30)) writeWorkerCredentialFile(f.homeDir, { - ...readWorkerCredentialFile(f.homeDir), access_token: 'SYNTHETIC_NEW_AT', refresh_token: 'SYNTHETIC_NEW_RT', + ...readWorkerCredentialFile(f.homeDir), + access_token: 'SYNTHETIC_NEW_AT', + refresh_token: 'SYNTHETIC_NEW_RT', }) return { code: 0, stdout: '{"ok":true,"status":200,"body":{"refreshed":true}}' } } - const results = await Promise.all([runSlotOauth(f.exec, 'refresh', { runDocker }), runSlotOauth(f.exec, 'refresh', { runDocker })]) + const results = await Promise.all([ + runSlotOauth(f.exec, 'refresh', { runDocker }), + runSlotOauth(f.exec, 'refresh', { runDocker }), + ]) assert.equal(calls, 1) assert.ok(results.every((result) => result.ok)) const rows = await f.run(() => f.db.query('SELECT finished_at,shared_result FROM background_task_runs')) @@ -71,15 +84,29 @@ test('a slot refresh timeout retains its shared operation and task', async (t) = test('implicit worker refresh commands cannot execute while another inference is active', async (t) => { const f = await fixture(t) - const gate = await f.run(() => new ExecutionAdmissionsRepo(f.db).tryReserve({ - accountId: 'slot-refresh-account', vmId: 'vm-01', accountLimit: 2, vmLimit: 2, rpmLimit: 0, fableLimit: 0, - })) + const gate = await f.run(() => + new ExecutionAdmissionsRepo(f.db).tryReserve({ + accountId: 'slot-refresh-account', + vmId: 'vm-01', + accountLimit: 2, + vmLimit: 2, + rpmLimit: 0, + fableLimit: 0, + }), + ) let calls = 0 try { - await assert.rejects(runSlotOauth(f.exec, 'models', { runDocker: async () => { - calls++ - return { code: 0, stdout: '{"ok":true,"status":200}' } - } }), { code: 'vm_execution_active' }) + await assert.rejects( + runSlotOauth(f.exec, 'models', { + runDocker: async () => { + calls++ + return { code: 0, stdout: '{"ok":true,"status":200}' } + }, + }), + { code: 'vm_execution_active' }, + ) assert.equal(calls, 0) - } finally { await f.run(() => gate.release()) } + } finally { + await f.run(() => gate.release()) + } }) From e5bd87080d8599c8a5f8ef76f29103ab837a8327 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:21:36 +0800 Subject: [PATCH 298/473] test: keep attached proxy networks after confirmed helper shutdown --- test/unit/egress-stop-proof.test.mjs | 22 ++++++++++++ test/unit/panel-proxy-vm-mutations.test.mjs | 39 +++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs index 3a1a1f9b4..bf037c515 100644 --- a/test/unit/egress-stop-proof.test.mjs +++ b/test/unit/egress-stop-proof.test.mjs @@ -189,6 +189,28 @@ test('Docker network inspection failure cannot be treated as absent during proxy assert.equal(result.vmOperationUncertain, true) }) +test('a confirmed helper stop may retain the same network with known container endpoints', async (t) => { + const f = fixture(t) + let attempted = false + const result = await stopProxyEgress(f.root, 'synthetic', { + ...noDocker, + runDocker: (args) => { + if (args[1] === 'rm') { + attempted = true + return { + ok: false, + stderr: 'Error response from daemon: error while removing network: network synthetic has active endpoints', + } + } + const containers = attempted ? { immutableContainer: { Name: 'kin-synthetic' } } : {} + return { ok: true, stdout: '192.0.2.0/24|192.0.2.1|same-network-id|' + JSON.stringify(containers) } + }, + }) + assert.equal(result.ok, true) + assert.equal(result.network_retained, true) + assert.equal(result.ended, false) // standalone legacy compatibility is not a termination proof. +}) + for (const mode of ['normal', 'noop', 'replacement']) { test(`proxy network teardown verifies immutable ID and observed absence (${mode})`, async (t) => { const f = fixture(t) diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index ddfa79b59..18d74bf11 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -13,6 +13,7 @@ import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admiss import { SharedProxyOperations } from '../../src/lib/vm/shared-proxy-operations.mjs' import { buildVmPackage } from '../../src/lib/vm/vm-package.mjs' import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { stopProxyEgress } from '../../src/lib/vm/egress.mjs' async function fixture(t) { const project = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-proxy-claims-')) @@ -223,6 +224,44 @@ test('proxy unbind awaits confirmed helper stop before releasing its VM claim', await next.release() }) +test('last unbind can retain a known attached network after its helper has ended', async (t) => { + const f = await fixture(t) + let attempted = false + const h = f.handler( + {}, + { + vmMutationOps: { + stopProxyEgress: (root, id) => + stopProxyEgress(root, id, { + runIptables: () => ({ ok: true }), + runDocker: (args) => { + if (args[1] === 'rm') { + attempted = true + return { + ok: false, + stderr: + 'Error response from daemon: error while removing network: network synthetic has active endpoints', + } + } + return { + ok: true, + stdout: + '192.0.2.0/24|192.0.2.1|same-network-id|' + + JSON.stringify(attempted ? { immutableContainer: { Name: 'kin-a' } } : {}), + } + }, + }), + }, + }, + ) + await h.request('POST', '/api/panel/proxies/' + f.proxyId + '/unbind') + assert.equal(h.response.status, 200, JSON.stringify(h.response.body)) + assert.deepEqual(f.pool.snapshot().proxies[0].bound_vm_ids, []) + const next = await new SharedProxyOperations(f.peer).tryAcquire() + assert.equal(next.ok, true) + await next.release() +}) + test('package update rejects global executions before its first metadata or credential write', async (t) => { const f = await fixture(t) const file = path.join(f.project, 'vms', 'vm-a.json') From 71a25183e606dee01a3742c890696d34aacfc2fe Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:21:49 +0800 Subject: [PATCH 299/473] test: reproduce bridge healing bypassing VM and proxy ownership --- test/unit/shared-bridge-heal.test.mjs | 80 +++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/unit/shared-bridge-heal.test.mjs diff --git a/test/unit/shared-bridge-heal.test.mjs b/test/unit/shared-bridge-heal.test.mjs new file mode 100644 index 000000000..680062248 --- /dev/null +++ b/test/unit/shared-bridge-heal.test.mjs @@ -0,0 +1,80 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { createSlotBridgeHealer } from '../../src/lib/vm/slot-recovery.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { SharedProxyOperations } from '../../src/lib/vm/shared-proxy-operations.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' +import { createKernelWatchdog } from '../../src/lib/transport/kernel-watchdog.mjs' + +const proxy = { id: 'bridge-proxy', scheme: 'socks5', host: '192.0.2.1', port: 1080 } +const vm = { id: 'bridge-vm', status: 'running', proxy } + +function healer(overrides = {}) { + const db = getDb() + let starts = 0 + return { + starts: () => starts, + heal: createSlotBridgeHealer({ + projectRoot: '/unused', + logger: { warn() {} }, + operations: new SharedVmOperations(db), + proxyOperations: new SharedProxyOperations(db), + ops: { + getVm: (_root, id) => ({ ...vm, id }), + egressAlive: () => false, + ensureProxyEgress: async () => { starts++; return { ok: true } }, + cluster: () => null, + ...overrides, + }, + }), + } +} + +test('bridge healing cannot bypass an existing VM mutation', async () => { + const db = getDb() + const held = await new SharedVmOperations(db).tryAcquire({ vmId: vm.id }) + const fx = healer() + try { + await assert.rejects(fx.heal([vm]), { code: 'vm_operation_active' }) + assert.equal(fx.starts(), 0) + const pool = await new SharedProxyOperations(db).tryAcquire() + assert.equal(pool.ok, true, 'a refusal before any helper work returns the pool claim') + await pool.release() + } finally { await held.release() } +}) + +test('bridge healing takes claims for other VMs sharing the same helper', async () => { + const db = getDb() + await db.prepare('INSERT INTO proxies (id, bound_vm_id) VALUES ($1,$2)').run(proxy.id, JSON.stringify([vm.id, 'other-bridge-vm'])) + const held = await new SharedVmOperations(db).tryAcquire({ vmId: 'other-bridge-vm' }) + const fx = healer() + try { + await assert.rejects(fx.heal([vm]), { code: 'vm_operation_active' }) + assert.equal(fx.starts(), 0) + } finally { await held.release() } +}) + +test('an unknown helper result is returned and preserves VM and pool claims', async () => { + const db = getDb() + const fx = healer({ ensureProxyEgress: async () => ({ ok: false, vmOperationUncertain: true }) }) + const result = await fx.heal([vm]) + assert.equal(result?.vmOperationUncertain, true) + assert.equal((await new SharedVmOperations(db).tryAcquire({ vmId: vm.id })).code, 'vm_operation_active') + assert.equal((await new SharedProxyOperations(db).tryAcquire()).code, 'proxy_operation_active') +}) + +test('watchdog bridge activity remains unknown when the healer cannot confirm completion', async () => { + const db = getDb() + const watchdog = createKernelWatchdog({ + listTargets: () => [], + sharedTasks: new SharedBackgroundTasks(db), + operations: new SharedVmOperations(db), + healBridges: async () => ({ ok: false, vmOperationUncertain: true }), + }) + await watchdog.tick() + await watchdog.flush() + const row = await db.prepare("SELECT unknown_at, finished_at FROM background_task_runs WHERE task_key='kernel_bridge_heal'").get() + assert.ok(row.unknown_at) + assert.equal(row.finished_at, null) +}) From f355d929d10964670aa06490b1fcc798e41562a5 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:22:54 +0800 Subject: [PATCH 300/473] test: coordinate quota recovery refreshes across requests --- .../unit/openai-quota-shared-refresh.test.mjs | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 test/unit/openai-quota-shared-refresh.test.mjs diff --git a/test/unit/openai-quota-shared-refresh.test.mjs b/test/unit/openai-quota-shared-refresh.test.mjs new file mode 100644 index 000000000..6b139730e --- /dev/null +++ b/test/unit/openai-quota-shared-refresh.test.mjs @@ -0,0 +1,75 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import pg from 'pg' +import { getDb, getActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' +import { queryOpenaiQuota } from '../../src/lib/oauth/openai-quota.mjs' + +async function fixture(t, callback) { + const db = getDb() + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-shared-refresh-')) + const vmId = 'vm-codex-refresh' + fs.mkdirSync(path.join(root, 'vms', vmId), { recursive: true }) + fs.writeFileSync(path.join(root, 'vms', vmId + '.json'), JSON.stringify({ + id: vmId, platform: 'openai', family: 'codex', proxy: { url: 'socks5h://127.0.0.1:19999' }, + })) + fs.writeFileSync(path.join(root, 'vms', vmId, 'codex-credentials.json'), JSON.stringify({ + accounts: [{ id: 'synthetic', access_token: 'SYNTHETIC_OLD_AT', refresh_token: 'SYNTHETIC_RT', chatgpt_account_id: 'synthetic-account' }], + })) + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(async () => { + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + const coordinator = getActivityCoordinator(db) + await coordinator?.stop() + const client = new pg.Client({ connectionString: db.connectionString }) + await client.connect() + try { if (coordinator) await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) } + finally { await client.end(); fs.rmSync(root, { recursive: true, force: true }) } + }) + await withDatabaseActivities([db], () => callback({ db, root, vmId })) +} + +const response = (status, payload) => ({ status, json: async () => payload }) + +test('concurrent quota 401 recovery shares one published refresh attempt', async (t) => fixture(t, async ({ db, root, vmId }) => { + let refreshes = 0 + const fetchImpl = async (url, init) => { + if (url.includes('/oauth/token')) { + refreshes++ + await new Promise((resolve) => setTimeout(resolve, 30)) + return response(200, { access_token: 'SYNTHETIC_NEW_AT', refresh_token: 'SYNTHETIC_NEW_RT' }) + } + if (init.headers.authorization === 'Bearer SYNTHETIC_OLD_AT') return response(401, {}) + return response(200, { rate_limit: {}, rate_limit_reset_credits: { available_count: 0 } }) + } + const results = await Promise.all([1, 2].map(() => queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }))) + assert.equal(refreshes, 1) + assert.ok(results.every((result) => result.ok)) + const rows = (await db.query('SELECT finished_at,shared_result FROM background_task_runs')).rows + assert.equal(rows.length, 1) + assert.ok(rows[0].finished_at) + assert.equal(JSON.stringify(rows).includes('SYNTHETIC_'), false) +})) + +test('quota refresh transport failure leaves both durable claims unavailable', async (t) => fixture(t, async ({ db, root, vmId }) => { + let refreshes = 0 + const fetchImpl = async (url) => { + if (url.includes('/oauth/token')) { refreshes++; throw new Error('synthetic token disconnect') } + return response(401, {}) + } + const first = await queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }) + assert.equal(first.ok, false) + await queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }) + assert.equal(refreshes, 1) + const task = (await db.query('SELECT finished_at,unknown_at FROM background_task_runs')).rows[0] + assert.equal(task.finished_at, null) + assert.ok(task.unknown_at) + const operation = (await db.query('SELECT released_at,uncertain_at FROM vm_operations')).rows[0] + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) +})) From c567d77e239069bb1ed043bbbfb8b0a18ee593ea Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:24:36 +0800 Subject: [PATCH 301/473] test: retain Search admission when credential refresh is uncertain --- test/unit/shared-codex-search.test.mjs | 71 +++++++++++++++++++++----- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/test/unit/shared-codex-search.test.mjs b/test/unit/shared-codex-search.test.mjs index a70b8ab12..895f4ebf5 100644 --- a/test/unit/shared-codex-search.test.mjs +++ b/test/unit/shared-codex-search.test.mjs @@ -7,7 +7,9 @@ import http from 'node:http' import { EventEmitter } from 'node:events' import { setTimeout as delay } from 'node:timers/promises' import fetch from 'node-fetch' -import { getDb } from '../../src/lib/db/database.mjs' +import pg from 'pg' +import { getDb, getActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' import { createRepositoryTestDatabase as createDatabase } from '../repository-database.mjs' import { handleCodexSearch } from '../../src/lib/protocol/handle-codex.mjs' import { SharedCodexRuntime } from '../../src/lib/pool/shared-codex-runtime.mjs' @@ -86,7 +88,18 @@ async function fixture(t, handle = (_req, res) => answer(res), { vms = 1 } = {}) t.after(async () => { server.closeAllConnections() await new Promise((resolve) => server.close(resolve)) + const coordinators = [db, other].map((connection) => getActivityCoordinator(connection)).filter(Boolean) + for (const coordinator of coordinators) await coordinator.stop() await other.close() + if (coordinators.length) { + const client = new pg.Client({ connectionString: db.connectionString }) + await client.connect() + try { + await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinators[0].schema)} CASCADE`) + } finally { + await client.end() + } + } fs.rmSync(root, { recursive: true, force: true }) if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = previous @@ -245,26 +258,56 @@ test('shared Search does not follow an HTTP redirect as an unregistered second j for (const status of [401, 403]) { test(`Search completes its ${status} job before refresh and retries with a new persisted ID`, async (t) => { const fx = await fixture(t, (_req, res, count) => answer(res, count === 1 ? status : 200)) + let refreshes = 0 + await withDatabaseActivities([fx.db, fx.other], async () => { + const result = await fx.run({ + refresh: async () => { + refreshes += 1 + const previous = await fx.other.prepare('SELECT * FROM execution_jobs').get() + assert.ok(previous?.ended_at, 'the first HTTP job must be durably complete before refresh/retry') + return { + ok: true, + attemptFinished: true, + access_token: 'synthetic-new-access', + refresh_token: 'synthetic-new-refresh', + } + }, + }) + assert.equal(result.res.statusCode, 200) + assert.equal(refreshes, 1) + assert.equal(fx.calls.length, 2) + assert.ok(fx.bindings.every(Boolean)) + assert.notEqual(fx.calls[0].headers['x-client-request-id'], fx.calls[1].headers['x-client-request-id']) + const jobs = await fx.other.prepare('SELECT * FROM execution_jobs').all() + assert.equal(jobs.length, 2) + assert.ok(jobs.every((job) => job.ended_at)) + assert.equal((await fx.counts()).accountInflight, 0) + }) + }) +} + +test('an uncertain Search token refresh retains its original admission and both refresh claims', async (t) => { + const fx = await fixture(t, (_req, res) => answer(res, 401)) + await withDatabaseActivities([fx.db, fx.other], async () => { let refreshes = 0 const result = await fx.run({ refresh: async () => { - refreshes += 1 - const previous = await fx.other.prepare('SELECT * FROM execution_jobs').get() - assert.ok(previous?.ended_at, 'the first HTTP job must be durably complete before refresh/retry') - return { ok: true, access_token: 'synthetic-new-access', refresh_token: 'synthetic-new-refresh' } + refreshes++ + return { ok: false, attemptFinished: false, error: 'fetch_failed' } }, }) - assert.equal(result.res.statusCode, 200) + assert.equal(result.res.statusCode, 502) assert.equal(refreshes, 1) - assert.equal(fx.calls.length, 2) - assert.ok(fx.bindings.every(Boolean)) - assert.notEqual(fx.calls[0].headers['x-client-request-id'], fx.calls[1].headers['x-client-request-id']) - const jobs = await fx.other.prepare('SELECT * FROM execution_jobs').all() - assert.equal(jobs.length, 2) - assert.ok(jobs.every((job) => job.ended_at)) - assert.equal((await fx.counts()).accountInflight, 0) + assert.equal(fx.calls.length, 1) + assert.equal((await fx.counts()).accountInflight, 1) + const operation = await fx.other.prepare('SELECT released_at,uncertain_at FROM vm_operations').get() + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + const task = await fx.other.prepare('SELECT finished_at,unknown_at FROM background_task_runs').get() + assert.equal(task.finished_at, null) + assert.ok(task.unknown_at) }) -} +}) for (const phase of ['timeout', 'cancel']) { test(`Search ${phase} after response headers retains its job and capacity`, async (t) => { From d824fa2ae56561dbd8fe8c90e7bbac3a515158df Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:25:00 +0800 Subject: [PATCH 302/473] fix: retain confirmed attached proxy networks after helper stop --- src/lib/vm/egress.mjs | 51 ++++++++++++++++++++++++++-- test/unit/egress-stop-proof.test.mjs | 46 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/src/lib/vm/egress.mjs b/src/lib/vm/egress.mjs index f0473c5f4..80ab9ed6b 100644 --- a/src/lib/vm/egress.mjs +++ b/src/lib/vm/egress.mjs @@ -322,7 +322,13 @@ export function readEgressListenTcp(projectRoot, proxyId) { export function inspectEgressNetwork(proxyId, run = docker, { strict = false, target = null } = {}) { const name = target || networkName(proxyId) if (!name) return null - const r = run(['network', 'inspect', '-f', '{{range .IPAM.Config}}{{.Subnet}}|{{.Gateway}}{{end}}|{{.Id}}', name]) + const r = run([ + 'network', + 'inspect', + '-f', + '{{range .IPAM.Config}}{{.Subnet}}|{{.Gateway}}{{end}}|{{.Id}}|{{json .Containers}}', + name, + ]) if (!r.ok) { if ( strict && @@ -334,12 +340,29 @@ export function inspectEgressNetwork(proxyId, run = docker, { strict = false, ta } return null } - const [subnet, gateway, id] = String(r.stdout || '').split('|') + const [subnet, gateway, id, attached] = String(r.stdout || '').split('|') if (strict && !id?.trim()) throw Object.assign(new Error('Docker network identity unavailable'), { code: 'egress_network_unknown' }) + let containers = null + if (attached) { + try { + containers = JSON.parse(attached) + if ( + !containers || + Array.isArray(containers) || + typeof containers !== 'object' || + Object.values(containers).some((item) => !item || typeof item.Name !== 'string' || !item.Name) + ) + throw new Error('Invalid Docker network endpoints') + } catch (error) { + if (strict) throw Object.assign(error, { code: 'egress_network_unknown' }) + containers = null + } + } return { name, id: id?.trim() || null, + containers, network: name, subnet: String(subnet || '').trim(), gateway: String(gateway || '').trim() || gatewayFromSubnet(String(subnet || '').trim()), @@ -1077,7 +1100,29 @@ export function stopProxyEgress(projectRoot, proxyId, { subnet, runDocker = dock const rules = removeIptables(plan, runIptables, { strict: true }) if (!rules.ok) return { ...rules, vmOperationUncertain: true } if (net) { - runDocker(['network', 'rm', net.id]) + const retained = () => { + const byId = inspectEgressNetwork(proxyId, runDocker, { strict: true, target: net.id }) + const byName = inspectEgressNetwork(proxyId, runDocker, { strict: true }) + if ( + byId?.id !== net.id || + byName?.id !== net.id || + !Object.keys(byId.containers || {}).length || + !Object.keys(byName.containers || {}).length + ) + throw new Error('Attached egress network identity was not confirmed') + return { ok: true, ended: stopped.ended, network_retained: true, ...(stopped.legacy ? { legacy: true } : {}) } + } + // Unbinding edits VM metadata; an idle container may intentionally remain + // attached until its next recreation. Its network is not an active helper. + if (Object.keys(net.containers || {}).length) return retained() + const removed = runDocker(['network', 'rm', net.id]) + if ( + !removed.ok && + /^Error response from daemon: error while removing network: network [^\r\n]+ has active endpoints(?:[^\r\n]*)$/i.test( + String(removed.stderr || '').trim(), + ) + ) + return retained() if ( inspectEgressNetwork(proxyId, runDocker, { strict: true, target: net.id }) || inspectEgressNetwork(proxyId, runDocker, { strict: true }) diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs index bf037c515..e9410edb1 100644 --- a/test/unit/egress-stop-proof.test.mjs +++ b/test/unit/egress-stop-proof.test.mjs @@ -211,6 +211,52 @@ test('a confirmed helper stop may retain the same network with known container e assert.equal(result.ended, false) // standalone legacy compatibility is not a termination proof. }) +test('known attached endpoints retain their network without attempting removal', async (t) => { + const f = fixture(t) + const result = await stopProxyEgress(f.root, 'synthetic', { + ...noDocker, + runDocker: (args) => { + assert.equal(args[1], 'inspect') + return { + ok: true, + stdout: '192.0.2.0/24|192.0.2.1|same-network-id|{"immutableContainer":{"Name":"kin-synthetic"}}', + } + }, + }) + assert.equal(result.ok, true) + assert.equal(result.network_retained, true) +}) + +for (const mode of ['replacement', 'permission', 'malformed']) { + test(`attached endpoints cannot conceal an unknown network identity (${mode})`, async (t) => { + const f = fixture(t) + let inspected = false + const result = await stopProxyEgress(f.root, 'synthetic', { + ...noDocker, + runDocker: () => { + if (!inspected) { + inspected = true + return { + ok: true, + stdout: '192.0.2.0/24|192.0.2.1|original-network-id|{"immutableContainer":{"Name":"kin-synthetic"}}', + } + } + if (mode === 'permission') return { ok: false, stderr: 'permission denied connecting to Docker daemon' } + return { + ok: true, + stdout: + '192.0.2.0/24|192.0.2.1|' + + (mode === 'replacement' ? 'replacement-network-id' : 'original-network-id') + + '|' + + (mode === 'malformed' ? '{"immutableContainer":{}}' : '{"immutableContainer":{"Name":"kin-synthetic"}}'), + } + }, + }) + assert.equal(result.ok, false) + assert.equal(result.vmOperationUncertain, true) + }) +} + for (const mode of ['normal', 'noop', 'replacement']) { test(`proxy network teardown verifies immutable ID and observed absence (${mode})`, async (t) => { const f = fixture(t) From a657c504879d3cc8cb2153d156589baa783c9a19 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:26:04 +0800 Subject: [PATCH 303/473] test: give synthetic Search backend observer its own SQL activity --- test/unit/shared-codex-search.test.mjs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/unit/shared-codex-search.test.mjs b/test/unit/shared-codex-search.test.mjs index 895f4ebf5..46985085c 100644 --- a/test/unit/shared-codex-search.test.mjs +++ b/test/unit/shared-codex-search.test.mjs @@ -77,10 +77,18 @@ async function fixture(t, handle = (_req, res) => answer(res), { vms = 1 } = {}) const chunks = [] for await (const chunk of req) chunks.push(chunk) calls.push({ headers: req.headers, body: JSON.parse(Buffer.concat(chunks).toString()) }) - bindings.push( - await other + const observe = () => + other .prepare('SELECT * FROM execution_jobs WHERE request_id = $1') - .get(req.headers['x-client-request-id'] || ''), + .get(req.headers['x-client-request-id'] || '') + const coordinator = getActivityCoordinator(other) + bindings.push( + await (coordinator + ? coordinator.runActivity({ kind: 'synthetic_backend_observer' }, async () => ({ + ended: true, + value: await observe(), + })) + : observe()), ) await handle(req, res, calls.length) })().catch((error) => res.destroy(error)) From 1a724e76c3717aee948cc209f83a49141a94f04e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:27:03 +0800 Subject: [PATCH 304/473] test: reproduce bootstrap usage timeout losing execution ownership --- test/unit/official-cc-bootstrap.test.mjs | 27 +++++++ .../official-cc-shared-lifecycle.test.mjs | 73 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/test/unit/official-cc-bootstrap.test.mjs b/test/unit/official-cc-bootstrap.test.mjs index aafb840e0..5dc1f59e0 100644 --- a/test/unit/official-cc-bootstrap.test.mjs +++ b/test/unit/official-cc-bootstrap.test.mjs @@ -688,6 +688,33 @@ test('slot /usage stops once Fable limits arrive', async () => { fs.rmSync(home, { recursive: true, force: true }) }) +test('slot /usage stops after an unconfirmed timeout instead of overwriting it with a retry', async (t) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-cc-usage-unknown-')) + t.after(() => fs.rmSync(home, { recursive: true, force: true })) + fs.mkdirSync(path.join(home, '.claude'), { recursive: true }) + let calls = 0 + const run = await runOfficialCcUsage({ + homeDir: home, + retryDelayMs: 0, + turn: async ({ outFile }) => { + calls++ + fs.writeFileSync( + outFile, + calls === 1 + ? '' + : usageLines([ + { kind: 'session', percent: 10 }, + { kind: 'weekly_all', percent: 30 }, + ]), + ) + return calls === 1 ? { ok: false, code: null, timed_out: true } : { ok: true, code: 0, timed_out: false } + }, + }) + assert.equal(calls, 1) + assert.equal(run.turn.timed_out, true) + assert.equal(run.attemptFinished, false) +}) + test('collectOfficialCcAccount reads tier from profile and model ids via slot worker', async () => { const ops = [] const slotOauth = async (_exec, op) => { diff --git a/test/unit/official-cc-shared-lifecycle.test.mjs b/test/unit/official-cc-shared-lifecycle.test.mjs index 07d9b3839..036d743f9 100644 --- a/test/unit/official-cc-shared-lifecycle.test.mjs +++ b/test/unit/official-cc-shared-lifecycle.test.mjs @@ -14,6 +14,7 @@ import { readOfficialCcStatus, scheduleOfficialCcBootstrap, } from '../../src/lib/oauth/official-cc-bootstrap.mjs' +import { writeWorkerCredentialFile } from '../../src/lib/oauth/oauth-credentials.mjs' async function fixture(t) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'official-shared-lifecycle-')) @@ -59,6 +60,78 @@ async function fixture(t) { return { db, peer, tasks, other, options, finish, vm, root } } +function prepareRawBootstrap(f, t) { + const home = officialCcHome(f.root, f.vm.id) + f.vm.claude.has_access = true + fs.writeFileSync(path.join(f.root, 'vms', f.vm.id + '.json'), JSON.stringify(f.vm)) + writeWorkerCredentialFile(home, { + mode: 'oauth', + access_token: 'synthetic-access', + expires_at: '2099-01-01T00:00:00Z', + }) + fs.mkdirSync(path.join(home, '.kin'), { recursive: true }) + fs.writeFileSync(path.join(home, '.kin', 'cli-node'), '#!/bin/sh\nprintf synthetic-version\n', { mode: 0o700 }) + const bin = path.join(f.root, 'bin') + fs.mkdirSync(bin) + // Every Docker call uses this synthetic executable; no daemon or VM is contacted. + fs.writeFileSync(path.join(bin, 'docker'), '#!/bin/sh\nprintf "{}"\n', { mode: 0o700 }) + const previous = process.env.PATH + process.env.PATH = bin + path.delimiter + previous + t.after(() => { + process.env.PATH = previous + }) + delete f.options.runBootstrap + Object.assign(f.options, { + wipe: false, + applySeed: false, + reconcileFingerprint: false, + syncTelemetry: false, + retryDelayMs: 0, + slotOauth: async (_exec, op) => ({ + ok: true, + attemptFinished: true, + body: op === 'profile' ? { account: {}, organization: {} } : { data: [] }, + }), + }) +} + +test('a raw bootstrap usage timeout keeps task and VM ownership and prevents a later retry', async (t) => { + const f = await fixture(t) + prepareRawBootstrap(f, t) + let calls = 0 + f.options.runTurn = async ({ prompt, outFile }) => { + if (prompt !== '/usage') return { ok: true, code: 0, timed_out: false } + calls++ + fs.writeFileSync( + outFile, + calls === 1 + ? '' + : [ + JSON.stringify({ + type: 'assistant', + usage_report: { + rate_limits: { + limits: [ + { kind: 'session', percent: 10 }, + { kind: 'weekly_all', percent: 30 }, + ], + }, + }, + }), + JSON.stringify({ type: 'result', result: 'ok' }), + ].join('\n'), + ) + return calls === 1 ? { ok: false, code: null, timed_out: true } : { ok: true, code: 0, timed_out: false } + } + await scheduleOfficialCcBootstrap(f.options) + await f.tasks.flush() + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) + const claim = await f.peer.prepare('SELECT unknown_at, finished_at FROM background_task_runs').get() + assert.notEqual(claim.unknown_at, null) + assert.equal(claim.finished_at, null) + assert.equal(calls, 1) +}) + test('manual bootstrap rejects a busy VM before queued status or background effects are written', async (t) => { const f = await fixture(t) await new ExecutionAdmissionsRepo(f.peer).tryReserve({ From 8f3ff3fb9fa2eb97fda6e72d2e87f98673769c1e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:27:49 +0800 Subject: [PATCH 305/473] test: persist account exclusion while a credential refresh remains active --- test/unit/shared-refresh-admission.test.mjs | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/unit/shared-refresh-admission.test.mjs b/test/unit/shared-refresh-admission.test.mjs index 59717bee1..fd4012bfc 100644 --- a/test/unit/shared-refresh-admission.test.mjs +++ b/test/unit/shared-refresh-admission.test.mjs @@ -78,6 +78,45 @@ test('a refresh can hold its exact completed admission while blocking every new } }) +test('an account refresh blocks inference and another mutation on a different VM', async (t) => { + const { db, peer } = await fixture(t) + const operations = new SharedVmOperations(db) + const operation = await operations.tryAcquire({ vmId: 'refresh-vm', accountId: 'refresh-account' }) + let gate, other + try { + gate = await new ExecutionAdmissionsRepo(peer).tryReserve(execution({ vmId: 'other-refresh-vm' })) + assert.equal(gate.ok, false) + assert.equal(gate.code, 'vm_operation_active') + other = await new SharedVmOperations(peer).tryAcquire({ vmId: 'other-refresh-vm', accountId: 'refresh-account' }) + assert.equal(other.ok, false) + assert.equal(other.code, 'vm_operation_active') + } finally { + await gate?.release?.() + await other?.release?.() + await operation.release() + } +}) + +test('an uncertain account refresh remains exclusive across VMs', async (t) => { + const { db, peer } = await fixture(t) + const operation = await new SharedVmOperations(db).tryAcquire({ vmId: 'refresh-vm', accountId: 'refresh-account' }) + await operation.markUncertain() + const gate = await new ExecutionAdmissionsRepo(peer).tryReserve(execution({ vmId: 'other-refresh-vm' })) + try { assert.equal(gate.ok, false) } + finally { await gate.release?.() } +}) + +test('a nested refresh binds its account to the manual VM operation until that operation ends', async (t) => { + const { db, peer } = await fixture(t) + const operations = new SharedVmOperations(db) + await operations.run('refresh-vm', async () => { + await operations.run('refresh-vm', async () => {}, { accountId: 'refresh-account' }) + const gate = await new ExecutionAdmissionsRepo(peer).tryReserve(execution({ vmId: 'other-refresh-vm' })) + try { assert.equal(gate.ok, false) } + finally { await gate.release?.() } + }) +}) + test('refresh never excludes a pending job or another active admission', async (t) => { const { db, peer } = await fixture(t) const gate = await new ExecutionAdmissionsRepo(db).tryReserve(execution()) From 54399fea1b082e181ad493aa2cb2e0bdde7fde67 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:28:34 +0800 Subject: [PATCH 306/473] fix: persist account-wide exclusion for active VM operations --- .../018_vm_operation_accounts.sql | 6 ++++ .../db/repos/execution-admissions-repo.mjs | 10 ++++--- src/lib/vm/shared-vm-operations.mjs | 30 +++++++++++++------ test/unit/shared-refresh-admission.test.mjs | 14 ++++++--- 4 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 src/lib/db/postgres-migrations/018_vm_operation_accounts.sql diff --git a/src/lib/db/postgres-migrations/018_vm_operation_accounts.sql b/src/lib/db/postgres-migrations/018_vm_operation_accounts.sql new file mode 100644 index 000000000..74cf7de16 --- /dev/null +++ b/src/lib/db/postgres-migrations/018_vm_operation_accounts.sql @@ -0,0 +1,6 @@ +-- Account-wide credential changes exclude executions on every VM for that account. +ALTER TABLE vm_operations ADD COLUMN account_id text; + +CREATE INDEX vm_operations_active_account + ON vm_operations (account_id) + WHERE released_at IS NULL AND account_id IS NOT NULL; diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 175076600..83f015185 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -111,8 +111,9 @@ export class ExecutionAdmissionsRepo { await this.assertRoutingRevision(input.routingRevision) await this.lockResources({ accountId, vmIds: [vmId] }) const operation = await this.db - .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') - .get(physicalVmAliases(vmId)) + .prepare(`SELECT id FROM vm_operations WHERE released_at IS NULL + AND (vm_id = ANY($1::text[]) OR account_id=$2) LIMIT 1`) + .get(physicalVmAliases(vmId), accountId) if (operation) return denied('vm_operation_active', 0, vmLimit) // A candidate may have been read before a completed management hold. // Check every physical alias under the same VM locks after the hold commits. @@ -217,8 +218,9 @@ export class ExecutionAdmissionsRepo { } if (nodeBinding && nodeBinding.vmId !== admission.vm_id) throw jobError('execution_node_binding_mismatch') const operation = await this.db - .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') - .get(physicalVmAliases(admission.vm_id)) + .prepare(`SELECT id FROM vm_operations WHERE released_at IS NULL + AND (vm_id = ANY($1::text[]) OR account_id=$2) LIMIT 1`) + .get(physicalVmAliases(admission.vm_id), admission.account_id) if (operation) throw jobError('vm_operation_active') const pending = await this.db .prepare('SELECT request_id FROM execution_jobs WHERE admission_id = $1 AND ended_at IS NULL LIMIT 1') diff --git a/src/lib/vm/shared-vm-operations.mjs b/src/lib/vm/shared-vm-operations.mjs index 1d2064021..0ba23e329 100644 --- a/src/lib/vm/shared-vm-operations.mjs +++ b/src/lib/vm/shared-vm-operations.mjs @@ -88,8 +88,9 @@ export class SharedVmOperations { // Identical namespace and VM resource to ExecutionAdmissionsRepo.tryReserve. await this.executions.lockResources({ accountId, vmIds: [vmId] }) const active = await this.db - .prepare('SELECT id FROM vm_operations WHERE vm_id = ANY($1::text[]) AND released_at IS NULL LIMIT 1') - .get(physicalVmAliases(vmId)) + .prepare(`SELECT id FROM vm_operations WHERE released_at IS NULL + AND (vm_id = ANY($1::text[]) OR account_id=$2) LIMIT 1`) + .get(physicalVmAliases(vmId), accountId) if (active) return { ok: false, code: 'vm_operation_active' } const exception = await this._executionException({ vmId, accountId, admission }) if (!exception.ok) return exception @@ -97,9 +98,9 @@ export class SharedVmOperations { if (execution) return { ok: false, code: 'vm_execution_active' } const operationId = crypto.randomUUID() await this.db - .prepare('INSERT INTO vm_operations (id, vm_id, owner_id, kind) VALUES ($1, $2, $3, $4)') - .run(operationId, vmId, this.ownerId, kind) - return this._handle({ operationId, vmId, ownerId: this.ownerId, kind }) + .prepare('INSERT INTO vm_operations (id, vm_id, owner_id, kind, account_id) VALUES ($1, $2, $3, $4, $5)') + .run(operationId, vmId, this.ownerId, kind, accountId) + return this._handle({ operationId, vmId, ownerId: this.ownerId, kind, accountId }) }) } @@ -184,15 +185,26 @@ export class SharedVmOperations { const valid = await withTransaction(this.db, async () => { await this.executions.lockResources({ accountId, vmIds: [vmId] }) const current = await this.db - .prepare(`SELECT id FROM vm_operations + .prepare(`SELECT id,account_id FROM vm_operations WHERE id=$1 AND owner_id=$2 AND vm_id=$3 AND released_at IS NULL AND uncertain_at IS NULL`) .get(nested.operation.operationId, nested.operation.ownerId, vmId) if (!current) return { ok: false, code: 'vm_operation_inactive' } + if (accountId && current.account_id && current.account_id !== accountId) + return { ok: false, code: 'vm_operation_account_mismatch' } + if (accountId) { + const other = await this.db + .prepare(`SELECT id FROM vm_operations + WHERE account_id=$1 AND id<>$2 AND released_at IS NULL LIMIT 1`) + .get(accountId, current.id) + if (other) return { ok: false, code: 'vm_operation_active' } + } const exception = await this._executionException({ vmId, accountId, admission }) if (!exception.ok) return exception - return (await this._otherExecution(vmId, accountId, exception.admissionId)) - ? { ok: false, code: 'vm_execution_active' } - : { ok: true } + if (await this._otherExecution(vmId, accountId, exception.admissionId)) + return { ok: false, code: 'vm_execution_active' } + if (accountId && !current.account_id) + await this.db.prepare('UPDATE vm_operations SET account_id=$1 WHERE id=$2').run(accountId, current.id) + return { ok: true } }) if (!valid.ok) throw operationError(valid.code) return callback(nested.operation) diff --git a/test/unit/shared-refresh-admission.test.mjs b/test/unit/shared-refresh-admission.test.mjs index fd4012bfc..da1e89e93 100644 --- a/test/unit/shared-refresh-admission.test.mjs +++ b/test/unit/shared-refresh-admission.test.mjs @@ -102,8 +102,11 @@ test('an uncertain account refresh remains exclusive across VMs', async (t) => { const operation = await new SharedVmOperations(db).tryAcquire({ vmId: 'refresh-vm', accountId: 'refresh-account' }) await operation.markUncertain() const gate = await new ExecutionAdmissionsRepo(peer).tryReserve(execution({ vmId: 'other-refresh-vm' })) - try { assert.equal(gate.ok, false) } - finally { await gate.release?.() } + try { + assert.equal(gate.ok, false) + } finally { + await gate.release?.() + } }) test('a nested refresh binds its account to the manual VM operation until that operation ends', async (t) => { @@ -112,8 +115,11 @@ test('a nested refresh binds its account to the manual VM operation until that o await operations.run('refresh-vm', async () => { await operations.run('refresh-vm', async () => {}, { accountId: 'refresh-account' }) const gate = await new ExecutionAdmissionsRepo(peer).tryReserve(execution({ vmId: 'other-refresh-vm' })) - try { assert.equal(gate.ok, false) } - finally { await gate.release?.() } + try { + assert.equal(gate.ok, false) + } finally { + await gate.release?.() + } }) }) From b231fd550bf2f3aded7502e4a3decc90c57976af Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:28:53 +0800 Subject: [PATCH 307/473] test: reproduce bootstrap auxiliary results releasing VM claims --- .../official-cc-shared-lifecycle.test.mjs | 35 ++++++++++++++++++ test/unit/panel-proxy-vm-mutations.test.mjs | 36 +++++++++++++++++++ test/unit/vm-operation-producers.test.mjs | 23 ++++++++++++ 3 files changed, 94 insertions(+) diff --git a/test/unit/official-cc-shared-lifecycle.test.mjs b/test/unit/official-cc-shared-lifecycle.test.mjs index 036d743f9..2dc0b98ea 100644 --- a/test/unit/official-cc-shared-lifecycle.test.mjs +++ b/test/unit/official-cc-shared-lifecycle.test.mjs @@ -13,6 +13,7 @@ import { officialCcHome, readOfficialCcStatus, scheduleOfficialCcBootstrap, + runOfficialCcBootstrap, } from '../../src/lib/oauth/official-cc-bootstrap.mjs' import { writeWorkerCredentialFile } from '../../src/lib/oauth/oauth-credentials.mjs' @@ -132,6 +133,40 @@ test('a raw bootstrap usage timeout keeps task and VM ownership and prevents a l assert.equal(calls, 1) }) +for (const auxiliary of ['profile', 'models', 'identity']) { + test(`raw bootstrap reports an unknown ${auxiliary} even when hello and usage succeed`, async (t) => { + const f = await fixture(t) + prepareRawBootstrap(f, t) + f.options.runTurn = async ({ prompt, outFile }) => { + fs.writeFileSync( + outFile, + prompt === '/usage' + ? JSON.stringify({ five_hour: { utilization: 0.1 }, seven_day: { utilization: 0.2 } }) + : '{}', + ) + return { ok: true, code: 0, timed_out: false } + } + const call = f.options.slotOauth + f.options.slotOauth = (exec, op) => + op === auxiliary + ? { ok: false, status: 0, attemptFinished: false, body: { error: { code: 'timeout' } } } + : call(exec, op) + if (auxiliary === 'identity') f.options.collectIdentity = async () => ({ ok: false, error: 'synthetic timeout' }) + let raw + f.options.runBootstrap = async (options) => { + raw = await runOfficialCcBootstrap(options) + return raw + } + await scheduleOfficialCcBootstrap(f.options) + await f.tasks.flush() + assert.equal(raw.ok, true) + assert.equal(raw.attemptFinished, false) + assert.equal(raw.vmOperationUncertain, true) + assert.equal((await f.other.tryAcquire({ vmId: f.vm.id })).ok, false) + assert.equal((await f.peer.prepare('SELECT finished_at FROM background_task_runs').get()).finished_at, null) + }) +} + test('manual bootstrap rejects a busy VM before queued status or background effects are written', async (t) => { const f = await fixture(t) await new ExecutionAdmissionsRepo(f.peer).tryReserve({ diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 18d74bf11..1140fb803 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -313,3 +313,39 @@ test('Kubernetes package creation refuses a committed alias missing from this Po assert.equal(h.response.status, 409) assert.equal(fs.existsSync(path.join(f.project, 'vms', 'new.json')), false) }) + +for (const auxiliary of ['profile', 'models', 'identity']) { + test(`package bootstrap retains its VM and proxy claims after an unknown ${auxiliary}`, async (t) => { + const f = await fixture(t) + const body = buildVmPackage({ + vm: { id: 'vm-new', timezone: 'UTC', claude: { mode: 'oauth' } }, + proxy: f.pool.getProxyByIdWithAuth(f.proxyId), + claudeCred: { mode: 'oauth', access_token: 'synthetic-access', refresh_token: 'synthetic-refresh' }, + }) + let calls = 0 + const h = f.handler(body, { + officialCcStatsHandler: () => async () => {}, + accountQuota: { + snapshot: () => ({ accounts: [] }), + ensure() {}, + repo: { get: () => ({ unified: {} }), observe: async () => {} }, + beginObservation: async () => 'synthetic', + ingestOAuthUsage() {}, + setAccountTier() {}, + flush: async () => {}, + }, + vmMutationOps: { + startSlotReady: async () => ({ ok: true }), + runOfficialCcBootstrap: async () => { + calls++ + return { ok: true, status: { [`${auxiliary}_error`]: 'synthetic timeout' } } + }, + }, + }) + await h.request('POST', '/api/panel/vms/package') + assert.equal(h.response.status, 201, JSON.stringify(h.response.body)) + assert.equal(calls, 1) + assert.equal((await f.otherOperations.tryAcquire({ vmId: 'vm-new' })).ok, false) + assert.equal((await new SharedProxyOperations(f.peer).tryAcquire()).ok, false) + }) +} diff --git a/test/unit/vm-operation-producers.test.mjs b/test/unit/vm-operation-producers.test.mjs index 08e4311c4..c23d3db60 100644 --- a/test/unit/vm-operation-producers.test.mjs +++ b/test/unit/vm-operation-producers.test.mjs @@ -303,6 +303,29 @@ test('Kubernetes credential import awaits official initialization while holding assert.equal(await f.active(), 0) }) +for (const auxiliary of ['profile', 'models', 'identity']) { + test(`credential import retains the VM operation after an unknown bootstrap ${auxiliary}`, async (t) => { + const f = await fixture(t) + process.env.KIN_CRS_MOCK = '0' + let calls = 0 + const commit = createImportCommit({ + ...f.ctx, + routingConfig: { official_cc: { enabled: true } }, + reloadSlotReady: async () => ({ ok: true, runtime: {} }), + importWorkerCredential: async () => ({ ok: true }), + runOfficialCcBootstrap: async () => { + calls++ + return { ok: true, status: { [`${auxiliary}_error`]: 'synthetic timeout' } } + }, + }).commitImportedOauth + const result = await commit({ ...f.input, skipOfficialCc: false }) + assert.equal(result.official_cc_bootstrap.completed, true) + assert.equal(calls, 1) + assert.equal(await f.active(), 1) + assert.equal((await admission(f.other)).code, 'vm_operation_active') + }) +} + test('Kubernetes remote unsupported or disabled official initialization never falls back to local execution', async (t) => { const f = await fixture(t) process.env.KIN_CRS_MOCK = '0' From d299a7664e098f04fc83b7bfa72ef5b32317555a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:29:23 +0800 Subject: [PATCH 308/473] test: preserve refresh uncertainty after a completed model execution --- test/unit/failover-runner.test.mjs | 33 ++++++++++++++++++++++++++++++ test/unit/kernel-router.test.mjs | 31 ++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/test/unit/failover-runner.test.mjs b/test/unit/failover-runner.test.mjs index e465acc5b..1149efe9d 100644 --- a/test/unit/failover-runner.test.mjs +++ b/test/unit/failover-runner.test.mjs @@ -106,6 +106,39 @@ test('unsupported classifier runtime stops without counting execution or cooling assert.equal(scheduler.selectCalls, 1) }) +test('a finished model job with uncertain credential refresh retains its admission and cannot fail over', async () => { + let marked = 0 + const first = { + ...candidate(1), + bindExecution: async () => {}, + markUncertain: async () => { + marked++ + }, + } + const scheduler = new Scheduler([first, candidate(2)]) + const runner = new FailoverRunner({ scheduler }) + let calls = 0 + const result = await runner.run({ + requestId: 'uncertain-refresh', + model: 'claude-test', + canonicalBody: { model: 'claude-test' }, + callAttempt: async () => { + calls++ + if (calls > 1) return success() + return { + ok: false, + status: 503, + execution: { ended: true }, + executionUncertain: true, + body: { error: { code: 'credential_refresh_unknown' } }, + } + }, + }) + assert.equal(calls, 1) + assert.equal(marked, 1) + assert.equal(result.executionUncertain, true) +}) + test('invalid panel cache layout stops before execution without rotating or cooling accounts', async () => { const scheduler = new Scheduler([candidate(1), candidate(2)]) const attempts = new Attempts() diff --git a/test/unit/kernel-router.test.mjs b/test/unit/kernel-router.test.mjs index a397d4281..0777e7556 100644 --- a/test/unit/kernel-router.test.mjs +++ b/test/unit/kernel-router.test.mjs @@ -491,6 +491,37 @@ unixTest('needs_refresh retries once via Go ensure', async () => { } }) +unixTest('an uncertain credential refresh stays distinct from the first completed kernel job', async () => { + const previous = process.env.KIN_KERNEL_BIN + process.env.KIN_KERNEL_BIN = '/bin/true' + let hits = 0 + const fx = await kernelFixture((_req, res) => { + hits++ + res.writeHead(401, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ type: 'error', error: { code: 'needs_refresh' } })) + }) + try { + const result = await dispatchCallInference({ + exec: fx.exec, + body: { model: 'claude-test', messages: [{ role: 'user', content: 'synthetic' }] }, + ensureRust: async () => ({ ok: true }), + ensureCredential: async () => ({ + ok: false, + attemptFinished: false, + credentialRefreshUnknown: true, + error: { code: 'worker_timeout' }, + }), + }) + assert.equal(hits, 1) + assert.equal(result.execution.ended, true) + assert.equal(result.executionUncertain, true) + } finally { + await fx.close() + if (previous == null) delete process.env.KIN_KERNEL_BIN + else process.env.KIN_KERNEL_BIN = previous + } +}) + unixTest('failed Go credential ensure is not followed by a blind Rust retry', async () => { const previous = process.env.KIN_KERNEL_BIN process.env.KIN_KERNEL_BIN = '/bin/true' From bb39ce027743325433a1cb2ffd7622dff161540a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:30:11 +0800 Subject: [PATCH 309/473] test: finish worker refresh only after its VM projection is persisted --- test/unit/slot-oauth-shared.test.mjs | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/unit/slot-oauth-shared.test.mjs b/test/unit/slot-oauth-shared.test.mjs index 7c31607f6..fd81cc5c0 100644 --- a/test/unit/slot-oauth-shared.test.mjs +++ b/test/unit/slot-oauth-shared.test.mjs @@ -9,6 +9,7 @@ import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenanc import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { readWorkerCredentialFile, writeWorkerCredentialFile } from '../../src/lib/oauth/oauth-credentials.mjs' import { runSlotOauth } from '../../src/lib/transport/slot-oauth.mjs' +import { ensureWorkerCredential } from '../../src/lib/transport/go-worker-client.mjs' async function fixture(t) { const db = getDb() @@ -28,10 +29,14 @@ async function fixture(t) { account_uuid: 'slot-refresh-account', }) const previous = process.env.VM2API_CONTROL_PLANE_MODE + const dockerBin = process.env.KIN_DOCKER_BIN + process.env.KIN_DOCKER_BIN = path.join(root, 'missing-synthetic-docker') process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' t.after(async () => { if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = previous + if (dockerBin == null) delete process.env.KIN_DOCKER_BIN + else process.env.KIN_DOCKER_BIN = dockerBin await coordinator.stop() const client = new pg.Client({ connectionString: db.connectionString }) await client.connect() @@ -82,6 +87,37 @@ test('a slot refresh timeout retains its shared operation and task', async (t) = assert.ok(rows.rows[0].uncertain_at) }) +test('ensure publishes its VM projection before the shared refresh and VM operation finish', async (t) => { + const f = await fixture(t) + let publications = 0 + const result = await ensureWorkerCredential(f.exec, { + force: true, + runDocker: async () => { + writeWorkerCredentialFile(f.homeDir, { + ...readWorkerCredentialFile(f.homeDir), + access_token: 'SYNTHETIC_NEW_AT', + refresh_token: 'SYNTHETIC_NEW_RT', + }) + return { code: 0, stdout: '{"ok":true,"status":200,"body":{"refreshed":true}}' } + }, + publishCredential: async () => { + publications++ + assert.equal((await f.db.prepare('SELECT count(*) AS n FROM vm_operations WHERE released_at IS NULL').get()).n, 1) + assert.equal( + (await f.db.prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get()).n, + 1, + ) + }, + }) + assert.equal(result.ok, true) + assert.equal(result.attemptFinished, true) + assert.equal(publications, 1) + assert.equal( + (await f.run(() => f.db.prepare('SELECT count(*) AS n FROM vm_operations WHERE released_at IS NULL').get())).n, + 0, + ) +}) + test('implicit worker refresh commands cannot execute while another inference is active', async (t) => { const f = await fixture(t) const gate = await f.run(() => From b24ee7eb2bfa3027c3924509e936fabeec55b4c2 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:30:24 +0800 Subject: [PATCH 310/473] fix: retain bridge repair ownership across proxy and VM effects --- src/lib/transport/kernel-watchdog.mjs | 8 +-- src/lib/vm/slot-recovery.mjs | 91 ++++++++++++++++++++++++++- test/unit/shared-bridge-heal.test.mjs | 72 +++++++++++++++++++-- 3 files changed, 160 insertions(+), 11 deletions(-) diff --git a/src/lib/transport/kernel-watchdog.mjs b/src/lib/transport/kernel-watchdog.mjs index b86c8470b..e07353bd0 100644 --- a/src/lib/transport/kernel-watchdog.mjs +++ b/src/lib/transport/kernel-watchdog.mjs @@ -222,10 +222,10 @@ export function createKernelWatchdog({ healingBridges = true const work = () => healBridges(vms) const pending = tasks - ? tasks.run({ taskKey: 'kernel_bridge_heal', intervalMs: config.interval_sec * 1000 }, async () => ({ - terminal: true, - value: await work(), - })) + ? tasks.run({ taskKey: 'kernel_bridge_heal', intervalMs: config.interval_sec * 1000 }, async () => { + const value = await work() + return { terminal: value?.ok === true && value.vmOperationUncertain !== true, value } + }) : Promise.resolve().then(work) trackSideEffect(pending) .catch((error) => console.warn(`[kernel-watchdog] bridge heal failed: ${error?.message || error}`)) diff --git a/src/lib/vm/slot-recovery.mjs b/src/lib/vm/slot-recovery.mjs index 5fa0832cb..41cee9964 100644 --- a/src/lib/vm/slot-recovery.mjs +++ b/src/lib/vm/slot-recovery.mjs @@ -34,6 +34,13 @@ import { isKvmRuntime } from './runtime-kind.mjs' import { slotDockerCpus } from './machine-spec.mjs' import { bindVmProxy, getVm, patchVmRecord, vmHasSchedulableCredential } from './vm-registry.mjs' import { isCredentialRefreshFailed } from '../pool/availability.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { getDb } from '../db/database.mjs' +import { SharedProxyOperations } from './shared-proxy-operations.mjs' +import { SharedVmOperations } from './shared-vm-operations.mjs' +import { withVmMutations } from './vm-mutations.mjs' +import { parseBoundVmIds } from './proxy-pool.mjs' +import { flushVmDbSync } from './vm-db-sync.mjs' const HEALTH_TIMEOUT_MS = 1500 const BRIDGE_HEAL_MIN_MS = 60_000 @@ -611,7 +618,13 @@ export function createSlotBridgeHealer({ now = () => Date.now(), ops = {}, logger = console, + operations = null, + proxyOperations = null, }) { + const shared = !!operations || !!proxyOperations || isKubernetesMode() + const db = shared ? operations?.db || proxyOperations?.db || getDb() : null + const mutations = shared ? operations || new SharedVmOperations(db) : null + const proxies = shared ? proxyOperations || new SharedProxyOperations(db) : null const egressAlive = ops.egressAlive || ((proxyId) => inspectEgressProcess(projectRoot, proxyId).ok) const startEgress = ops.ensureProxyEgress || ((proxy) => ensureProxyEgress(projectRoot, proxy)) const getCluster = ops.cluster || cluster @@ -622,7 +635,7 @@ export function createSlotBridgeHealer({ lastAttempt.set(key, at) return true } - return async function healBridges(vms) { + async function perform(vms) { const seenProxies = new Set() for (const vm of vms) { if (vm?.status !== 'running') continue @@ -638,6 +651,7 @@ export function createSlotBridgeHealer({ if (healed.length) logger.warn(`[bridge-heal] ${vm.id} rebuilt ${healed.join(', ')}`) } catch (error) { logger.warn(`[bridge-heal] ${vm.id} relay rebuild failed: ${error?.message || error}`) + if (shared) return { ok: false, vmOperationUncertain: true } } continue } @@ -650,6 +664,81 @@ export function createSlotBridgeHealer({ const result = await startEgress(proxy) if (result?.ok) logger.warn(`[bridge-heal] kin-egress ${proxy.id} was down; restarted`) else logger.warn(`[bridge-heal] kin-egress ${proxy.id} restart failed: ${result?.error || 'unknown'}`) + if (shared && (result?.ok !== true || result?.vmOperationUncertain === true)) + return { ok: false, vmOperationUncertain: true } + } + return { ok: true, vmOperationUncertain: false } + } + + return async function healBridges(vms) { + if (!shared) return perform(vms) + // Read-only planning avoids taking every VM out of scheduling on healthy ticks. + const plans = [] + for (const vm of vms) { + if (vm?.status !== 'running') continue + const nodeId = vmNodeId(vm) + if (nodeId) { + const status = getCluster()?.slotBridgeStatus(nodeId, vm.id) + if ( + status?.link?.state === 'ready' && + !(status.bridge?.alive && status.relays?.kernel?.alive && status.relays?.worker?.alive) + ) + plans.push({ vm, nodeId }) + } else { + const proxy = (ops.getVm || getVm)(projectRoot, vm.id)?.proxy + if ( + !isCodexVm(vm) && + hasBoundExit(proxy) && + !isLocalEgressProxy(proxy) && + !proxyBlockedReason(proxy) && + !egressAlive(proxy.id) + ) + plans.push({ vm, proxyId: proxy.id }) + } + } + if (!plans.length) return { ok: true, vmOperationUncertain: false } + let entered = false + try { + return await proxies.run(async () => { + const ids = new Set(plans.map((plan) => plan.vm.id)) + const proxyIds = [...new Set(plans.map((plan) => plan.proxyId).filter(Boolean))] + if (proxyIds.length) { + const rows = await db.prepare('SELECT bound_vm_id FROM proxies WHERE id = ANY($1::text[])').all(proxyIds) + for (const row of rows) for (const id of parseBoundVmIds(row.bound_vm_id)) ids.add(id) + } + // A node Docker bridge is shared by its slots. Include stored VMs even + // when the watchdog target snapshot omitted them. + if (plans.some((plan) => plan.nodeId)) { + for (const row of await db.prepare('SELECT id FROM vms').all()) ids.add(row.id) + } + let result + try { + result = await withVmMutations( + projectRoot, + [...ids], + async () => { + entered = true + const targets = plans.flatMap((plan) => { + const current = (ops.getVm || getVm)(projectRoot, plan.vm.id) || plan.vm + const sameTarget = plan.nodeId ? vmNodeId(current) === plan.nodeId : current.proxy?.id === plan.proxyId + return sameTarget ? [current] : [] + }) + const outcome = await perform(targets) + await flushVmDbSync() + return outcome + }, + { operations: mutations, kind: 'kernel_bridge_heal' }, + ) + } catch (error) { + if (!entered) error.proxyOperationNotStarted = true + throw error + } + return { terminal: result?.ok === true && result.vmOperationUncertain !== true, value: result } + }) + } catch (error) { + if (!entered && ['vm_execution_active', 'vm_operation_active', 'proxy_operation_active'].includes(error.code)) + error.taskNotStarted = true + throw error } } } diff --git a/test/unit/shared-bridge-heal.test.mjs b/test/unit/shared-bridge-heal.test.mjs index 680062248..6c51d9004 100644 --- a/test/unit/shared-bridge-heal.test.mjs +++ b/test/unit/shared-bridge-heal.test.mjs @@ -1,6 +1,8 @@ import test from 'node:test' import assert from 'node:assert/strict' -import { getDb } from '../../src/lib/db/database.mjs' +import pg from 'pg' +import { getDb, registerActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { createSlotBridgeHealer } from '../../src/lib/vm/slot-recovery.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { SharedProxyOperations } from '../../src/lib/vm/shared-proxy-operations.mjs' @@ -23,7 +25,10 @@ function healer(overrides = {}) { ops: { getVm: (_root, id) => ({ ...vm, id }), egressAlive: () => false, - ensureProxyEgress: async () => { starts++; return { ok: true } }, + ensureProxyEgress: async () => { + starts++ + return { ok: true } + }, cluster: () => null, ...overrides, }, @@ -41,18 +46,24 @@ test('bridge healing cannot bypass an existing VM mutation', async () => { const pool = await new SharedProxyOperations(db).tryAcquire() assert.equal(pool.ok, true, 'a refusal before any helper work returns the pool claim') await pool.release() - } finally { await held.release() } + } finally { + await held.release() + } }) test('bridge healing takes claims for other VMs sharing the same helper', async () => { const db = getDb() - await db.prepare('INSERT INTO proxies (id, bound_vm_id) VALUES ($1,$2)').run(proxy.id, JSON.stringify([vm.id, 'other-bridge-vm'])) + await db + .prepare('INSERT INTO proxies (id, bound_vm_id) VALUES ($1,$2)') + .run(proxy.id, JSON.stringify([vm.id, 'other-bridge-vm'])) const held = await new SharedVmOperations(db).tryAcquire({ vmId: 'other-bridge-vm' }) const fx = healer() try { await assert.rejects(fx.heal([vm]), { code: 'vm_operation_active' }) assert.equal(fx.starts(), 0) - } finally { await held.release() } + } finally { + await held.release() + } }) test('an unknown helper result is returned and preserves VM and pool claims', async () => { @@ -74,7 +85,56 @@ test('watchdog bridge activity remains unknown when the healer cannot confirm co }) await watchdog.tick() await watchdog.flush() - const row = await db.prepare("SELECT unknown_at, finished_at FROM background_task_runs WHERE task_key='kernel_bridge_heal'").get() + const row = await db + .prepare("SELECT unknown_at, finished_at FROM background_task_runs WHERE task_key='kernel_bridge_heal'") + .get() assert.ok(row.unknown_at) assert.equal(row.finished_at, null) }) + +test('bridge work obtains its own activity after bootstrap ends and waits for all claims', async (t) => { + const db = getDb() + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + registerActivityCoordinator(db, coordinator) + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(async () => { + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + await coordinator.stop() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + const bootstrap = await coordinator.admitBootstrap() + await coordinator.acknowledgeEpoch(bootstrap.epoch) + await bootstrap.complete({ ended: true }) + await assert.rejects(db.query('SELECT 1'), { code: 'maintenance_activity_closed' }) + const fx = healer() + const watchdog = createKernelWatchdog({ + listTargets: () => [], + sharedTasks: new SharedBackgroundTasks(db), + operations: new SharedVmOperations(db), + healBridges: () => fx.heal([vm]), + }) + await watchdog.tick() + await watchdog.flush() + assert.equal(fx.starts(), 1) + const check = await coordinator.admit({ kind: 'test_observer' }) + try { + await check.run(async () => { + assert.equal((await db.prepare('SELECT count(*) AS n FROM vm_operations WHERE released_at IS NULL').get()).n, 0) + assert.equal( + (await db.prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get()).n, + 0, + ) + }) + } finally { + await check.complete({ ended: true }) + } +}) From befd24d1d6a3327cfbcde4c78cc4a09cf9333ed8 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:30:24 +0800 Subject: [PATCH 311/473] fix: pass credential refresh options through inference callbacks --- src/lib/protocol/handle-protocol.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 175a0eb34..71556713c 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -468,7 +468,7 @@ export function createHandleProtocol(deps) { requestContext, slotWaitMs: candidate.slotWaitMs, noGoFallback, - ensureCredential: async (exec) => await ensureWorkerCredential(exec), + ensureCredential: async (exec, opts) => await ensureWorkerCredential(exec, opts), onEvent: async (line) => { if (/response_headers/.test(String(line))) return applyClaudeSSELineToMessage(restoreToolNamesInSSELine(line, toolNames), assembler) @@ -1621,7 +1621,7 @@ export function createHandleProtocol(deps) { routing: getRouting(), slotWaitMs: candidate.slotWaitMs, noGoFallback: !!pinVmId, - ensureCredential: async (exec) => await ensureWorkerCredential(exec), + ensureCredential: async (exec, opts) => await ensureWorkerCredential(exec, opts), onCommit: () => { if (protocol === 'anthropic.messages' && !res.headersSent) writeSSEHeaders(res) onCommit() From 0b1e7457210879b10915cab88b5c3048f2f9f09a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:30:48 +0800 Subject: [PATCH 312/473] test(slotgate): require durable allocation receipts and private live grants --- test/unit/node-management-grants.test.mjs | 83 +++++++++++++++++++++++ worker/internal/slotgate/allocate_test.go | 49 +++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 test/unit/node-management-grants.test.mjs create mode 100644 worker/internal/slotgate/allocate_test.go diff --git a/test/unit/node-management-grants.test.mjs b/test/unit/node-management-grants.test.mjs new file mode 100644 index 000000000..bbb25d090 --- /dev/null +++ b/test/unit/node-management-grants.test.mjs @@ -0,0 +1,83 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import pg from 'pg' +import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { NodeManagementGrants, nodeAuthoritySchemaFor } from '../../src/lib/cluster/node-management-grants.mjs' + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const maintenance = new GlobalMaintenanceCoordinator(db) + await maintenance.start() + await maintenance.acknowledgeEpoch(1) + const service = new NodeManagementGrants(db) + await service.initialize() + const held = [] + t.after(async () => { + for (const operation of held) { operation.resume(); await operation.pending } + await maintenance.stop() + await peer.close() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE`) + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(maintenance.schema)} CASCADE`) + } finally { await cleanup.end() } + }) + async function issue(allocationId = 'allocation-one', patch = {}) { + let resume + let ready + const waiting = new Promise((resolve) => { resume = resolve }) + const received = new Promise((resolve) => { ready = resolve }) + const pending = new SharedVmOperations(db, { ownerId: 'synthetic-controller' }).run('synthetic-vm', async (operation) => { + const grant = await service.issueAllocation({ operation, vmId: 'synthetic-vm', nodeId: 'synthetic-node', allocationId, ...patch }) + ready({ grant, operation }) + await waiting + }, { kind: 'node_allocate' }) + pending.catch((error) => ready({ error })) + held.push({ pending, resume }) + const value = await received + if (value.error) throw value.error + return { ...value, resume, pending } + } + return { db, peer, service, maintenance, issue } +} + +test('allocation grants require the exact current operation scope and persist outside the application schema', async (t) => { + const fx = await fixture(t) + await assert.rejects(fx.service.issueAllocation({ operation: { operationId: 'forged', ownerId: 'synthetic-controller' }, + vmId: 'synthetic-vm', nodeId: 'synthetic-node', allocationId: 'forged' }), { code: 'node_operation_scope_required' }) + const { grant } = await fx.issue() + assert.equal(grant.action, 'allocate') + assert.equal(grant.nodeId, 'synthetic-node') + assert.notEqual(fx.service.schema, fx.db.schema) + assert.equal(fx.service.schema, nodeAuthoritySchemaFor(fx.db.schema)) + const row = (await fx.peer.pool.query(`SELECT * FROM ${quoteIdentifier(fx.service.schema)}.allocations`)).rows[0] + assert.equal(row.operation_id, grant.operationId) + assert.equal(row.revoked_at, null) +}) + +test('operation release or uncertainty revokes its private allocation grant', async (t) => { + const fx = await fixture(t) + const held = await fx.issue() + await held.operation.markUncertain() + const row = (await fx.peer.pool.query(`SELECT revoked_at FROM ${quoteIdentifier(fx.service.schema)}.allocations`)).rows[0] + assert.ok(row.revoked_at) +}) + +test('once enabled, missing private grant state prevents release rather than becoming legacy mode', async (t) => { + const fx = await fixture(t) + const held = await fx.issue() + await fx.peer.pool.query(`DROP SCHEMA ${quoteIdentifier(fx.service.schema)} CASCADE`) + await assert.rejects(held.operation.release(), { code: 'node_authority_unavailable' }) + const row = await fx.peer.prepare('SELECT released_at FROM vm_operations WHERE id=$1').get(held.operation.operationId) + assert.equal(row.released_at, null) + // The callback deliberately leaves an unknown operation; cleanup must not retry a normal release. + await fx.peer.prepare('UPDATE vm_operations SET uncertain_at=clock_timestamp() WHERE id=$1').run(held.operation.operationId) + held.resume() + await assert.rejects(held.pending) + // Prevent the shared fixture cleanup from observing the already asserted rejection twice. + held.pending.catch(() => {}) +}) diff --git a/worker/internal/slotgate/allocate_test.go b/worker/internal/slotgate/allocate_test.go new file mode 100644 index 000000000..99a98e6aa --- /dev/null +++ b/worker/internal/slotgate/allocate_test.go @@ -0,0 +1,49 @@ +package slotgate + +import ( + "context" + "errors" + "path/filepath" + "sync" + "testing" +) + +func allocationIntent(id string) AllocationIntent { + return AllocationIntent{NodeID: "synthetic-node", VMID: "v", AllocationID: id, OperationID: "synthetic-operation", OwnerID: "synthetic-controller"} +} + +func TestAllocationReceiptCreatesOnlyOneInitialGenerationAndSurvivesReopen(t *testing.T) { + root := filepath.Join(shortRoot(t), "authority") + a, err := OpenAuthority(root, nil) + if err != nil { t.Fatal(err) } + intent := allocationIntent("allocation-one") + var wg sync.WaitGroup + receipts := make(chan AllocationReceipt, 2) + for range 2 { + wg.Add(1) + go func() { defer wg.Done(); receipt, err := a.Allocate(intent, "1"); if err != nil { t.Error(err); return }; receipts <- receipt }() + } + wg.Wait() + close(receipts) + var first AllocationReceipt + for receipt := range receipts { + if receipt.Fence.Generation != 1 || receipt.Fence.Executor != (ExecutorIdentity{}) { t.Fatalf("not a vacant initial generation: %+v", receipt) } + if first.AllocationID != "" && first != receipt { t.Fatalf("duplicate allocation changed receipt") } + first = receipt + } + if first.AllocationID == "" { t.Fatal("no receipt") } + if _, err = a.Allocate(allocationIntent("other-allocation"), "1"); !errors.Is(err, ErrAllocationConflict) { t.Fatalf("replace intent: %v", err) } + if err = a.Close(); err != nil { t.Fatal(err) } + a, err = OpenAuthority(root, nil) + if err != nil { t.Fatal(err) } + t.Cleanup(func() { a.Close() }) + replayed, err := a.Allocate(intent, "1") + if err != nil || replayed != first { t.Fatalf("lost-response replay: %+v %v", replayed, err) } +} + +func TestAllocationDeniedByAuthorizerMakesNoLocalSlot(t *testing.T) { + a := authorityForTest(t, filepath.Join(shortRoot(t), "authority"), nil) + manager := allocationManager{authority: a, nodeID: "synthetic-node", authorize: func(context.Context, AllocationIntent, func(string) error) error { return ErrManagementDenied }} + if _, err := manager.Allocate(context.Background(), allocationIntent("denied")); !errors.Is(err, ErrManagementDenied) { t.Fatal(err) } + if _, err := a.State("v"); !errors.Is(err, ErrInvalidAuthority) { t.Fatalf("denied allocation left state: %v", err) } +} From 98c6c464bd04687fdc1c0215738f1d0ea9e86303 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:31:12 +0800 Subject: [PATCH 313/473] test: verify eighteen migrations and older snapshot restore --- test/unit/backup-service.test.mjs | 2 +- test/unit/database.test.mjs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index b7fb9c719..381d49eb8 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -113,7 +113,7 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 15) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 18) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index 5892f9be2..00fa9ff34 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -117,7 +117,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 15) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 18) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -133,7 +133,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 15) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 18) fs.rmSync(dir, { recursive: true, force: true }) }) From 25df5ba8c7b6d9bf6e3896e64f6352dc2a622419 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:31:13 +0800 Subject: [PATCH 314/473] docs: record independent reviews and fixed-source verification progress --- .../2026-10-21-kubernetes-control-plane.md | 2 + .../2026-10-21-kubernetes-review-progress.md | 49 +++ .../independent-review-0ffae4b0.md | 59 ++++ .../independent-review-66a580de.md | 77 +++++ .../review-progress/verification.json | 297 ++++++++++++++++++ 5 files changed, 484 insertions(+) create mode 100644 docs/plans/2026-10-21-kubernetes-review-progress.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-0ffae4b0.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-66a580de.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/verification.json diff --git a/docs/plans/2026-10-21-kubernetes-control-plane.md b/docs/plans/2026-10-21-kubernetes-control-plane.md index 600ca138b..6bbd69707 100644 --- a/docs/plans/2026-10-21-kubernetes-control-plane.md +++ b/docs/plans/2026-10-21-kubernetes-control-plane.md @@ -3,6 +3,8 @@ 日期：2026-10-21。分支：`codex/kubernetes-control-plane`。 依赖：`codex/postgresql-migration@cf9462d5`（#647）。 +后续复核已恢复依赖分支 `fcd568c9` 的 NUL 日志修复。最新全量检查、两轮独立会话审查及后续定向验证见 [审查与验证进度](2026-10-21-kubernetes-review-progress.md)。当前仍保留单活动限制，多节点验收未完成。 + ## 目标和验收条件 目标为 #648 的 Kubernetes 部署与多活动控制面水平扩容。至少两个副本须部署在不同节点，同时承载重叠请求；认证、配置、业务数据和全局限额保持一致；后台任务不重复，异常退出后能够安全接管；HTTP、SSE、WebSocket、扩缩容和滚动更新均需运行验证。全部推理转交给单个 leader 不能代替此验收。 diff --git a/docs/plans/2026-10-21-kubernetes-review-progress.md b/docs/plans/2026-10-21-kubernetes-review-progress.md new file mode 100644 index 000000000..e06d27057 --- /dev/null +++ b/docs/plans/2026-10-21-kubernetes-review-progress.md @@ -0,0 +1,49 @@ +# Kubernetes 独立审查与定向验证（#648，进行中） + +本报告记录固定源码的检查结果，不作为多节点高可用验收。控制面仍保留 `ControlPlaneLock` 和 `replicas: 1`。已发布的 Draft PR #651 仍为早期提交；后续本地实现尚未推送。 + +## 全量检查 + +固定源码 `0ffae4b04c2429f472bef5a42add4a00a1ed7fcc` 在独立源码归档检查： + +| 检查 | 文件 | 测试数 | 通过 | 失败 | 跳过 | +| --- | ---: | ---: | ---: | ---: | ---: | +| unit | 342 | 3932 | 3915 | 7 | 10 | +| E2E | 30 | 127 | 109 | 17 | 1 | + +全部失败名称和规范化断言摘要与 `6732a864` 的记录一致。没有将这些失败改为通过或跳过。该全量结果不覆盖之后的源码修复。`6732a864` 的单节点部署镜像也不包含后续增量。 + +## 独立会话审查 + +两轮审查均使用单独 Codex CLI 会话和固定 detached worktree。审查会话只读源码、测试、文档与 Git 对象，没有执行测试或访问部署环境。主会话另行动态复现并验证修复。 + +首轮源码为 `0ffae4b0`，独立会话为 `01a15ee0-3324-7490-928c-8343beb13b5e`。发现 4 项 P1、1 项 P2。对应失败规格和定向修复如下： + +| 问题 | 失败规格 | 修复或验证提交 | 定向结果 | +| --- | --- | --- | --- | +| NUL 摘要使 PostgreSQL 日志写入失败 | `97e00cf3` | `1c034cc6` | 工作区日志 unit/E2E 56/56 | +| 后台监控启动后没有有效数据库活动 | `04a016b2` | `f666c923` | 固定提交 96/96 | +| 初始化后台原始 Promise 未单独跟踪 | `3e5b3b1a` | `3304c452` | 固定提交 85/85 | +| 包导入首次文件写入前未取得 VM 互斥 | `2a58a9e9` | `4fed2928` | 固定提交 83/83 | +| 代理面板入口未取得共享池和 VM 操作声明 | `2ebc3f42` | `b9fc7462` | 固定提交 107/107 | + +NUL 问题动态复现得到 PostgreSQL `22021`。修复恢复依赖分支已有的摘要转义，debug JSON 保留原始内容。其测试是工作区定向检查，不能称为整个 `1c034cc6` 的独立归档验证。 + +第二轮源码为 `66a580de17670a954638fbfb46fee25740adab78`，独立会话为 `01a15f03-648a-7d92-bf3d-f4d20d4cf03e`。复核了首轮修复，另发现 4 项 P1： + +- 初始化 `/usage` 重试覆盖前一次未知执行。 +- 包启动和凭据导入没有统一处理初始化的辅助操作未知结果。 +- 已有容器端点的代理网络清理可能长期保留整个代理池声明。 +- watchdog bridge 修复没有取得共享代理/VM 互斥，且丢弃未知结果。 + +这 4 项正在复现和修复。本报告不将第二轮审查写为通过。两轮原始发现及静态审查边界见 [首轮报告](evidence/kubernetes/review-progress/independent-review-0ffae4b0.md) 和 [第二轮报告](evidence/kubernetes/review-progress/independent-review-66a580de.md)。 + +## 后续固定源码检查 + +- `f4a685b98f31d467b6a2c8c5a8b7cf7749bf15c5`：watchdog 共享恢复预算、故障准入及相关回归 99/99。 +- `bf7615f8be9adf4f569c2a7d802be514a4c58e64`：共享熔断及原准入、面板、备份和重试 142/142。失败次数在 PostgreSQL 累计；半开状态的探测绑定持久 admission；未知结果不随时间释放；旧结果不能覆盖重置后的代次。 +- `e1e5d2a9e6cea391fb96aacc6b0757fa3ad0360f`：KSG2 客户端、PG 绑定及相关回归 156/156。实际 Go gate、合成 Unix kernel 与两条 PG 连接验证：正常持久完成后可释放；内部错误仍保留。发送前绑定原 Fence；响应后另取原 ticket 的节点持久完成状态。绑定新 job 与凭据刷新 VM 操作使用同一资源锁。 + +上述检查只使用专用测试 PostgreSQL、合成请求、模拟节点或测试自己的子进程。KSG2 集成使用合成 executor 身份，没有调用真实 Docker Engine；它不能证明可信启动或权限隔离。生产 capability 来源、节点文件发布和管理授权、凭据刷新接线、多节点运行及故障接管仍在实现。旧预编译 payload 继续复用。 + +结构化统计、日志摘要与边界见 [验证证据](evidence/kubernetes/review-progress/verification.json)。任务测试资源按精确所有权清单清理；专用 PostgreSQL 与共享 Go 缓存在后续验证结束前保留。没有修改生产环境。 diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-0ffae4b0.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-0ffae4b0.md new file mode 100644 index 000000000..3b60c0375 --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-0ffae4b0.md @@ -0,0 +1,59 @@ +发现 **4 项 P1、1 项 P2**。以下结论来自静态调用链；复现步骤交由主会话执行，本轮未运行。 + +**1. P1：NUL 字符可使请求日志失败，并持续阻断面板查询和备份** + +位置：[src/lib/admin/request-log.mjs:588](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/admin/request-log.mjs#L588) + +- **触发条件**：日志模式为 normal 或 debug，已认证请求的 `model` 包含 JSON 转义字符 `\u0000`。 +- **现象及影响**：PostgreSQL 拒绝写入包含 NUL 的文本。错误随后被进程级持久化状态保留，后续面板 GET 和备份持续失败。Kubernetes 模式还会留下不确定活动记录。 +- **依据**：`src/lib/protocol/handle-protocol.mjs:631`、`:632` 将原始模型名写入日志字段，`:583` 注册响应结束后的持久化。候选删除了 BASE 中的 NUL 转义，直接执行日志插入。`src/lib/db/pending-writes.mjs:10` 保留首次错误，`:19` 在每次 flush 时重抛。`src/lib/admin/panel-routes.mjs:1226` 的面板 GET 必经该 flush。`src/server.mjs:1243` 同样调用它，并在失败后将活动标记为不确定。 +- **建议最小修复**：恢复摘要字符串字段的 NUL 转义，保留调试正文的原始 JSON 内容。恢复本次删除的对应回归测试。 +- **最小复现**：使用隔离测试数据库和模拟推理发送 `model: "invalid\u0000model"`。随后查询请求日志、API key 列表并创建备份。检查日志插入错误及后续 flush 持续失败。BASE 中已删除的 `NUL-containing rejected requests preserve panel queries and backups` 测试可直接作为复现依据。 + +**2. P1：后台监控在取得活动准入前查询数据库，启动后无法正常执行** + +位置：[src/lib/oauth/credential-refresh-monitor.mjs:117](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/oauth/credential-refresh-monitor.mjs#L117)、[src/lib/admin/vm-test-schedule.mjs:660](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/admin/vm-test-schedule.mjs#L660) + +- **触发条件**：Kubernetes 模式正常完成启动，随后启动监控或触发定时器。 +- **现象及影响**：凭据刷新、健康探测和用量探测在进入实际任务前失败。定时测试调度器的初次加载失败，因而不会安装后续发现定时器。 +- **依据**：`src/server.mjs:1704` 先完成 bootstrap 活动，`:1709` 才调用定时测试调度器的 `start()`。凭据监控在调用 `sharedTasks.run()` 前执行 `routing.refresh()`；健康监控的 `src/lib/admin/health-probe.mjs:826`、用量监控的 `src/lib/oauth/usage-probe-monitor.mjs:316` 存在相同顺序。调度器在 `:660` 直接读取计划，`:712` 也在任务准入前刷新设置。这些查询进入 `src/lib/db/database.mjs:93` 后，由 `src/lib/db/global-maintenance.mjs:371` 拒绝已关闭或缺失的活动作用域，错误码为 `maintenance_activity_closed`。 +- **建议最小修复**：后台入口在首次数据库访问前取得独立活动准入。覆盖计划发现、配置刷新和 cron 回调的准备阶段。 +- **最小复现**：在合成 fixture 中初始化共享路由和协调器，完成初始化活动。退出该活动作用域后调用上述监控的执行方法及 `VmTestScheduleRunner.start()`。回调使用计数器代替外部操作。检查执行前的 `maintenance_activity_closed`，以及回调计数仍为零。测试不能一直包在 `withDatabaseActivities()` 内，否则会掩盖问题。 + +**3. P1：手动 Claude Code 初始化任务未被排空跟踪，可在恢复后继续产生副作用** + +位置：[src/lib/admin/panel-routes.mjs:3960](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/admin/panel-routes.mjs#L3960) + +- **触发条件**：调用 `POST /api/panel/vms/:id/official-cc-bootstrap`，任务停留在异步 CLI 或用量请求中。HTTP 响应结束后再执行备份恢复。 +- **现象及影响**：任务仍在执行时，请求活动已被标记完成。恢复可以越过该任务，旧任务随后继续修改恢复后的 VM 文件或重新加载槽位。 +- **依据**：接口调用 `scheduleOfficialCcBootstrap()` 后立即响应。`src/lib/oauth/official-cc-bootstrap.mjs:1468` 将任务保存到本地 `running` Map，但没有建立独立活动、后台任务声明或 VM 操作声明。`src/server.mjs:1251` 对没有执行记录的请求判定为结束；`:1677` 的维护监控列表和 `:1689` 的 flush 均不等待这个 Map。任务后续在 `src/lib/oauth/official-cc-bootstrap.mjs:1304` 写入指纹，在 `:1328` 调用会写文件并重新加载槽位的遥测初始化。 +- **建议最小修复**：为排队任务建立独立活动和 VM 操作声明，保持到任务、持久化及远端结束确认全部完成。维护排空必须等待这些任务。 +- **最小复现**：使用合成槽位及假的 Docker、CLI，将 `runTurn` 暂停在 deferred Promise。模拟接口完成并结束请求活动。此时申请维护，检查它是否错误地成功排空。模拟恢复 VM 文件后释放 Promise，检查旧任务是否继续写入文件或调用 reload。 + +**4. P1：包更新接口绕过 VM 互斥，可在执行占用期间改写凭据** + +位置：[src/lib/admin/panel-routes.mjs:3992](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/admin/panel-routes.mjs#L3992)、[src/lib/vm/vm-package.mjs:678](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/vm/vm-package.mjs#L678) + +- **触发条件**：目标 VM 或其物理别名存在未释放执行准入，另一个请求调用 `PUT /api/panel/vms/:id/package`。 +- **现象及影响**：接口没有先拒绝占用，而是改写 VM 配置和凭据。后续代理绑定即使拒绝操作，前面的文件修改也已经发生。 +- **依据**：接口直接调用 `commitVmPackage()`，没有经过 `mutateVms()` 或 `withVmLock()`。该函数在 `:678` 写 VM 文件，`:688` 写凭据；凭据写入函数位于 `:486`。代理处理直到 `:696` 才执行。接口成功后还可在 `src/lib/admin/panel-routes.mjs:4006` 直接重新加载槽位。以上路径均未查询执行准入，也未取得 VM 操作声明。 +- **建议最小修复**：从首次文件修改前取得现有 VM 操作声明，并保持到凭据、代理、运行时更新和持久化结束。占用拒绝必须发生在任何写入前。 +- **最小复现**：扩展 `test/unit/panel-vm-shared-mutations.test.mjs` 的 fixture。先为合成 VM 建立执行准入，再提交包含不同合成凭据的包更新，并设置 `packageImportBringUp: false`。断言应返回占用错误且文件不变；当前调用顺序会先改写文件。再使用物理别名重复该场景。 + +**5. P2：代理管理接口未取得新增要求的 VM 操作声明，空闲状态也会失败** + +位置：[src/lib/admin/panel-routes.mjs:5957](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/admin/panel-routes.mjs#L5957)、[src/lib/vm/proxy-pool.mjs:518](https://github.com/liubing0806/vm2api/blob/0ffae4b04c2429f472bef5a42add4a00a1ed7fcc/src/lib/vm/proxy-pool.mjs#L518) + +- **触发条件**：Kubernetes 模式下，通过正常面板接口给空闲 VM 绑定代理，或修改已经绑定 VM 的代理。 +- **现象及影响**：底层抛出 `vm_mutation_claim_required`。正常代理绑定和配置修改失败。该错误没有被面板包装器处理，HTTP 层会返回 500。 +- **依据**：代理池在 `src/lib/vm/proxy-pool.mjs:450` 强制校验当前回调持有的 VM 操作声明，`:518` 拒绝未覆盖的目标。面板的绑定 `:5957`、代理编辑 `:5796`、配置更新 `:5753` 都直接调用代理池，没有先取得声明。`src/lib/admin/panel-routes.mjs:6142` 的错误处理列表不包含该错误码。 +- **建议最小修复**：面板入口根据权威绑定集合取得现有 VM 操作声明，再调用代理池，并保持到相关运行时副作用结束。保留底层集合复核。 +- **最小复现**：使用实际 `ProxyPool` 和合成 VM，确保没有执行占用或 VM 操作声明。通过 `createPanelHandler()` 调用代理绑定接口。检查其抛出 `vm_mutation_claim_required`。已有 `test/unit/proxy-vm-mutations.test.mjs:40` 覆盖了底层拒绝条件，但没有覆盖面板入口取得声明的过程。 + +**未验证范围与审查边界** + +- 已核对 HEAD 为 `0ffae4b04c2429f472bef5a42add4a00a1ed7fcc`，基线为 `fcd568c92f50103fb3f9694de7dbffc3f238c764`。工作树无未提交改动。 +- 重点静态追踪了共享准入、执行释放、会话队列、维护恢复、后台任务、认证一次性状态、VM 物理别名、代理 revision 和管理接口副作用。 +- 未逐行穷尽全部 256 个改动文件。未执行测试、启动服务或访问外部环境。上述复现结果仍需主会话动态确认。 +- 未独立验证内核二进制内部的执行终止语义。 +- 本轮未发现可证实的 P0。未将保留 `ControlPlaneLock`、单副本部署或尚未启用的 node authority/KSG2 后续能力列为缺陷。本审查不构成发布批准。 diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-66a580de.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-66a580de.md new file mode 100644 index 000000000..26aeb0c20 --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-66a580de.md @@ -0,0 +1,77 @@ +发现 **4 项 P1 问题**。以下均为源码审查结论，动态复现未执行。未发现有证据支持的 P0。 + +1. **P1：manual bootstrap 会丢失 `/usage` 重试前的未知执行，并释放 claims。** + + 位置：[src/lib/oauth/official-cc-bootstrap.mjs:1554](https://github.com/liubing0806/vm2api/blob/66a580de17670a954638fbfb46fee25740adab78/src/lib/oauth/official-cc-bootstrap.mjs#L1554)。 + + 触发条件：hello 成功；某次 `/usage` 超时；后续重试取得有效限额；其他辅助操作成功；resident 关闭。 + + 调用链：`scheduleSharedOfficialCcBootstrap` → `runOfficialCcBootstrap` → `runOfficialCcUsage`。同文件第 759 行在发送 SIGTERM 后立即返回超时，没有等待退出证明。第 1047 行用后一次结果覆盖前一次结果。第 1337 行只根据 hello 和限额数据计算 `ok`。返回对象也没有汇总历史超时。因此，第 1561 行可以判定 terminal，随后释放 VM、后台任务和 activity，而早先的容器内命令仍可能执行。 + + 最小修复：汇总全部已启动操作的结束证据。遇到未知执行时停止重试，并将未知状态传递到所有 bootstrap 调用者。 + + 交主会话复现：在隔离测试中令第一次 `/usage` 超时且未确认退出，第二次成功提供限额。检查重试次数，以及 `vm_operations.released_at`、后台任务 `finished_at`。预期应保留未知；当前路径会按成功完成。 + +2. **P1：package 启动及凭证导入绕过新增 bootstrap 辅助操作检查。** + + 位置：[src/lib/admin/panel-routes.mjs:887](https://github.com/liubing0806/vm2api/blob/66a580de17670a954638fbfb46fee25740adab78/src/lib/admin/panel-routes.mjs#L887)、[src/lib/oauth/import-commit.mjs:302](https://github.com/liubing0806/vm2api/blob/66a580de17670a954638fbfb46fee25740adab78/src/lib/oauth/import-commit.mjs#L302)。 + + 触发条件：hello 和 `/usage` 成功，但 profile、models 或 identity 收集超时。 + + 调用链： + - `POST /api/panel/vms/package` → `mutateProxyVms` → `bringUpImportedVm` → 原始 `runOfficialCcBootstrap`。 + - 凭证导入 → `commitImportedOauth` → 原始 `runOfficialCcBootstrap`。 + + 原始函数在 `official-cc-bootstrap.mjs:1272` 收集 profile/models，在第 1312 行收集 identity，但第 1337 行的 `ok` 不包含这些操作。package 的 `vmEffect` 只检查顶层结果；导入路径只检查 `completed.ok`。新增的 `profile_error`、`models_error` 和辅助调用观察逻辑只存在于调度包装器中。这两条启用路径因此可能释放 claims，并在 HTTP 返回后允许其他管理操作或全局恢复。 + + 最小修复：让原始 bootstrap 返回统一的执行完成状态，并让三类入口使用同一判定。 + + 交主会话复现：分别覆盖 package 和凭证导入。令 bootstrap 返回实际可能产生的 `{ok:true,status:{profile_error:'timeout'}}`，检查后续 VM claim 是否仍被拒绝。当前调用者会释放 claim。 + +3. **P1：正常解绑或删除代理可能永久占用整个代理池。** + + 位置：[src/lib/vm/egress.mjs:1079](https://github.com/liubing0806/vm2api/blob/66a580de17670a954638fbfb46fee25740adab78/src/lib/vm/egress.mjs#L1079)、[src/lib/admin/panel-routes.mjs:6252](https://github.com/liubing0806/vm2api/blob/66a580de17670a954638fbfb46fee25740adab78/src/lib/admin/panel-routes.mjs#L6252)。 + + 触发条件：本地 VM 容器仍存在并连接代理网络；没有执行中的 admission；解绑最后一个绑定，或删除该代理。 + + 调用链：unbind/delete → 更新代理绑定 → `bindVmProxy(null)` → `stopProxyEgress` → `docker network rm`。`bindVmProxy` 只修改记录（`vm-registry.mjs:573`）；容器由 `vm-runtime.mjs:1090` 接入该网络，解绑路径没有停止或断开容器。网络删除因此会遇到 active endpoints。第 1085 行把这个确定存在的网络判为未知。随后 `proxyEffect` 将 pool 和 VM claims 标为 uncertain，接口返回 502。 + + 影响：绑定已经移除，但后续代理管理被持久 claim 拒绝。全局维护也会被未完成 claim 拒绝。新增确认逻辑使正常清理成为长期故障。 + + 最小修复：分开处理 helper 终止和网络回收。仍有已知容器端点时保留网络；需要断开端点时，在现有 claims 内按精确容器身份执行并确认。 + + 交主会话复现：扩展 panel proxy fixture，让 network inspect 持续返回同一网络 ID，network rm 返回 active endpoints，同时确认 helper 已结束。执行最后一次解绑，检查 502、已清空绑定和未释放 claims。 + +4. **P1：watchdog 的 bridge-heal 仍绕过共享代理互斥，并丢弃未知结果。** + + 位置：[src/lib/transport/kernel-watchdog.mjs:220](https://github.com/liubing0806/vm2api/blob/66a580de17670a954638fbfb46fee25740adab78/src/lib/transport/kernel-watchdog.mjs#L220)、[src/lib/vm/slot-recovery.mjs:625](https://github.com/liubing0806/vm2api/blob/66a580de17670a954638fbfb46fee25740adab78/src/lib/vm/slot-recovery.mjs#L625)。 + + 这是必要基线调用链中的遗留遗漏，本次新增互斥仍未覆盖。 + + 触发条件：watchdog 启用；运行中的本地 SOCKS VM 被判断为 helper 不存活。 + + 调用链：`tick:349` → `healBridgesOffPath` → `createSlotBridgeHealer` → `ensureProxyEgress`。该路径只有 `kernel_bridge_heal` 后台任务，没有 pool claim 或受影响 VM claims。`slot-recovery.mjs:650` 得到失败结果后只记录日志，丢弃 `vmOperationUncertain`；watchdog 第 226 行始终返回 `terminal:true`。 + + 影响：修复可以与已有 VM operation 或代理配置变更重叠。新增的 egress 身份检查即使返回未知，后台任务也会结束，无法继续阻止维护或重复修复。单活动控制面不能消除同进程中的并发。 + + 最小修复：首次副作用前取得 pool 和受影响 VM claims，并传递未知结果。 + + 交主会话复现：预先持有 VM operation，再触发 bridge healer，断言 egress 启动调用次数应为零。另令启动返回 `{ok:false,vmOperationUncertain:true}`，断言后台任务和 activity 应保持未知。 + +首轮五项核对如下。状态仅代表源码证据，不代表测试通过。 + +| 首轮项目 | 状态 | 证据与回归 | +|---|---|---| +| NUL 请求日志导致 PG 持久失败 | 原问题已修正 | `request-log.mjs:588` 在 summary 入库前转义全部字符串字段；debug 通过 JSON 序列化写入文本。新增测试覆盖拒绝请求、后续查询和备份，但本轮未运行。 | +| monitor 初始 SQL 先于 activity | 指定路径已修正 | credential、health、usage 配置读取先进入独立 activity；VM schedule 的 discovery 和 prepare 也已包裹。 | +| manual bootstrap 原始 Promise 未独立跟踪 | 跟踪已修正，完成判定仍有遗漏 | `official-cc-bootstrap.mjs:1506` 跟踪原始 job，独立取得 activity 和 VM claim，并纳入 shutdown flush；发现 1、2 仍影响未知执行保留。 | +| package 首次文件写前未取得 VM 互斥 | 首次写入问题已修正 | PUT 在 `panel-routes.mjs:4145` 取得 claims；POST 固定目标 ID，在第 4240 行取得 claims，并检查持久记录及物理 alias，避免改名后越出已持有身份。启动后的问题见发现 2。 | +| proxy 面板入口未取得 pool/VM claim | 主要入口已修正，存在清理回归及旁路 | `mutateProxyVms:676` 先取得 pool claim，再刷新绑定并取得 VM claims。正常清理回归见发现 3；后台旁路见发现 4。 | + +本轮核对的提交为 **`66a580de17670a954638fbfb46fee25740adab78`**，增量为 `0ffae4b04c2429f472bef5a42add4a00a1ed7fcc..HEAD`。开始和结束时 HEAD 一致，工作树无改动。 + +已覆盖共享熔断的事务锁、跨控制面累计、重复结果、代次重置，以及当前执行 hook、job 完成和 admission 释放顺序。按静态调用链未发现额外问题。也检查了 fleet、artifact、metadata 入口的 claims、物理 alias、HTTP 返回后的活动，以及全局维护对未完成业务 claims 的检查。 + +节点 restricted launch、Claude terminal parser 和 command-status 已检查持久顺序及 fence/ticket 身份约束。文档明确说明 KSG2 尚未接生产客户端和 PG；未将该后续能力或已提交测试规格视为已交付功能。`ControlPlaneLock` 和 `replicas: 1` 均仍保留。 + +本轮未运行任何测试、服务或外部操作。真实 PG 并发、Docker 端点、远端命令退出及 restore 时序均未动态验证。以上复现步骤交主会话执行，**本报告不批准发布**。 diff --git a/docs/plans/evidence/kubernetes/review-progress/verification.json b/docs/plans/evidence/kubernetes/review-progress/verification.json new file mode 100644 index 000000000..33f0ae1cc --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/verification.json @@ -0,0 +1,297 @@ +{ + "status": "in_progress_not_ha_acceptance", + "full_suite_source_sha": "0ffae4b04c2429f472bef5a42add4a00a1ed7fcc", + "full_suites": { + "unit": { + "files": 342, + "totals": { + "tests": 3932, + "pass": 3915, + "fail": 7, + "cancelled": 0, + "skipped": 10 + }, + "failures": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "error_sha256": "1afab608e2476a88783807d547c9b239ad4ac831679731a3ed4d44812b0c31d2", + "matches_6732a864_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_6732a864_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_6732a864_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "error_sha256": "7862d1144e6748d63aa6e98dabf5a9fa7d8d1582f7fbfe851d978eb15f6d5c92", + "matches_6732a864_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "error_sha256": "81b2ee3a4bb8b1ab4f351bede4398032dc9637c683b82b146630fd6857e94c2e", + "matches_6732a864_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "error_sha256": "01732f4307f58f479e9b3d8329d92a93d9c748c58028276022257d3645e77e87", + "matches_6732a864_error": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_6732a864_error": true + } + ], + "log_sha256": "c3a343df30076763d8e5405f1385b89a11bc9896a507275b4788333f204c0fcc" + }, + "e2e": { + "files": 30, + "totals": { + "tests": 127, + "pass": 109, + "fail": 17, + "cancelled": 0, + "skipped": 1 + }, + "failures": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "error_sha256": "a25cb59468e4decc45b1d8b3a66b0a4fbd2195af9f498da39aa83e92169825b5", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "error_sha256": "84e06146cf59485c3b46e05070f6a9a184ecb010a57ef8ab1b62a3ebae90c36a", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "error_sha256": "dfd84d60931072a864eb94f48ce13d4223e963ae919898236f1859bedc8d917e", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "error_sha256": "6ea819807eaee98e6a4c38817c9f011fb426e6a5ff499fa441d8f7bb07371461", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "error_sha256": "c23279127cad70a092d0cc7bb6ac400e05aff03e4614d1eaaa40252878c221c2", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "error_sha256": "97ae4156650fa0de2f613aa7753666781545cb2fe7818350a6df904ab696cdbc", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "error_sha256": "a2c6319267f1b752b63d96a41e15b1a5133952e88f0c145bc96dda53fd31cbe5", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "error_sha256": "ba0ac5e90ea008839723c328e8b172228820837ab895e425d2190c1aa5caa708", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "error_sha256": "8353154a58e84007d91c2d217c516dc771e359ddb43d527c7db6734ab5cff6ca", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "error_sha256": "085c13ab1e54fb324bce5520b12006fbf7f84b4a69a1c5cf9e09727b5b33da99", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "error_sha256": "0be25feaedaa697c09835d15c8cfdae694de2710330cadfadcbdfa146625a099", + "matches_6732a864_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_6732a864_error": true + } + ], + "log_sha256": "0b07581d3588fa0d7042cbc0926ef75c66e93e7a73182b0cd63fd94ba71f4843" + } + }, + "independent_review": { + "source_sha": "0ffae4b04c2429f472bef5a42add4a00a1ed7fcc", + "base_sha": "fcd568c92f50103fb3f9694de7dbffc3f238c764", + "session_id": "01a15ee0-3324-7490-928c-8343beb13b5e", + "method": "separate Codex CLI session, read-only detached worktree, static source review only", + "findings": { + "P1": 4, + "P2": 1 + }, + "raw_report_sha256": "f1b0fed39c969477b546b658363e0b6b90e5bccaace03e48e890fae684bdfdbf", + "fixes": [ + { + "finding": "NUL summary write", + "red_sha": "97e00cf3", + "fix_sha": "1c034cc6", + "verification": { + "files": 1, + "totals": { + "tests": 56, + "pass": 56, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "9530c4872e2c287c3e5798cf843958ad08f36cbeb54e00f18bbc2a2babf26f82" + }, + "verification_scope": "workspace request-log unit/E2E; summary sanitization matches fix commit, not a frozen whole-source check" + }, + { + "finding": "monitor preparation admission", + "red_sha": "04a016b2", + "fix_sha": "f666c923", + "reported_tests": 96 + }, + { + "finding": "official bootstrap original promise", + "red_sha": "3e5b3b1a", + "fix_sha": "3304c452", + "reported_tests": 85 + }, + { + "finding": "package writes before VM claim", + "red_sha": "2a58a9e9", + "fix_sha": "4fed2928", + "reported_tests": 83 + }, + { + "finding": "panel proxy claims", + "red_sha": "2ebc3f42", + "fix_sha": "b9fc7462", + "reported_tests": 107 + } + ], + "delta_review": { + "source_sha": "66a580de17670a954638fbfb46fee25740adab78", + "status": "completed_4_P1_reproduction_and_fixes_in_progress", + "session_id": "01a15f03-648a-7d92-bf3d-f4d20d4cf03e", + "findings": { + "P1": 4 + }, + "raw_report_sha256": "771838ebe137f75df8d9456bfa7590672fe90fa9562dcdda33219f22d0ef99bc" + } + }, + "additional_fixed_source_checks": [ + { + "source_sha": "f4a685b98f31d467b6a2c8c5a8b7cf7749bf15c5", + "scope": "watchdog recovery state and global fault gate", + "result": { + "files": 1, + "totals": { + "tests": 99, + "pass": 99, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "235ee5b910bba7e813bf7fabfb46ee58e6068aa0d8ea1388a046694ba2dbd09a" + } + }, + { + "source_sha": "bf7615f8be9adf4f569c2a7d802be514a4c58e64", + "scope": "shared circuits, execution admission, panel read/reset, failover, database and backup", + "result": { + "files": 1, + "totals": { + "tests": 142, + "pass": 142, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "eec09eea8dd6cd844d1b3ddf4b82d2dfd3efdc840e40a169d242625399a28fba" + } + }, + { + "source_sha": "e1e5d2a9e6cea391fb96aacc6b0757fa3ad0360f", + "scope": "captured KSG2/PG binding, actual synthetic Go gate plus two PG connections, immutable proof and bind/refresh exclusion", + "result": { + "files": 1, + "totals": { + "tests": 156, + "pass": 156, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "675fc3b7320b857a06df520380f210a4b72a5ee7f9c487f0f97dcf03e40e0402" + } + } + ], + "environment_boundary": "Dedicated task PostgreSQL plus synthetic/mock workers only. No production changes. Node-side and later controller increments have not been deployed to the singleton image.", + "remaining": [ + "shared refresh integration", + "captured node execution binding", + "trusted node-owned mutation and permissions", + "multi-node active/active and fault takeover acceptance" + ] +} From d2b622080ccf2b4897868643c5b8ae17f8dcf176 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:32:16 +0800 Subject: [PATCH 315/473] fix: preserve bootstrap execution proof across all producers --- src/lib/admin/panel-routes.mjs | 8 +- src/lib/oauth/import-commit.mjs | 3 +- src/lib/oauth/official-cc-bootstrap.mjs | 176 ++++++++++++++---- test/unit/official-cc-bootstrap.test.mjs | 34 ++++ .../official-cc-shared-lifecycle.test.mjs | 44 ++++- 5 files changed, 222 insertions(+), 43 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index e7b53e017..6814a7cc7 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -71,6 +71,7 @@ import { officialCcHome, readOfficialCcStatus, runOfficialCcBootstrap, + officialCcBootstrapFinished, scheduleOfficialCcBootstrap, } from '../oauth/official-cc-bootstrap.mjs' import { invalidateLiveCredentialCache } from './panel-live-credentials.mjs' @@ -884,9 +885,10 @@ export function createPanelHandler(ctx) { credentialMode: credentialModeOfVm(vm), } if (isKubernetesMode()) { - const completed = await vmEffect(operation, () => - (ctx.vmMutationOps?.runOfficialCcBootstrap || runOfficialCcBootstrap)(options), - ) + const completed = await vmEffect(operation, async () => { + const result = await (ctx.vmMutationOps?.runOfficialCcBootstrap || runOfficialCcBootstrap)(options) + return officialCcBootstrapFinished(result, options) ? result : { ...result, vmOperationUncertain: true } + }) bootstrap = { scheduled: true, vmId: id, completed: true, ok: completed?.ok === true } } else bootstrap = scheduleOfficialCcBootstrap(options) } diff --git a/src/lib/oauth/import-commit.mjs b/src/lib/oauth/import-commit.mjs index e54d4ec3c..a34babe51 100644 --- a/src/lib/oauth/import-commit.mjs +++ b/src/lib/oauth/import-commit.mjs @@ -27,6 +27,7 @@ import { officialCcUidGid, scheduleOfficialCcBootstrap, runOfficialCcBootstrap, + officialCcBootstrapFinished, normalizeOfficialCcConfig, loadOfficialCcConfig, materializeOfficialClaudeCredentials, @@ -300,7 +301,7 @@ export function createImportCommit(ctx) { } if (vmOperation) { const completed = await (ctx.runOfficialCcBootstrap || runOfficialCcBootstrap)(bootstrapOptions) - if (!completed?.ok && completed?.vmOperationNotStarted !== true) await vmOperation.markUncertain() + if (!officialCcBootstrapFinished(completed, bootstrapOptions)) await vmOperation.markUncertain() officialCc = { scheduled: true, vmId, completed: true, ok: completed?.ok === true } } else { officialCc = scheduleOfficialCcBootstrap(bootstrapOptions) diff --git a/src/lib/oauth/official-cc-bootstrap.mjs b/src/lib/oauth/official-cc-bootstrap.mjs index fd7f7b24c..d294b38ca 100644 --- a/src/lib/oauth/official-cc-bootstrap.mjs +++ b/src/lib/oauth/official-cc-bootstrap.mjs @@ -302,8 +302,9 @@ export function persistSlotRuntimeMemory(projectRoot, vmId, memory) { } export function restoreSlotMemoryAfterOfficialCc(vmId, idleMemory, { keepCli = false, vm } = {}) { - if (!keepCli) stopOfficialCcLeftovers(vmId, vm) - return applyContainerMemory(vmId, idleMemory || slotIdleMemory(vm), vm) + const stopped = keepCli ? null : stopOfficialCcLeftovers(vmId, vm) + const restored = applyContainerMemory(vmId, idleMemory || slotIdleMemory(vm), vm) + return stopped?.ok === false ? { ...restored, vmOperationUncertain: true } : restored } export function startOfficialCcResidentProcess({ vmId, projectRoot, uid, gid, timezone, locale, vm }) { @@ -602,6 +603,11 @@ export function writeOfficialCcStatus(homeDir, status) { resident_pid: Number.isFinite(Number(status.resident_pid)) ? Number(status.resident_pid) : null, bridge_port: Number.isFinite(Number(status.bridge_port)) ? Number(status.bridge_port) : null, error: status.error ? String(status.error).slice(0, 300) : null, + profile_error: status.profile_error || null, + models_error: status.models_error || null, + identity_error: status.identity_error || null, + stats_error: status.stats_error || null, + attempt_finished: status.attempt_finished === true, force: !!status.force, } fs.writeFileSync(officialCcStatusPath(homeDir), JSON.stringify(safe, null, 2) + '\n', { mode: 0o600 }) @@ -921,6 +927,7 @@ export async function runOfficialCcTurn({ const ok = officialCcTurnSucceeded({ code: finished.code, timedOut: finished.timed_out, raw, slash }) return { ok, + attemptFinished: officialCcTurnFinished(finished), timed_out: !!finished.timed_out, code: finished.code, error: finished.error || null, @@ -969,17 +976,23 @@ export async function finalizeOfficialCcTelemetry( applySeedAfterOfficialInit(homeDir, vm, seed, projectRoot) const tel = writeOfficialCcTelemetry(projectRoot, vmId) let reloaded = false + let uncertain = false if (reload && tel.enabled) { try { const r = await reloadSlot(vm, projectRoot, { routing }) reloaded = r?.ok === true && r?.rust_ok !== false - } catch {} + if (!reloaded || r.vmOperationUncertain === true) uncertain = true + } catch { + uncertain = true + } } let touched = false if (tel.enabled) { try { touched = !!touchTelemetrySession(projectRoot, vmId).wrote - } catch {} + } catch { + uncertain = true + } } return { wrote: !!tel.wrote, @@ -990,6 +1003,7 @@ export async function finalizeOfficialCcTelemetry( seed_aligned: true, reloaded, touched, + vmOperationUncertain: uncertain, } } @@ -1023,6 +1037,35 @@ export function writeOfficialCcTelemetry(projectRoot, vmId) { const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +// A normal foreground docker-exec exit follows the existing CLI protocol. A +// timeout only sends SIGTERM; it never proves that the guest command exited. +function officialCcTurnFinished(result) { + return ( + result?.vmOperationUncertain !== true && + !result?.timed_out && + result?.attemptFinished !== false && + (result?.attemptFinished === true || Number.isInteger(result?.code)) + ) +} + +/** Shared completion check for manual initialization, packages, and credential imports. */ +export function officialCcBootstrapFinished(result, { resident = false, syncTelemetry = false } = {}) { + if (!result || result.vmOperationUncertain === true || result.attemptFinished === false || result.timed_out) + return false + if (result.notStarted === true || result.vmOperationNotStarted === true) return true + if (resident || result.resident || result.status?.resident) return false + if (typeof result.attemptFinished === 'boolean') return result.attemptFinished + // Compatibility for injected/older producers without an explicit execution proof. + return ( + result.ok === true && + !result.status?.profile_error && + !result.status?.models_error && + !result.status?.identity_error && + !result.status?.stats_error && + !(syncTelemetry && result.status?.telemetry_enabled && !result.status.sidecar_reloaded) + ) +} + /** * Official `/usage` inside the slot (egress = slot SOCKS5 / transparent exit). * One try plus OFFICIAL_USAGE_RETRIES. A reply without the server limits[] @@ -1044,7 +1087,8 @@ export async function runOfficialCcUsage({ try { stats = parseOfficialCcStats(fs.readFileSync(outFile, 'utf8')) } catch {} - last = { turn: result, stats, attempts: attempt + 1 } + last = { turn: result, stats, attempts: attempt + 1, attemptFinished: officialCcTurnFinished(result) } + if (!last.attemptFinished) break if (stats?.ok && stats.limits_present) break } return last @@ -1053,13 +1097,15 @@ export async function runOfficialCcUsage({ /** Tier (from /api/oauth/profile) and model ids (from /v1/models) via the slot worker. */ export async function collectOfficialCcAccount(exec, { slotOauth } = {}) { const call = slotOauth || (await import('../transport/slot-oauth.mjs')).runSlotOauth - const [profile, models] = await Promise.all([ - call(exec, 'profile', { timeoutMs: 30_000 }), - call(exec, 'models', { timeoutMs: 30_000 }), + const responses = await Promise.allSettled([ + Promise.resolve().then(() => call(exec, 'profile', { timeoutMs: 30_000 })), + Promise.resolve().then(() => call(exec, 'models', { timeoutMs: 30_000 })), ]) + const [profile, models] = responses.map((response) => (response.status === 'fulfilled' ? response.value : null)) const tier = profile?.ok ? tierFromOauthProfile(profile.body) : null const list = Array.isArray(models?.body?.data) ? models.body.data : [] return { + attemptFinished: profile?.attemptFinished === true && models?.attemptFinished === true, account_tier: tier, profile_ok: !!profile?.ok, profile_error: profile?.ok ? null : profile?.body?.error?.code || `http_${profile?.status || 0}`, @@ -1116,10 +1162,31 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { const idleMemory = keepResident ? slotIdleMemory(vm, routing) : chooseIdleMemory(inspectContainerMemory(vmId), memory, slotIdleMemory(vm, routing)) - stopOfficialCcResident(vmId, projectRoot) - applyContainerMemory(vmId, memory, vm) + let uncertain = stopOfficialCcResident(vmId, projectRoot)?.ok === false + if (applyContainerMemory(vmId, memory, vm)?.ok === false) uncertain = true + let outcome + const complete = (value) => { + outcome = value + return value + } + const unknown = (step, error, extra = {}) => { + uncertain = true + return complete({ + ok: false, + ...extra, + status: writeOfficialCcStatus(homeDir, { + ...(readOfficialCcStatus(homeDir) || {}), + status: 'error', + vm_id: vmId, + started_at: startedAt, + step, + error, + }), + }) + } let helloOk = false try { + if (uncertain) return unknown('prepare', 'official Claude Code preparation did not confirm completion') writeOfficialCcStatus(homeDir, { status: 'running', vm_id: vmId, @@ -1133,8 +1200,12 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { doWipe === false ? { wiped: false, kept_credentials: true } : wipeOfficialFirstUseHome(homeDir, { uid, gid }) if (isSlotProxyDesynced(vm, projectRoot)) { try { - await reloadSlotReady(vm, projectRoot, { routing }) - } catch {} + const reloaded = await reloadSlotReady(vm, projectRoot, { routing }) + if (!reloaded?.ok || reloaded.rust_ok === false || reloaded.vmOperationUncertain === true) uncertain = true + } catch { + uncertain = true + } + if (uncertain) return unknown('reload', 'slot reload did not confirm completion') try { vm = JSON.parse(fs.readFileSync(vmPath, 'utf8')) } catch {} @@ -1157,6 +1228,8 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { }, { force: true, timeoutMs: 60_000 }, ) + if (refreshed?.vmOperationUncertain === true || refreshed?.attemptFinished === false) + return unknown('refresh', 'worker credential refresh did not confirm completion') if (!refreshed?.ok && refreshed?.status !== 200) { const status = writeOfficialCcStatus(homeDir, { status: 'error', @@ -1167,7 +1240,7 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { step: 'refresh', error: refreshed?.error?.message || `worker credential refresh failed (${refreshed?.status || 0})`, }) - return { ok: false, error: status.error, status } + return complete({ ok: false, error: status.error, status }) } } writeOfficialCcStatus(homeDir, { @@ -1188,7 +1261,7 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { wiped: wiped.wiped, error: login.error || 'failed to materialize official .credentials.json', }) - return { ok: false, error: status.error, status } + return complete({ ok: false, error: status.error, status }) } writeOfficialCcStatus(homeDir, { status: 'running', @@ -1210,9 +1283,11 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { wiped: wiped.wiped, error: 'slot cli-node missing; run wrap-cli sync first', }) - return { ok: false, error: status.error, status } + return complete({ ok: false, error: status.error, status }) } - const claudeVersion = (await spawnCaptured([cliNode, '--version'], { timeoutMs: 15_000 })).stdout || null + const version = await spawnCaptured([cliNode, '--version'], { timeoutMs: 15_000 }) + if (!officialCcTurnFinished(version)) return unknown('version', 'slot cli-node version check did not confirm exit') + const claudeVersion = version.stdout || null const timezone = vm.timezone || vm.fingerprint?.timezone || 'UTC' const locale = vm.locale || vm.fingerprint?.locale || 'en_US.UTF-8' @@ -1226,7 +1301,7 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { force, step: 'hello', }) - const hello = await runOfficialCcTurn({ + const hello = await runTurn({ vmId, uid, gid, @@ -1239,6 +1314,7 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { vm, }) helloOk = !!hello.ok + if (!officialCcTurnFinished(hello)) return unknown('hello', 'official Claude Code hello did not confirm exit') const turn = ({ prompt: text, outFile, errFile }) => runTurn({ vmId, uid, gid, timezone, locale, prompt: text, outFile, errFile, timeoutMs, vm }) writeOfficialCcStatus(homeDir, { @@ -1257,6 +1333,8 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { : { turn: { ok: false, code: null, timed_out: false, error: null }, stats: null, attempts: 0 } const statsTurn = usageRun.turn let stats = usageRun.stats + if (usageRun.attemptFinished === false) + return unknown('usage', 'official Claude Code /usage did not confirm exit', { stats }) writeOfficialCcStatus(homeDir, { status: 'running', @@ -1270,6 +1348,7 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { step: 'profile', }) const account = await collectOfficialCcAccount({ vmId, homeDir, vm }, { slotOauth }) + if (!account.attemptFinished) uncertain = true let tierSource = null if (stats?.limits_present === true && !stats.account_tier) { const accountTier = inferTierFromOfficialStats('', stats) @@ -1277,10 +1356,18 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { } if (!tierSource && stats?.account_tier) tierSource = 'usage' if (stats) stats = { ...stats, account_tier_source: tierSource } + let statsError = null if (typeof onStats === 'function' && stats) { try { - await onStats(stats, { vmId, projectRoot, homeDir }) - } catch {} + const saved = await onStats(stats, { vmId, projectRoot, homeDir }) + if (saved?.ok === false || saved?.vmOperationUncertain === true || saved?.attemptFinished === false) { + uncertain = true + statsError = 'stats_update_not_confirmed' + } + } catch { + uncertain = true + statsError = 'stats_update_failed' + } } const usageOk = officialCcQuotaSucceeded(stats) @@ -1307,10 +1394,18 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { } catch {} } if (reconcileFingerprint !== false) persistOfficialCcFingerprint(vmPath, homeDir) - if (typeof collectIdentity === 'function') { + let identityError = null + if (!uncertain && typeof collectIdentity === 'function') { try { - await collectIdentity(projectRoot, JSON.parse(fs.readFileSync(vmPath, 'utf8'))) - } catch {} + const identity = await collectIdentity(projectRoot, JSON.parse(fs.readFileSync(vmPath, 'utf8'))) + if (identity?.ok === false || identity?.vmOperationUncertain === true || identity?.attemptFinished === false) { + uncertain = true + identityError = 'identity_not_confirmed' + } + } catch { + uncertain = true + identityError = 'identity_failed' + } } writeOfficialCcStatus(homeDir, { status: 'running', @@ -1329,9 +1424,10 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { step: 'telemetry', }) const telemetry = - syncTelemetry !== false && hello.ok + !uncertain && syncTelemetry !== false && hello.ok ? await finalizeOfficialCcTelemetry(projectRoot, vmId, { reload: true, enable: true, routing }) : writeOfficialCcTelemetry(projectRoot, vmId) + if (telemetry.vmOperationUncertain === true) uncertain = true const homeAfter = summarizeOfficialCcHome(homeDir) const ok = !!(hello.ok && usageOk) @@ -1357,6 +1453,8 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { available_models: account.available_models, profile_error: account.profile_error, models_error: account.models_error, + identity_error: identityError, + stats_error: statsError, force, step: ok ? 'done' : hello.ok ? 'usage' : 'hello', telemetry_wrote: telemetry.wrote, @@ -1382,7 +1480,7 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { `official claude hello=${hello.code} usage=${usageOk ? 'ok' : 'fail'}`, }) let residentInfo = null - if (keepResident && hello.ok) { + if (!uncertain && keepResident && hello.ok) { writeOfficialCcStatus(homeDir, { ...status, step: 'resident', resident: true }) residentInfo = await keepOfficialCcResidentAfterHello({ vmId, @@ -1405,28 +1503,39 @@ export async function runOfficialCcBootstrap(rawOpts = {}) { error: status.error, }) } - return { + return complete({ ok, status: readOfficialCcStatus(homeDir) || status, stats, memory: { init: memory, idle: idleMemory }, resident: residentInfo, - } + }) } finally { if (!(keepResident && helloOk)) { - const restored = restoreSlotMemoryAfterOfficialCc(vmId, idleMemory, { vm }) + // Do not send cleanup commands after an unconfirmed foreground command. + // Cleanup cannot turn its timeout into proof that the original command ended. + const restored = uncertain ? null : restoreSlotMemoryAfterOfficialCc(vmId, idleMemory, { vm }) + if (restored?.ok === false || restored?.vmOperationUncertain === true) uncertain = true try { const prev = readOfficialCcStatus(homeDir) || {} writeOfficialCcStatus(homeDir, { ...prev, memory_init: memory, memory_idle: idleMemory, - memory_restored: restored.ok !== false, + memory_restored: restored?.ok === true && restored?.vmOperationUncertain !== true, resident: false, resident_ok: false, }) } catch {} } + if (outcome) { + outcome.attemptFinished = !uncertain && !outcome.resident + outcome.vmOperationUncertain = !outcome.attemptFinished + outcome.status = writeOfficialCcStatus(homeDir, { + ...(readOfficialCcStatus(homeDir) || outcome.status), + attempt_finished: outcome.attemptFinished, + }) + } } } @@ -1549,16 +1658,7 @@ async function scheduleSharedOfficialCcBootstrap(options) { }) await options.flush?.() await flushVmDbSync() - // Normal successful foreground CLI completion follows the existing protocol. - // A resident CLI remains active; a timeout/error supplies no termination proof. - const notStarted = result?.notStarted === true || result?.vmOperationNotStarted === true - const terminal = - !auxiliaryUnknown && - !result?.status?.profile_error && - !result?.status?.models_error && - !(options.syncTelemetry && result?.status?.telemetry_enabled && !result.status.sidecar_reloaded) && - result?.vmOperationUncertain !== true && - (notStarted || (result?.ok === true && !result.timed_out && options.resident !== true)) + const terminal = !auxiliaryUnknown && officialCcBootstrapFinished(result, options) if (!terminal) await operation.markUncertain() return { terminal, value: result } } catch (error) { diff --git a/test/unit/official-cc-bootstrap.test.mjs b/test/unit/official-cc-bootstrap.test.mjs index 5dc1f59e0..cf7694464 100644 --- a/test/unit/official-cc-bootstrap.test.mjs +++ b/test/unit/official-cc-bootstrap.test.mjs @@ -20,6 +20,7 @@ import { OFFICIAL_USAGE_RETRIES, runOfficialCcUsage, collectOfficialCcAccount, + officialCcBootstrapFinished, DEFAULT_OFFICIAL_CC_CONFIG, normalizeOfficialCcConfig, loadOfficialCcConfig, @@ -730,6 +731,39 @@ test('collectOfficialCcAccount reads tier from profile and model ids via slot wo assert.deepEqual(got.available_models, ['claude-fable-5', 'claude-sonnet-4-5']) }) +test('bootstrap completion requires execution proof separately from business success', () => { + assert.equal(officialCcBootstrapFinished({ ok: true, attemptFinished: false }), false) + assert.equal(officialCcBootstrapFinished({ ok: true, attemptFinished: true, vmOperationUncertain: true }), false) + assert.equal(officialCcBootstrapFinished({ ok: false, attemptFinished: true }), true) + assert.equal(officialCcBootstrapFinished({ ok: false, vmOperationNotStarted: true }), true) + assert.equal(officialCcBootstrapFinished({ ok: true, status: { models_error: 'timeout' } }), false) + assert.equal(officialCcBootstrapFinished({ ok: true, attemptFinished: true, resident: { ok: true } }), false) +}) + +test('account collection waits for both original operations when one rejects early', async () => { + let finish + const pending = new Promise((resolve) => { + finish = resolve + }) + let completed = false + const collection = collectOfficialCcAccount( + {}, + { + slotOauth: async (_exec, op) => { + if (op === 'profile') throw new Error('synthetic profile timeout') + return pending + }, + }, + ).then((value) => { + completed = true + return value + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(completed, false) + finish({ ok: true, attemptFinished: true, body: { data: [] } }) + assert.equal((await collection).attemptFinished, false) +}) + test('bootstrap source never calls Anthropic from the host', () => { const src = fs.readFileSync( path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../src/lib/oauth/official-cc-bootstrap.mjs'), diff --git a/test/unit/official-cc-shared-lifecycle.test.mjs b/test/unit/official-cc-shared-lifecycle.test.mjs index 2dc0b98ea..80627c412 100644 --- a/test/unit/official-cc-shared-lifecycle.test.mjs +++ b/test/unit/official-cc-shared-lifecycle.test.mjs @@ -133,7 +133,7 @@ test('a raw bootstrap usage timeout keeps task and VM ownership and prevents a l assert.equal(calls, 1) }) -for (const auxiliary of ['profile', 'models', 'identity']) { +for (const auxiliary of ['profile', 'models', 'identity', 'stats']) { test(`raw bootstrap reports an unknown ${auxiliary} even when hello and usage succeed`, async (t) => { const f = await fixture(t) prepareRawBootstrap(f, t) @@ -152,6 +152,10 @@ for (const auxiliary of ['profile', 'models', 'identity']) { ? { ok: false, status: 0, attemptFinished: false, body: { error: { code: 'timeout' } } } : call(exec, op) if (auxiliary === 'identity') f.options.collectIdentity = async () => ({ ok: false, error: 'synthetic timeout' }) + if (auxiliary === 'stats') + f.options.onStats = async () => { + throw new Error('synthetic persistence failure') + } let raw f.options.runBootstrap = async (options) => { raw = await runOfficialCcBootstrap(options) @@ -167,6 +171,44 @@ for (const auxiliary of ['profile', 'models', 'identity']) { }) } +for (const rejectProfile of [false, true]) { + test(`raw bootstrap releases after confirmed foreground completion (profile rejected: ${rejectProfile})`, async (t) => { + const f = await fixture(t) + prepareRawBootstrap(f, t) + f.options.runTurn = async ({ prompt, outFile }) => { + fs.writeFileSync( + outFile, + prompt === '/usage' + ? JSON.stringify({ five_hour: { utilization: 0.1 }, seven_day: { utilization: 0.2 } }) + : '{}', + ) + return { ok: true, code: 0, timed_out: false, attemptFinished: true } + } + if (rejectProfile) { + const call = f.options.slotOauth + f.options.slotOauth = (exec, op) => + op === 'profile' + ? { ok: false, status: 403, attemptFinished: true, body: { error: { code: 'permission_denied' } } } + : call(exec, op) + } + let raw + f.options.runBootstrap = async (options) => { + raw = await runOfficialCcBootstrap(options) + return raw + } + await scheduleOfficialCcBootstrap(f.options) + await f.tasks.flush() + assert.equal(raw.ok, true) + assert.equal(raw.attemptFinished, true) + assert.equal(raw.status.attempt_finished, true) + assert.equal(raw.vmOperationUncertain, false) + const next = await f.other.tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, true) + await next.release() + assert.notEqual((await f.peer.prepare('SELECT finished_at FROM background_task_runs').get()).finished_at, null) + }) +} + test('manual bootstrap rejects a busy VM before queued status or background effects are written', async (t) => { const f = await fixture(t) await new ExecutionAdmissionsRepo(f.peer).tryReserve({ From 068919906653acf0e6dd0f749f83cb808f9f36f1 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:32:20 +0800 Subject: [PATCH 316/473] feat: coordinate request credential refreshes and retain uncertain executions --- src/lib/oauth/openai-quota.mjs | 21 ++++ src/lib/oauth/shared-codex-refresh.mjs | 65 ++++++++++ src/lib/oauth/shared-credential-refresh.mjs | 7 +- src/lib/pool/failover-runner.mjs | 50 +++++--- src/lib/pool/shared-codex-runtime.mjs | 7 +- src/lib/protocol/handle-codex.mjs | 28 ++++- src/lib/transport/go-worker-client.mjs | 15 ++- src/lib/transport/kernel-router.mjs | 1 + src/lib/transport/slot-oauth.mjs | 14 +-- src/server.mjs | 24 +++- .../unit/openai-quota-shared-refresh.test.mjs | 111 +++++++++++------- test/unit/shared-codex-search.test.mjs | 37 ++++++ 12 files changed, 305 insertions(+), 75 deletions(-) create mode 100644 src/lib/oauth/shared-codex-refresh.mjs diff --git a/src/lib/oauth/openai-quota.mjs b/src/lib/oauth/openai-quota.mjs index 91451fc0e..9446baa0a 100644 --- a/src/lib/oauth/openai-quota.mjs +++ b/src/lib/oauth/openai-quota.mjs @@ -17,6 +17,9 @@ import { } from '../protocol/codex-usage.mjs' import { CODEX_OAUTH_ORIGINATOR, makeProxyFetch, refreshCodexAccessToken } from '../protocol/codex-models.mjs' import { openaiIdentityFromJwt } from './openai-jwt.mjs' +import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { refreshSharedCodexCredential } from './shared-codex-refresh.mjs' +import { credentialGeneration } from './shared-credential-refresh.mjs' export const CHATGPT_USAGE_URL = 'https://chatgpt.com/backend-api/wham/usage' export const CHATGPT_RESET_CREDITS_URL = 'https://chatgpt.com/backend-api/wham/rate-limit-reset-credits' @@ -290,6 +293,24 @@ async function quotaFetch(url, { method = 'GET', headers, body, proxyUrl, fetchI async function rotateIfNeeded(slot, projectRoot, vmId, fetchImpl) { if (!slot.refresh) return slot + if (isKubernetesMode()) { + const result = await refreshSharedCodexCredential({ + projectRoot, + vm: slot.vm, + fetchImpl, + proxyUrl: slot.proxyUrl, + timeoutMs: OPENAI_QUOTA_TIMEOUT_MS, + expectedGeneration: credentialGeneration(slot.first), + }) + if (result.unknown || !result.value?.ok) + return { ...slot, rotateError: result.value?.error || 'credential_refresh_unknown' } + return { + ...slot, + first: result.current.credential, + access: result.current.credential.access_token, + refresh: result.current.credential.refresh_token, + } + } const tok = await refreshCodexAccessToken({ refreshToken: slot.refresh, proxyUrl: slot.proxyUrl, diff --git a/src/lib/oauth/shared-codex-refresh.mjs b/src/lib/oauth/shared-codex-refresh.mjs new file mode 100644 index 000000000..705db8404 --- /dev/null +++ b/src/lib/oauth/shared-codex-refresh.mjs @@ -0,0 +1,65 @@ +import fs from 'node:fs' +import { getDb } from '../db/database.mjs' +import { codexCredentialPath, writeCodexAccounts } from '../vm/codex-slot.mjs' +import { flushVmDbSync } from '../vm/vm-db-sync.mjs' +import { refreshCodexAccessToken } from '../protocol/codex-models.mjs' +import { + credentialGeneration, + currentRefreshAdmission, + getSharedCredentialRefresh, +} from './shared-credential-refresh.mjs' + +/** Shared authority is the committed credential file, never a VM metadata fallback. */ +export async function refreshSharedCodexCredential({ + projectRoot, + vm, + db = getDb(), + refresh = refreshCodexAccessToken, + fetchImpl, + proxyUrl, + timeoutMs, + expectedGeneration, +}) { + const readCurrent = async () => { + const accounts = JSON.parse(fs.readFileSync(codexCredentialPath(projectRoot, vm.id), 'utf8'))?.accounts + const credential = accounts?.length === 1 ? accounts[0] : null + if (!credential?.chatgpt_account_id || !credential?.refresh_token) + throw Object.assign(new Error('shared Codex credential unavailable'), { code: 'codex_credential_unavailable' }) + return { credential, generation: credentialGeneration(credential) } + } + const observed = await readCurrent() + return getSharedCredentialRefresh(db).run( + { + platform: 'codex', + vmId: vm.id, + accountId: `codex:${observed.credential.chatgpt_account_id}`, + expectedGeneration: expectedGeneration === undefined ? observed.generation : expectedGeneration, + admission: currentRefreshAdmission(db), + }, + { + readCurrent, + refresh: async (current) => { + const value = await refresh({ refreshToken: current.credential.refresh_token, proxyUrl, fetchImpl, timeoutMs }) + return { finished: value?.attemptFinished === true, value } + }, + publish: async (current, value) => { + const fresh = await readCurrent() + if (fresh.generation !== current.generation) + throw Object.assign(new Error('Codex credential changed during refresh'), { + code: 'credential_generation_changed', + }) + writeCodexAccounts(projectRoot, vm.id, [ + { + ...fresh.credential, + access_token: value.access_token, + refresh_token: value.refresh_token || fresh.credential.refresh_token, + id_token: value.id_token || fresh.credential.id_token, + expires_at: value.expires_at || fresh.credential.expires_at, + }, + ]) + await flushVmDbSync() + return readCurrent() + }, + }, + ) +} diff --git a/src/lib/oauth/shared-credential-refresh.mjs b/src/lib/oauth/shared-credential-refresh.mjs index 5a879581d..009ac3f9b 100644 --- a/src/lib/oauth/shared-credential-refresh.mjs +++ b/src/lib/oauth/shared-credential-refresh.mjs @@ -17,9 +17,12 @@ export async function withRefreshAdmission(db, admission, callback) { } } -export function currentRefreshAdmission(db = getDb()) { +export function currentRefreshAdmission(db = null) { const context = admissionContext.getStore() - if (!context?.active || context.db.schema !== db.schema || context.db.connectionString !== db.connectionString) + if ( + !context?.active || + (db && (context.db.schema !== db.schema || context.db.connectionString !== db.connectionString)) + ) return null return context.admission } diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 4477e7cfd..6168b79fb 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -23,6 +23,8 @@ import { AttemptCoordinator } from './unit-decision.mjs' import { SOFT_COOLDOWN_REASONS } from './pool-scheduler.mjs' import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { SharedSessionTurns } from './shared-session-turns.mjs' +import { getDb } from '../db/database.mjs' +import { withRefreshAdmission } from '../oauth/shared-credential-refresh.mjs' // Seat wait for the one-shot repair hop when the rejected hop already spent the deadline. const REPAIR_HOP_WAIT_MS = 30_000 @@ -519,10 +521,22 @@ export class FailoverRunner { const beforeRefresh = String(before?.refresh_token || '') let refreshed try { - refreshed = await ensureWorkerCredential(exec, { force: true }) + refreshed = await this.withCredentialAdmission(selected, () => ensureWorkerCredential(exec, { force: true })) } catch (error) { + if (selected.bindExecution) { + await selected.markUncertain() + error.executionUncertain = true + throw error + } refreshed = { ok: false, error: { message: String(error?.message || error) } } } + if (selected.bindExecution && refreshed?.credentialRefreshUnknown) { + await selected.markUncertain() + throw Object.assign(new Error('credential refresh termination is unknown'), { + code: 'credential_refresh_unknown', + executionUncertain: true, + }) + } const after = readWorkerCredentialFile(exec.homeDir) const afterRefresh = String(after?.refresh_token || '') const blob = `${refreshed?.error?.code || ''} ${refreshed?.error?.message || ''} ${refreshed?.refresh_class || ''}` @@ -540,6 +554,12 @@ export class FailoverRunner { } } + withCredentialAdmission(selected, callback) { + if (!selected?.admissionId) return withRefreshAdmission(null, null, callback) + const db = this.scheduler?.executionAdmissions?.db || this.scheduler?.accountQuota?.db || getDb() + return withRefreshAdmission(db, selected, callback) + } + async forgetCredential(selected, policy, { familyKey = null, sessionKeys = [], versions = null } = {}) { if (familyKey) { await this.scheduler.releaseSticky?.(selected, null, sessionKeys, versions) @@ -962,20 +982,22 @@ export class FailoverRunner { if (!bound) return stickyConflict() } callEntered = true - result = await callAttempt({ - candidate: selected, - body, - attemptMeta, - attemptNo, - stream, - deliveryMode: deliveryMode || this.config.delivery_mode, - signal, - onCommit: () => { - committed = true - }, - }) + result = await this.withCredentialAdmission(selected, () => + callAttempt({ + candidate: selected, + body, + attemptMeta, + attemptNo, + stream, + deliveryMode: deliveryMode || this.config.delivery_mode, + signal, + onCommit: () => { + committed = true + }, + }), + ) executionEnded = result?.execution?.ended === true - if (selected.bindExecution && !executionEnded) { + if (selected.bindExecution && (!executionEnded || result?.executionUncertain)) { await selected.markUncertain() unknownExecution = true if (result) result.executionUncertain = true diff --git a/src/lib/pool/shared-codex-runtime.mjs b/src/lib/pool/shared-codex-runtime.mjs index 07a235bfc..5745972f1 100644 --- a/src/lib/pool/shared-codex-runtime.mjs +++ b/src/lib/pool/shared-codex-runtime.mjs @@ -4,6 +4,7 @@ import { SharedSessionTurns } from './shared-session-turns.mjs' import { readCodexAccounts } from '../vm/codex-slot.mjs' import { normalizeOpenAIQuotaPolicy } from './openai-quota-policy.mjs' import { getSharedRoutingState } from '../admin/shared-routing-config.mjs' +import { withRefreshAdmission } from '../oauth/shared-credential-refresh.mjs' function identityError() { return Object.assign(new Error('Shared Codex execution requires exactly one account with chatgpt_account_id'), { @@ -108,6 +109,10 @@ export class SharedCodexRuntime { } async invoke(lease, hop, args) { + return withRefreshAdmission(this.db, lease, () => this._invoke(lease, hop, args)) + } + + async _invoke(lease, hop, args) { let bound = false let completed = false try { @@ -126,7 +131,7 @@ export class SharedCodexRuntime { }, }) const notStarted = result?.execution?.started === false && result?.execution?.ended === true && !bound - if ((!bound || !completed || result?.execution?.ended !== true) && !notStarted) { + if (result?.executionUncertain || ((!bound || !completed || result?.execution?.ended !== true) && !notStarted)) { await lease.markUncertain() if (result) result.executionUncertain = true } diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index 15b88ee30..745d7e613 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -5,6 +5,8 @@ import path from 'node:path' import { randomUUID } from 'node:crypto' import fetch from 'node-fetch' import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { refreshSharedCodexCredential } from '../oauth/shared-codex-refresh.mjs' +import { credentialGeneration, currentRefreshAdmission } from '../oauth/shared-credential-refresh.mjs' import { SharedCodexRuntime } from '../pool/shared-codex-runtime.mjs' import { getVm, listVms, persistCodexUsage, syncCodexQuotaSchedule } from '../vm/vm-registry.mjs' import { isValidVmId } from '../vm/vm-file.mjs' @@ -1217,8 +1219,23 @@ function searchCredentials(projectRoot, vm) { * refresh token on its own, so first adopt a token it wrote meanwhile; otherwise refresh * once per slot, shared by concurrent callers. */ -async function refreshSearchAccess(projectRoot, vm, usedAccess, { proxyUrl, fetchImpl, refresh }) { +async function refreshSearchAccess(projectRoot, vm, usedAccess, { proxyUrl, fetchImpl, refresh, expectedGeneration }) { const now = searchCredentials(projectRoot, vm) + if (isKubernetesMode()) { + const result = await refreshSharedCodexCredential({ + projectRoot, + vm, + proxyUrl, + fetchImpl, + refresh, + expectedGeneration, + db: currentRefreshAdmission()?.db, + }) + return { + access: result.finished && result.value?.ok ? result.current.credential.access_token : '', + unknown: result.unknown, + } + } if (now.access && now.access !== usedAccess) return now.access if (!now.refresh) return '' let pending = searchRefreshes.get(vm.id) @@ -1322,7 +1339,14 @@ async function postCodexSearch({ if (!creds.access && !creds.refresh) return { status: 0, error: 'no_oauth_token', execution: notStarted } let out = creds.access ? await send(creds.access) : { status: 401, execution: notStarted } if ((out.status === 401 || out.status === 403) && !signal?.aborted) { - const access = await refreshSearchAccess(projectRoot, vm, creds.access, { proxyUrl, fetchImpl, refresh }) + const refreshed = await refreshSearchAccess(projectRoot, vm, creds.access, { + proxyUrl, + fetchImpl, + refresh, + expectedGeneration: credentialGeneration(creds.first), + }) + if (refreshed?.unknown) return { ...out, executionUncertain: true, error: 'credential_refresh_unknown' } + const access = typeof refreshed === 'string' ? refreshed : refreshed?.access if (access) out = await send(access) } return out diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index 1f4f43286..6c7c32952 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -34,6 +34,7 @@ import { } from '../core/errors.mjs' import { extraHeadersFromLimitError } from '../pool/quota-window.mjs' import { SlotgateClient } from './slotgate-client.mjs' +import { credentialGeneration } from '../oauth/shared-credential-refresh.mjs' const MAX_BODY = 64 * 1024 * 1024 const credentialRefreshTail = new Map() @@ -1119,7 +1120,7 @@ export async function workerHealth(exec, { timeoutMs = 3000, signal } = {}) { } } -export async function ensureWorkerCredential(exec, { force = false } = {}) { +export async function ensureWorkerCredential(exec, { force = false, ...refreshOptions } = {}) { if (isCrsMock()) { return { ok: true, @@ -1201,7 +1202,14 @@ export async function ensureWorkerCredential(exec, { force = false } = {}) { credential: credentialSummary(live, liveNow), } } - const result = await runSlotOauth(exec, 'refresh', { force: !!force }) + const result = await runSlotOauth(exec, 'refresh', { + ...refreshOptions, + expectedGeneration: + refreshOptions.expectedGeneration === undefined + ? credentialGeneration(credential) + : refreshOptions.expectedGeneration, + force: !!force, + }) const body = result?.body || {} const refreshedCredential = readWorkerCredentialFile(exec.homeDir) const mapped = { @@ -1211,6 +1219,9 @@ export async function ensureWorkerCredential(exec, { force = false } = {}) { refresh_class: result?.ok ? (body.refreshed ? 'rotated' : 'already_fresh') : undefined, credential: result?.ok ? credentialSummary({ ...(refreshedCredential || {}), ...body }, Date.now()) : undefined, error: result?.ok ? undefined : body.error, + attemptFinished: result?.attemptFinished === true, + ...(result?.credentialRefreshUnknown ? { credentialRefreshUnknown: true } : {}), + ...(result?.transportError ? { transportError: true } : {}), } if (!mapped.ok) mapped.refresh_class = classifyCredentialRefresh(mapped) return mapped diff --git a/src/lib/transport/kernel-router.mjs b/src/lib/transport/kernel-router.mjs index f356565df..db3527468 100644 --- a/src/lib/transport/kernel-router.mjs +++ b/src/lib/transport/kernel-router.mjs @@ -176,6 +176,7 @@ function credentialEnsureFailure(result, ensured) { }, }, credential_ensure_failed: true, + ...(ensured?.credentialRefreshUnknown ? { executionUncertain: true } : {}), } } diff --git a/src/lib/transport/slot-oauth.mjs b/src/lib/transport/slot-oauth.mjs index d7426f535..f58889820 100644 --- a/src/lib/transport/slot-oauth.mjs +++ b/src/lib/transport/slot-oauth.mjs @@ -127,7 +127,7 @@ async function runSharedSlotOauth(db, exec, op, options) { platform: 'claude', vmId, accountId, - expectedGeneration: options.expectedGeneration || current.generation, + expectedGeneration: options.expectedGeneration === undefined ? current.generation : options.expectedGeneration, admission, }, { @@ -168,12 +168,12 @@ async function runSharedSlotOauth(db, exec, op, options) { new SharedVmOperations(db).run( vmId, async () => { - entered = true - result = await runSlotOauthRaw(exec, op, options) - if (result?.attemptFinished === true && exec.homeDir) - await slotHost(exec.vm).afterRefresh(exec.vm, path.dirname(exec.homeDir)) - await flushVmDbSync() - finished = result?.attemptFinished === true + entered = true + result = await runSlotOauthRaw(exec, op, options) + if (result?.attemptFinished === true && exec.homeDir) + await slotHost(exec.vm).afterRefresh(exec.vm, path.dirname(exec.homeDir)) + await flushVmDbSync() + finished = result?.attemptFinished === true return { ...result, vmOperationUncertain: result?.attemptFinished !== true } }, { kind: `oauth_${op}`, accountId, admission }, diff --git a/src/server.mjs b/src/server.mjs index d1213a86c..41d4790e2 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -769,13 +769,29 @@ function publicCredentialRefreshResult(vmId, result, { force = false } = {}) { async function refreshWorkerCredentialForVm({ vmId, vmPath, homeDir, vm, force = false }) { // Host RefreshIfNeeded is the credential writer. Slot workers only read AT. - const result = await ensureWorkerCredential({ vmId, vm, homeDir }, { force: !!force }) + const result = await ensureWorkerCredential( + { vmId, vm, homeDir }, + { + force: !!force, + ...(kubernetesMode + ? { + publishCredential: async () => { + mirrorWorkerCredentialsToVm(vmPath, homeDir, { acceptLiveGrant: true }) + await flushVmDbSync() + invalidateLiveCredentialCache() + }, + } + : {}), + }, + ) const published = publicCredentialRefreshResult(vmId, result, { force }) if (result.ok) { - try { - mirrorWorkerCredentialsToVm(vmPath, homeDir, { acceptLiveGrant: true }) - } catch {} + if (!kubernetesMode) { + try { + mirrorWorkerCredentialsToVm(vmPath, homeDir, { acceptLiveGrant: true }) + } catch {} + } try { invalidateLiveCredentialCache() } catch {} diff --git a/test/unit/openai-quota-shared-refresh.test.mjs b/test/unit/openai-quota-shared-refresh.test.mjs index 6b139730e..c18bf9471 100644 --- a/test/unit/openai-quota-shared-refresh.test.mjs +++ b/test/unit/openai-quota-shared-refresh.test.mjs @@ -13,12 +13,28 @@ async function fixture(t, callback) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-shared-refresh-')) const vmId = 'vm-codex-refresh' fs.mkdirSync(path.join(root, 'vms', vmId), { recursive: true }) - fs.writeFileSync(path.join(root, 'vms', vmId + '.json'), JSON.stringify({ - id: vmId, platform: 'openai', family: 'codex', proxy: { url: 'socks5h://127.0.0.1:19999' }, - })) - fs.writeFileSync(path.join(root, 'vms', vmId, 'codex-credentials.json'), JSON.stringify({ - accounts: [{ id: 'synthetic', access_token: 'SYNTHETIC_OLD_AT', refresh_token: 'SYNTHETIC_RT', chatgpt_account_id: 'synthetic-account' }], - })) + fs.writeFileSync( + path.join(root, 'vms', vmId + '.json'), + JSON.stringify({ + id: vmId, + platform: 'openai', + family: 'codex', + proxy: { url: 'socks5h://127.0.0.1:19999' }, + }), + ) + fs.writeFileSync( + path.join(root, 'vms', vmId, 'codex-credentials.json'), + JSON.stringify({ + accounts: [ + { + id: 'synthetic', + access_token: 'SYNTHETIC_OLD_AT', + refresh_token: 'SYNTHETIC_RT', + chatgpt_account_id: 'synthetic-account', + }, + ], + }), + ) const previous = process.env.VM2API_CONTROL_PLANE_MODE process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' t.after(async () => { @@ -28,48 +44,57 @@ async function fixture(t, callback) { await coordinator?.stop() const client = new pg.Client({ connectionString: db.connectionString }) await client.connect() - try { if (coordinator) await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) } - finally { await client.end(); fs.rmSync(root, { recursive: true, force: true }) } + try { + if (coordinator) await client.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await client.end() + fs.rmSync(root, { recursive: true, force: true }) + } }) await withDatabaseActivities([db], () => callback({ db, root, vmId })) } const response = (status, payload) => ({ status, json: async () => payload }) -test('concurrent quota 401 recovery shares one published refresh attempt', async (t) => fixture(t, async ({ db, root, vmId }) => { - let refreshes = 0 - const fetchImpl = async (url, init) => { - if (url.includes('/oauth/token')) { - refreshes++ - await new Promise((resolve) => setTimeout(resolve, 30)) - return response(200, { access_token: 'SYNTHETIC_NEW_AT', refresh_token: 'SYNTHETIC_NEW_RT' }) +test('concurrent quota 401 recovery shares one published refresh attempt', async (t) => + fixture(t, async ({ db, root, vmId }) => { + let refreshes = 0 + const fetchImpl = async (url, init) => { + if (url.includes('/oauth/token')) { + refreshes++ + await new Promise((resolve) => setTimeout(resolve, 30)) + return response(200, { access_token: 'SYNTHETIC_NEW_AT', refresh_token: 'SYNTHETIC_NEW_RT' }) + } + if (init.headers.authorization === 'Bearer SYNTHETIC_OLD_AT') return response(401, {}) + return response(200, { rate_limit: {}, rate_limit_reset_credits: { available_count: 0 } }) } - if (init.headers.authorization === 'Bearer SYNTHETIC_OLD_AT') return response(401, {}) - return response(200, { rate_limit: {}, rate_limit_reset_credits: { available_count: 0 } }) - } - const results = await Promise.all([1, 2].map(() => queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }))) - assert.equal(refreshes, 1) - assert.ok(results.every((result) => result.ok)) - const rows = (await db.query('SELECT finished_at,shared_result FROM background_task_runs')).rows - assert.equal(rows.length, 1) - assert.ok(rows[0].finished_at) - assert.equal(JSON.stringify(rows).includes('SYNTHETIC_'), false) -})) + const results = await Promise.all([1, 2].map(() => queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }))) + assert.equal(refreshes, 1) + assert.ok(results.every((result) => result.ok)) + const rows = (await db.query('SELECT finished_at,shared_result FROM background_task_runs')).rows + assert.equal(rows.length, 1) + assert.ok(rows[0].finished_at) + assert.equal(JSON.stringify(rows).includes('SYNTHETIC_'), false) + })) -test('quota refresh transport failure leaves both durable claims unavailable', async (t) => fixture(t, async ({ db, root, vmId }) => { - let refreshes = 0 - const fetchImpl = async (url) => { - if (url.includes('/oauth/token')) { refreshes++; throw new Error('synthetic token disconnect') } - return response(401, {}) - } - const first = await queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }) - assert.equal(first.ok, false) - await queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }) - assert.equal(refreshes, 1) - const task = (await db.query('SELECT finished_at,unknown_at FROM background_task_runs')).rows[0] - assert.equal(task.finished_at, null) - assert.ok(task.unknown_at) - const operation = (await db.query('SELECT released_at,uncertain_at FROM vm_operations')).rows[0] - assert.equal(operation.released_at, null) - assert.ok(operation.uncertain_at) -})) +test('quota refresh transport failure leaves both durable claims unavailable', async (t) => + fixture(t, async ({ db, root, vmId }) => { + let refreshes = 0 + const fetchImpl = async (url) => { + if (url.includes('/oauth/token')) { + refreshes++ + throw new Error('synthetic token disconnect') + } + return response(401, {}) + } + const first = await queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }) + assert.equal(first.ok, false) + await queryOpenaiQuota({ projectRoot: root, vmId, fetchImpl }) + assert.equal(refreshes, 1) + const task = (await db.query('SELECT finished_at,unknown_at FROM background_task_runs')).rows[0] + assert.equal(task.finished_at, null) + assert.ok(task.unknown_at) + const operation = (await db.query('SELECT released_at,uncertain_at FROM vm_operations')).rows[0] + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + })) diff --git a/test/unit/shared-codex-search.test.mjs b/test/unit/shared-codex-search.test.mjs index 46985085c..779c9fb42 100644 --- a/test/unit/shared-codex-search.test.mjs +++ b/test/unit/shared-codex-search.test.mjs @@ -317,6 +317,43 @@ test('an uncertain Search token refresh retains its original admission and both }) }) +test('Search adopts a generation published while its unauthorized HTTP job was in flight', async (t) => { + let fixtureRoot + const fx = await fixture(t, (_req, res, count) => { + if (count === 1) { + fs.writeFileSync( + path.join(fixtureRoot, 'vms', vmId, 'codex-credentials.json'), + JSON.stringify({ + accounts: [ + { + id: 'synthetic-account', + access_token: 'synthetic-adopted-access', + refresh_token: 'synthetic-adopted-refresh', + chatgpt_account_id: 'synthetic-search-account', + }, + ], + }), + ) + return answer(res, 401) + } + return answer(res) + }) + fixtureRoot = fx.root + await withDatabaseActivities([fx.db, fx.other], async () => { + let refreshes = 0 + const result = await fx.run({ + refresh: async () => { + refreshes++ + return { ok: false, attemptFinished: true } + }, + }) + assert.equal(result.res.statusCode, 200) + assert.equal(refreshes, 0) + assert.equal(fx.calls[1].headers.authorization, 'Bearer synthetic-adopted-access') + assert.equal((await fx.counts()).accountInflight, 0) + }) +}) + for (const phase of ['timeout', 'cancel']) { test(`Search ${phase} after response headers retains its job and capacity`, async (t) => { let timer From 8dd813e8625b302d818f849228c36f9a175249bc Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:34:43 +0800 Subject: [PATCH 317/473] test: reproduce manual refresh bypassing VM claims and unknown retries --- .../panel-credential-refresh-claims.test.mjs | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 test/unit/panel-credential-refresh-claims.test.mjs diff --git a/test/unit/panel-credential-refresh-claims.test.mjs b/test/unit/panel-credential-refresh-claims.test.mjs new file mode 100644 index 000000000..0cea31b43 --- /dev/null +++ b/test/unit/panel-credential-refresh-claims.test.mjs @@ -0,0 +1,112 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import https from 'node:https' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { writeCodexAccounts } from '../../src/lib/vm/codex-slot.mjs' + +async function fixture(t, { codex = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-refresh-claims-')) + fs.mkdirSync(path.join(root, 'vms')) + const vm = { + id: 'vm-synthetic', + status: 'running', + ...(codex + ? { platform: 'openai', family: 'codex', codex: { has_refresh: true } } + : { claude: { mode: 'oauth', has_refresh: true } }), + } + fs.writeFileSync(path.join(root, 'vms', vm.id + '.json'), JSON.stringify(vm)) + if (codex) + writeCodexAccounts(root, vm.id, [ + { + id: 'synthetic-account', + chatgpt_account_id: 'synthetic-account', + access_token: 'synthetic-old-access', + refresh_token: 'synthetic-old-refresh', + }, + ]) + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + let calls = 0 + // Baseline Codex refresh must not send a real provider request, even before the guard exists. + t.mock.method(https, 'request', () => { + calls++ + throw new Error('synthetic network disconnect') + }) + t.after(async () => { + await peer.close() + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + fs.rmSync(root, { recursive: true, force: true }) + }) + function handler(extra = {}) { + const response = {} + const handle = createPanelHandler({ + cfg: { paths: { project: root } }, + routingConfig: {}, + vmOperations: new SharedVmOperations(db), + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + readBody: async () => ({ vm_id: vm.id }), + json(_res, status, body) { + response.status = status + response.body = body + return true + }, + refreshWorkerCredentialForVm: async () => { + calls++ + return { ok: true } + }, + ...extra, + }) + return { response, request: (route) => handle({ method: 'POST' }, {}, new URL('http://localhost' + route)) } + } + return { db, peer, root, vm, handler, calls: () => calls } +} + +for (const [codex, route] of [ + [true, '/api/panel/vms/vm-synthetic/oauth/refresh'], + [false, '/api/panel/vms/vm-synthetic/oauth/refresh'], + [false, '/admin/vm/oauth/refresh'], +]) { + test(`manual ${codex ? 'Codex' : 'Claude'} refresh ${route} rejects a global execution before sending`, async (t) => { + const f = await fixture(t, { codex }) + await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + vmId: f.vm.id, + accountId: 'synthetic-account', + vmLimit: 1, + accountLimit: 1, + rpmLimit: 0, + fableLimit: 0, + }) + const h = f.handler() + await h.request(route) + assert.equal(h.response.status, 409, JSON.stringify(h.response.body)) + assert.equal(f.calls(), 0) + }) +} + +test('an unknown manual Codex refresh is not resubmitted by the next HTTP request', async (t) => { + const f = await fixture(t, { codex: true }) + const first = f.handler() + await first.request('/api/panel/vms/vm-synthetic/oauth/refresh') + assert.equal(first.response.status, 502) + const second = f.handler() + await second.request('/api/panel/vms/vm-synthetic/oauth/refresh') + assert.equal(second.response.status, 409) + assert.equal(f.calls(), 1) + const operation = await f.peer.prepare('SELECT uncertain_at, released_at FROM vm_operations').get() + assert.notEqual(operation.uncertain_at, null) + assert.equal(operation.released_at, null) +}) From 6b003883cec310e563eb26e2b851a2312583a9ca Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:35:17 +0800 Subject: [PATCH 318/473] test: initialize Codex fixture before refresh guard regression --- test/unit/panel-credential-refresh-claims.test.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/unit/panel-credential-refresh-claims.test.mjs b/test/unit/panel-credential-refresh-claims.test.mjs index 0cea31b43..18274dda2 100644 --- a/test/unit/panel-credential-refresh-claims.test.mjs +++ b/test/unit/panel-credential-refresh-claims.test.mjs @@ -13,6 +13,7 @@ import { writeCodexAccounts } from '../../src/lib/vm/codex-slot.mjs' async function fixture(t, { codex = false } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-refresh-claims-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) fs.mkdirSync(path.join(root, 'vms')) const vm = { id: 'vm-synthetic', @@ -22,7 +23,8 @@ async function fixture(t, { codex = false } = {}) { : { claude: { mode: 'oauth', has_refresh: true } }), } fs.writeFileSync(path.join(root, 'vms', vm.id + '.json'), JSON.stringify(vm)) - if (codex) + if (codex) { + fs.mkdirSync(path.join(root, 'vms', vm.id)) writeCodexAccounts(root, vm.id, [ { id: 'synthetic-account', @@ -31,6 +33,7 @@ async function fixture(t, { codex = false } = {}) { refresh_token: 'synthetic-old-refresh', }, ]) + } const db = getDb() const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) const previous = process.env.VM2API_CONTROL_PLANE_MODE @@ -45,7 +48,6 @@ async function fixture(t, { codex = false } = {}) { await peer.close() if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = previous - fs.rmSync(root, { recursive: true, force: true }) }) function handler(extra = {}) { const response = {} From 4aa560cabce09ec3875ee6f56b63e44371d2b409 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:36:21 +0800 Subject: [PATCH 319/473] test: distinguish refused refreshes from uncertain started work during drain --- test/unit/slot-oauth-shared.test.mjs | 158 +++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/test/unit/slot-oauth-shared.test.mjs b/test/unit/slot-oauth-shared.test.mjs index fd81cc5c0..bf7e251ff 100644 --- a/test/unit/slot-oauth-shared.test.mjs +++ b/test/unit/slot-oauth-shared.test.mjs @@ -10,6 +10,9 @@ import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admiss import { readWorkerCredentialFile, writeWorkerCredentialFile } from '../../src/lib/oauth/oauth-credentials.mjs' import { runSlotOauth } from '../../src/lib/transport/slot-oauth.mjs' import { ensureWorkerCredential } from '../../src/lib/transport/go-worker-client.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { SharedCredentialRefresh } from '../../src/lib/oauth/shared-credential-refresh.mjs' async function fixture(t) { const db = getDb() @@ -52,6 +55,28 @@ async function fixture(t) { return { db, coordinator, homeDir, exec: { vmId: 'vm-01', homeDir, vm: {} }, run } } +async function startDrain(f, scope) { + let draining + const began = new Promise((resolve) => { + draining = resolve + }) + const pending = f.coordinator.beginMaintenance({ kind: 'backup', timeoutMs: 1000, onDraining: () => draining() }) + pending.catch(() => {}) + await began + return async () => { + await scope.complete({ ended: true }) + const operation = await pending.catch(() => null) + if (!operation) return false + try { + await operation.resume() + await operation.finish() + return true + } finally { + await operation.close() + } + } +} + test('the actual slot OAuth refresh shares a committed attempt after bootstrap has closed', async (t) => { const f = await fixture(t) let calls = 0 @@ -146,3 +171,136 @@ test('implicit worker refresh commands cannot execute while another inference is await f.run(() => gate.release()) } }) + +test('a DRAINING refusal after bootstrap closes proves the refresh never started', async (t) => { + const f = await fixture(t) + const scope = await f.coordinator.admit() + const finish = await startDrain(f, scope) + let calls = 0 + try { + await assert.rejects( + runSlotOauth(f.exec, 'refresh', { + runDocker: async () => { + calls++ + }, + }), + { + code: 'database_maintenance', + credentialRefreshNotStarted: true, + }, + ) + assert.equal(calls, 0) + } finally { + await finish() + } +}) + +test('a busy VM rejects refresh before mutation and completes its unused task claim', async (t) => { + const f = await fixture(t) + const operation = await f.run(() => new SharedVmOperations(f.db).tryAcquire({ vmId: 'vm-01' })) + let calls = 0 + try { + await assert.rejects( + runSlotOauth(f.exec, 'refresh', { + runDocker: async () => { + calls++ + }, + }), + { + code: 'vm_operation_active', + credentialRefreshNotStarted: true, + }, + ) + assert.equal(calls, 0) + assert.equal( + ( + await f.run(() => + f.db.prepare('SELECT count(*) AS n FROM background_task_runs WHERE finished_at IS NULL').get(), + ) + ).n, + 0, + ) + } finally { + await f.run(() => operation.release()) + } +}) + +for (const ended of [true, false]) { + test(`runner handles a DRAINING refresh refusal with first job ended=${ended}`, async (t) => { + const f = await fixture(t) + const scope = await f.coordinator.admit() + let gate + await scope.run(async () => { + gate = await new ExecutionAdmissionsRepo(f.db).tryReserve({ + accountId: 'slot-refresh-account', + vmId: 'vm-01', + accountLimit: 1, + vmLimit: 1, + rpmLimit: 0, + fableLimit: 0, + }) + await gate.bindExecution({ requestId: 'before-draining' }) + if (ended) + await gate.completeExecution({ + requestId: 'before-draining', + started: true, + ended: true, + reason: 'body_complete', + }) + }) + const runner = new FailoverRunner({ scheduler: { accountQuota: { db: f.db } } }) + const selected = { ...gate, exec: f.exec } + const finish = await startDrain(f, scope) + let drained + try { + await scope.run(async () => { + const recover = () => + runner.recoverCredential(selected, { reason: 'oauth_refresh_required' }, { executionEnded: ended }) + if (ended) { + const policy = await recover() + assert.equal(policy.credentialRefreshNotStarted, true) + assert.equal((await f.db.prepare('SELECT uncertain_at FROM execution_admissions').get()).uncertain_at, null) + assert.equal((await gate.release()).released, true) + } else { + await assert.rejects(recover(), { credentialRefreshNotStarted: true, executionUncertain: true }) + assert.ok((await f.db.prepare('SELECT uncertain_at FROM execution_admissions').get()).uncertain_at) + assert.equal((await gate.release()).released, false) + } + }) + } finally { + drained = await finish() + } + assert.equal(drained, ended) + }) +} + +test('the same maintenance error after entering refresh cannot claim it never started', async (t) => { + const f = await fixture(t) + let calls = 0 + await assert.rejects( + new SharedCredentialRefresh(f.db).run( + { + platform: 'claude', + vmId: 'vm-01', + accountId: 'slot-refresh-account', + expectedGeneration: 'generation-one', + }, + { + readCurrent: async () => ({ generation: 'generation-one', credential: { refresh_token: 'SYNTHETIC_RT' } }), + refresh: async () => { + calls++ + throw Object.assign(new Error('synthetic in-flight error'), { code: 'database_maintenance' }) + }, + publish: async () => assert.fail('no publication'), + }, + ), + (error) => { + assert.equal(error.credentialRefreshNotStarted, undefined) + return error.code === 'database_maintenance' + }, + ) + assert.equal(calls, 1) + const operation = await f.run(() => f.db.prepare('SELECT released_at,uncertain_at FROM vm_operations').get()) + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) +}) From 4efafc9b1fc8bfe04775e7269b665b600396bac1 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:36:47 +0800 Subject: [PATCH 320/473] test: stop after explicit unstarted refresh refusal with a completed model job --- test/unit/failover-runner.test.mjs | 35 ++++++++++++++++++++++++++++++ test/unit/kernel-router.test.mjs | 29 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/test/unit/failover-runner.test.mjs b/test/unit/failover-runner.test.mjs index 1149efe9d..a4d7a4db6 100644 --- a/test/unit/failover-runner.test.mjs +++ b/test/unit/failover-runner.test.mjs @@ -139,6 +139,41 @@ test('a finished model job with uncertain credential refresh retains its admissi assert.equal(result.executionUncertain, true) }) +for (const ended of [true, false]) { + test(`an explicit refresh refusal stops retry only after first job ended=${ended}`, async () => { + let marked = 0 + const first = { + ...candidate(1), + bindExecution: async () => {}, + markUncertain: async () => { + marked++ + }, + } + const scheduler = new Scheduler([first, candidate(2)]) + const runner = new FailoverRunner({ scheduler }) + let calls = 0 + const result = await runner.run({ + requestId: 'refused-refresh', + model: 'claude-test', + canonicalBody: { model: 'claude-test' }, + callAttempt: async () => { + calls++ + if (calls > 1) return success() + return { + ok: false, + status: 503, + execution: { ended }, + credentialRefreshNotStarted: true, + body: { error: { code: 'database_maintenance' } }, + } + }, + }) + assert.equal(calls, 1) + assert.equal(marked, ended ? 0 : 1) + assert.equal(result.executionUncertain === true, !ended) + }) +} + test('invalid panel cache layout stops before execution without rotating or cooling accounts', async () => { const scheduler = new Scheduler([candidate(1), candidate(2)]) const attempts = new Attempts() diff --git a/test/unit/kernel-router.test.mjs b/test/unit/kernel-router.test.mjs index 0777e7556..b91f34a69 100644 --- a/test/unit/kernel-router.test.mjs +++ b/test/unit/kernel-router.test.mjs @@ -522,6 +522,35 @@ unixTest('an uncertain credential refresh stays distinct from the first complete } }) +unixTest('a definitely unstarted refresh refusal preserves the completed kernel job result', async () => { + const previous = process.env.KIN_KERNEL_BIN + process.env.KIN_KERNEL_BIN = '/bin/true' + const fx = await kernelFixture((_req, res) => { + res.writeHead(401, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ type: 'error', error: { code: 'needs_refresh' } })) + }) + try { + const result = await dispatchCallInference({ + exec: fx.exec, + body: { model: 'claude-test', messages: [{ role: 'user', content: 'synthetic' }] }, + ensureRust: async () => ({ ok: true }), + ensureCredential: async () => { + throw Object.assign(new Error('synthetic explicit drain refusal'), { + code: 'database_maintenance', + credentialRefreshNotStarted: true, + }) + }, + }) + assert.equal(result.execution.ended, true) + assert.equal(result.credentialRefreshNotStarted, true) + assert.notEqual(result.executionUncertain, true) + } finally { + await fx.close() + if (previous == null) delete process.env.KIN_KERNEL_BIN + else process.env.KIN_KERNEL_BIN = previous + } +}) + unixTest('failed Go credential ensure is not followed by a blind Rust retry', async () => { const previous = process.env.KIN_KERNEL_BIN process.env.KIN_KERNEL_BIN = '/bin/true' From 2065da502bb2fabb96c99bdd2480e162cbe84daa Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:38:17 +0800 Subject: [PATCH 321/473] fix: release completed requests after explicitly refused credential refresh --- src/lib/oauth/shared-credential-refresh.mjs | 29 +++++++++++++----- src/lib/pool/failover-runner.mjs | 33 +++++++++++++++++++-- src/lib/transport/kernel-router.mjs | 15 +++++++++- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/lib/oauth/shared-credential-refresh.mjs b/src/lib/oauth/shared-credential-refresh.mjs index 009ac3f9b..cc493c81e 100644 --- a/src/lib/oauth/shared-credential-refresh.mjs +++ b/src/lib/oauth/shared-credential-refresh.mjs @@ -73,7 +73,13 @@ export class SharedCredentialRefresh { throw failure('credential_generation_required') if (!['claude', 'codex'].includes(input.platform) || !input.vmId || !input.accountId) throw failure('invalid_credential_refresh') - const activity = await getActivityCoordinator(this.db)?.admit({ kind: 'credential_refresh' }) + let activity + try { + activity = await getActivityCoordinator(this.db)?.admit({ kind: 'credential_refresh' }) + } catch (error) { + if (error.code === 'database_maintenance') error.credentialRefreshNotStarted = true + throw error + } const state = { unknown: false } try { const invoke = () => this._run(input, callbacks, state) @@ -118,11 +124,17 @@ export class SharedCredentialRefresh { observed.sharedResult.generation !== expectedGeneration ) throw failure('credential_projection_stale') - const claim = await this.tasks.tryClaim({ - taskKey, - executionId: randomUUID(), - ifPreviousExecutionId: observed?.executionId ?? null, - }) + let claim + try { + claim = await this.tasks.tryClaim({ + taskKey, + executionId: randomUUID(), + ifPreviousExecutionId: observed?.executionId ?? null, + }) + } catch (error) { + if (error.code === 'database_maintenance') error.credentialRefreshNotStarted = true + throw error + } if (!claim.ok) return this._joined(taskKey, await this.tasks.repo.latest(taskKey), callbacks) let acquired = false let metadata @@ -183,7 +195,10 @@ export class SharedCredentialRefresh { await claim.markUnknown() } else { // No mutation callback ran; the claim has caused no remote or credential effect. - await claim.complete({ sharedResult: { ok: false, code: 'refresh_not_started' } }) + if (!(await claim.complete({ sharedResult: { ok: false, code: 'refresh_not_started' } })).completed) + throw failure('credential_refresh_completion_unconfirmed') + if (['vm_operation_active', 'vm_execution_active'].includes(error.code)) + error.credentialRefreshNotStarted = true } throw error } diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 6168b79fb..da903ecc0 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -513,7 +513,7 @@ export class FailoverRunner { } } - async recoverCredential(selected, policy) { + async recoverCredential(selected, policy, { executionEnded = false } = {}) { if (policy?.reason !== 'oauth_refresh_required') return policy const exec = selected?.exec if (!exec?.homeDir) return permanentOAuthRevoke(policy) @@ -523,6 +523,15 @@ export class FailoverRunner { try { refreshed = await this.withCredentialAdmission(selected, () => ensureWorkerCredential(exec, { force: true })) } catch (error) { + if (selected.bindExecution && executionEnded === true && error.credentialRefreshNotStarted === true) + return { + ...policy, + retrySameAccount: false, + action: 'stop', + cooldownUntil: null, + reason: 'oauth_refresh_not_started', + credentialRefreshNotStarted: true, + } if (selected.bindExecution) { await selected.markUncertain() error.executionUncertain = true @@ -1071,6 +1080,8 @@ export class FailoverRunner { } if (unknownExecution) return incompleteHopResult(result, policy, { ...attribution(), executionUncertain: true }) + if (executionEnded && result?.credentialRefreshNotStarted === true) + return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } if (policy.reason === 'content_filter_refusal' || policy.reason === 'usage_policy_refusal') { await this.scheduler.markSuccess(selected, { workerStatus: result.workerStatus || null, @@ -1131,7 +1142,15 @@ export class FailoverRunner { if (!shouldContinue(policy)) { return { ...result, ...attribution(), finalState: result?.terminalState || 'rejected', policy } } - policy = await this.recoverCredential(selected, policy) + policy = await this.recoverCredential(selected, policy, { executionEnded }) + if (policy.credentialRefreshNotStarted === true) + return { + ...result, + ...attribution(), + finalState: result?.terminalState || 'rejected', + policy, + credentialRefreshNotStarted: true, + } const hopMs = Date.now() - attemptStarted if (isRetryableEmptyHop(policy)) { if (!emptyHopReleased(result) || pinVmId) return incompleteHopResult(result, policy, attribution()) @@ -1263,7 +1282,15 @@ export class FailoverRunner { return { ...result, ...attribution(), finalState: result.terminalState, policy } } } - policy = await this.recoverCredential(selected, policy) + policy = await this.recoverCredential(selected, policy, { executionEnded }) + if (policy.credentialRefreshNotStarted === true) + return { + ...result, + ...attribution(), + finalState: result.terminalState, + policy, + credentialRefreshNotStarted: true, + } const hopMs = Date.now() - attemptStarted if (budget.allowSameUnit(selected.accountId, policy, hopMs, this.config.same_account_retry_max_hop_ms)) { budget.noteSameUnit(selected.accountId) diff --git a/src/lib/transport/kernel-router.mjs b/src/lib/transport/kernel-router.mjs index db3527468..1b964e17c 100644 --- a/src/lib/transport/kernel-router.mjs +++ b/src/lib/transport/kernel-router.mjs @@ -177,6 +177,7 @@ function credentialEnsureFailure(result, ensured) { }, credential_ensure_failed: true, ...(ensured?.credentialRefreshUnknown ? { executionUncertain: true } : {}), + ...(ensured?.credentialRefreshNotStarted === true ? { credentialRefreshNotStarted: true } : {}), } } @@ -314,7 +315,19 @@ async function runHop({ mode, opts }) { } if (!isClientCancelledResult(result) && isNeedsRefreshResult(result) && canRetry(result)) { const ensure = opts.ensureCredential || ensureWorkerCredential - const ensured = await ensure(opts.exec, { force: true }) + let ensured + try { + ensured = await ensure(opts.exec, { force: true }) + } catch (error) { + if (error.credentialRefreshNotStarted !== true || result.execution?.ended !== true || result.executionUncertain) + throw error + ensured = { + ok: false, + status: 503, + credentialRefreshNotStarted: true, + error: { code: error.code, message: error.message }, + } + } if (ensured?.ok !== true) result = credentialEnsureFailure(result, ensured) else { // The CLI invalidates its OAuth cache from the shared credential file From 7e9039ef31506efde35c34ac72a4773e9a48db1d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:39:56 +0800 Subject: [PATCH 322/473] test: reproduce SFTP publication errors and early operation release --- test/unit/remote-slot-publication.test.mjs | 176 ++++++++++++++++++++ test/unit/remote-slot-restart-loop.test.mjs | 32 +++- 2 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 test/unit/remote-slot-publication.test.mjs diff --git a/test/unit/remote-slot-publication.test.mjs b/test/unit/remote-slot-publication.test.mjs new file mode 100644 index 000000000..d186b4d0b --- /dev/null +++ b/test/unit/remote-slot-publication.test.mjs @@ -0,0 +1,176 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { EventEmitter } from 'node:events' +import { bindPlacement, unbindPlacement } from '../../src/lib/cluster/placement.mjs' +import { queueSlotPush } from '../../src/lib/cluster/remote-slot-files.mjs' +import { writeKernelConfig } from '../../src/lib/transport/rust-kernel-supervisor.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' + +async function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'remote-publication-')) + const vm = { id: 'vm-synthetic', node_id: 'node-synthetic', timezone: 'UTC', status: 'stopped' } + const slotDir = path.join(root, 'vms', vm.id) + fs.mkdirSync(path.join(slotDir, 'run'), { recursive: true }) + fs.writeFileSync(path.join(root, 'vms', vm.id + '.json'), JSON.stringify(vm)) + fs.writeFileSync(path.join(slotDir, 'run', 'worker.json'), '{}') + let entered + const started = new Promise((resolve) => { + entered = resolve + }) + let completed = false + let failure = null + const waiting = [] + const writes = [] + const sftp = new EventEmitter() + Object.assign(sftp, { + open: (_file, _flags, _attrs, callback) => callback(null, Buffer.from('synthetic-handle')), + write: (_handle, _data, _offset, _length, _position, callback) => callback(null), + fchmod: (_handle, _mode, callback) => callback(null), + close: (_handle, callback) => callback(null), + unlink: (_file, callback) => callback(null), + ext_openssh_rename: (_from, to, callback) => { + writes.push(to) + entered() + if (completed) callback(failure) + else waiting.push(callback) + }, + }) + const client = { + sftp: (callback) => callback(null, sftp), + exec: (_command, callback) => { + const stream = new EventEmitter() + stream.stderr = new EventEmitter() + stream.close = () => {} + callback(null, stream) + queueMicrotask(() => { + stream.emit('exit', 0) + stream.emit('close') + }) + }, + } + bindPlacement({ + projectRoot: root, + manager: { + client: () => client, + remoteHost: async () => ({ root: '/synthetic-node', uid: 1000, gid: 1000 }), + }, + }) + const finish = (error = null) => { + completed = true + failure = error + for (const callback of waiting.splice(0)) callback(error) + } + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes' + t.after(async () => { + finish() + unbindPlacement() + await peer.close() + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + fs.rmSync(root, { recursive: true, force: true }) + }) + return { + root, + vm, + slotDir, + db, + peer, + started, + finish, + writes, + operations: new SharedVmOperations(db), + other: new SharedVmOperations(peer), + } +} + +test('queued SFTP errors reject the exact publication promise after logging', async (t) => { + const f = await fixture(t) + const logged = [] + const publication = queueSlotPush(f.vm, f.slotDir, ['run'], { logger: { warn: (line) => logged.push(line) } }) + await f.started + f.finish(new Error('synthetic connection lost after write')) + await assert.rejects(publication, { code: 'sftp_failed', vmOperationUncertain: true, executionUncertain: true }) + assert.equal(logged.length, 1) +}) + +test('kernel config publication keeps its VM claim until the original SFTP write completes', async (t) => { + const f = await fixture(t) + let ended = false + const operation = f.operations.run(f.vm.id, async () => { + const written = writeKernelConfig(f.root, f.vm, { token: 'synthetic-token' }) + await written.publication + ended = true + }) + try { + await f.started + const next = await f.other.tryAcquire({ vmId: f.vm.id }) + if (next.ok) await next.release() + assert.equal(ended, false) + assert.equal(next.ok, false) + } finally { + f.finish() + await operation + } + const next = await f.other.tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, true) + await next.release() +}) + +test('an unconfirmed kernel config publication retains persistent VM ownership', async (t) => { + const f = await fixture(t) + const operation = f.operations.run(f.vm.id, async () => { + const written = writeKernelConfig(f.root, f.vm, { token: 'synthetic-token' }) + await written.publication + }) + operation.catch(() => {}) + await f.started + f.finish(new Error('synthetic connection lost after write')) + await assert.rejects(operation, { code: 'sftp_failed' }) + const row = await f.peer.prepare('SELECT released_at, uncertain_at FROM vm_operations').get() + assert.equal(row.released_at, null) + assert.notEqual(row.uncertain_at, null) +}) + +test('panel timezone publication completes before HTTP success and VM claim release', async (t) => { + const f = await fixture(t) + let response + const handler = createPanelHandler({ + cfg: { paths: { project: f.root } }, + routingConfig: {}, + vmOperations: f.operations, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, + readBody: async () => ({ timezone: 'Europe/Paris' }), + json(_res, status, body) { + response = { status, body } + return true + }, + }) + const request = handler({ method: 'PATCH' }, {}, new URL('http://localhost/api/panel/vms/' + f.vm.id)) + try { + await f.started + assert.equal( + await Promise.race([request.then(() => true), new Promise((resolve) => setTimeout(() => resolve(false), 100))]), + false, + ) + assert.equal(response, undefined) + const next = await f.other.tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, false) + } finally { + f.finish() + await request + } + assert.equal(response.status, 200) +}) diff --git a/test/unit/remote-slot-restart-loop.test.mjs b/test/unit/remote-slot-restart-loop.test.mjs index 0f14e0530..d988b610b 100644 --- a/test/unit/remote-slot-restart-loop.test.mjs +++ b/test/unit/remote-slot-restart-loop.test.mjs @@ -20,6 +20,7 @@ function fixture({ mismatch = false, kind = 'docker', removeFails = false, + publication = null, } = {}) { const calls = [] const target = { @@ -60,7 +61,10 @@ function fixture({ if (removeFails) throw new Error('remove denied') info = null }, - writeWorkerFiles: () => calls.push('files'), + writeWorkerFiles: () => { + calls.push('files') + return { publication } + }, ensureGuestMachineIdFile: () => {}, pushSlotFiles: async () => {}, migrateClaudeDir: async () => {}, @@ -117,6 +121,32 @@ test('failed restart-loop removal does not create or claim successful recovery', assert.deepEqual(f.calls, ['remove']) }) +test('remote start awaits its original worker publication before creating or starting a container', async () => { + let finish + const publication = new Promise((resolve) => { + finish = resolve + }) + const f = fixture({ missing: true, publication }) + const starting = f.boot() + try { + await new Promise((resolve) => setImmediate(resolve)) + assert.deepEqual(f.calls, ['files']) + } finally { + finish() + await starting + } + assert.equal(f.calls.includes('create'), true) +}) + +test('remote start cannot hide a failed queued publication behind its later direct push', async () => { + const publication = Promise.reject(new Error('synthetic queued SFTP disconnect')) + publication.catch(() => {}) + const f = fixture({ missing: true, publication }) + const result = await f.boot() + assert.equal(result.ok, false) + assert.equal(f.calls.includes('create'), false) +}) + test('watchdog revival reaches the remote lifecycle and clears a restart loop without spending another attempt', async () => { const f = fixture({ restarting: true }) let now = 0 From d4c2d105b763824f3cd944377ceb56bfef2e0d64 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:41:11 +0800 Subject: [PATCH 323/473] test(slotgate): reject removal of an initialized management binding --- worker/internal/slotgate/allocate_test.go | 77 +++++++++++++++++++---- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/worker/internal/slotgate/allocate_test.go b/worker/internal/slotgate/allocate_test.go index 99a98e6aa..d803ee1a6 100644 --- a/worker/internal/slotgate/allocate_test.go +++ b/worker/internal/slotgate/allocate_test.go @@ -5,6 +5,7 @@ import ( "errors" "path/filepath" "sync" + "strings" "testing" ) @@ -15,35 +16,89 @@ func allocationIntent(id string) AllocationIntent { func TestAllocationReceiptCreatesOnlyOneInitialGenerationAndSurvivesReopen(t *testing.T) { root := filepath.Join(shortRoot(t), "authority") a, err := OpenAuthority(root, nil) - if err != nil { t.Fatal(err) } + if err != nil { + t.Fatal(err) + } intent := allocationIntent("allocation-one") var wg sync.WaitGroup receipts := make(chan AllocationReceipt, 2) for range 2 { wg.Add(1) - go func() { defer wg.Done(); receipt, err := a.Allocate(intent, "1"); if err != nil { t.Error(err); return }; receipts <- receipt }() + go func() { + defer wg.Done() + receipt, err := a.Allocate(intent, "1") + if err != nil { + t.Error(err) + return + } + receipts <- receipt + }() } wg.Wait() close(receipts) var first AllocationReceipt for receipt := range receipts { - if receipt.Fence.Generation != 1 || receipt.Fence.Executor != (ExecutorIdentity{}) { t.Fatalf("not a vacant initial generation: %+v", receipt) } - if first.AllocationID != "" && first != receipt { t.Fatalf("duplicate allocation changed receipt") } + if receipt.Fence.Generation != 1 || receipt.Fence.Executor != (ExecutorIdentity{}) { + t.Fatalf("not a vacant initial generation: %+v", receipt) + } + if first.AllocationID != "" && first != receipt { + t.Fatalf("duplicate allocation changed receipt") + } first = receipt } - if first.AllocationID == "" { t.Fatal("no receipt") } - if _, err = a.Allocate(allocationIntent("other-allocation"), "1"); !errors.Is(err, ErrAllocationConflict) { t.Fatalf("replace intent: %v", err) } - if err = a.Close(); err != nil { t.Fatal(err) } + if first.AllocationID == "" { + t.Fatal("no receipt") + } + if _, err = a.Allocate(allocationIntent("other-allocation"), "1"); !errors.Is(err, ErrAllocationConflict) { + t.Fatalf("replace intent: %v", err) + } + if err = a.Close(); err != nil { + t.Fatal(err) + } a, err = OpenAuthority(root, nil) - if err != nil { t.Fatal(err) } + if err != nil { + t.Fatal(err) + } t.Cleanup(func() { a.Close() }) replayed, err := a.Allocate(intent, "1") - if err != nil || replayed != first { t.Fatalf("lost-response replay: %+v %v", replayed, err) } + if err != nil || replayed != first { + t.Fatalf("lost-response replay: %+v %v", replayed, err) + } } func TestAllocationDeniedByAuthorizerMakesNoLocalSlot(t *testing.T) { a := authorityForTest(t, filepath.Join(shortRoot(t), "authority"), nil) manager := allocationManager{authority: a, nodeID: "synthetic-node", authorize: func(context.Context, AllocationIntent, func(string) error) error { return ErrManagementDenied }} - if _, err := manager.Allocate(context.Background(), allocationIntent("denied")); !errors.Is(err, ErrManagementDenied) { t.Fatal(err) } - if _, err := a.State("v"); !errors.Is(err, ErrInvalidAuthority) { t.Fatalf("denied allocation left state: %v", err) } + if _, err := manager.Allocate(context.Background(), allocationIntent("denied")); !errors.Is(err, ErrManagementDenied) { + t.Fatal(err) + } + if _, err := a.State("v"); !errors.Is(err, ErrInvalidAuthority) { + t.Fatalf("denied allocation left state: %v", err) + } +} + +func TestKSG2ForwardTokenAloneCannotAuthorizeManagement(t *testing.T) { + addr, _, server, state, _ := fencedGate(t) + server.authorizeManagement = nil + for _, op := range []string{"allocate", "launch", "revoke", "advance"} { + reply := gate2Control(t, addr, gate2Request{Op: op, VMID: "v", Fence: state.Fence, CommandID: "unauthorized"}) + if reply.OK || reply.Error != ErrManagementDenied.Error() { + t.Fatalf("%s: %+v", op, reply) + } + } +} + +func TestInitializedManagementBindingCannotBeRemovedOrRelabeled(t *testing.T) { + root := shortRoot(t) + management := &PGManagementConfig{ConnectionString:"postgres://synthetic@127.0.0.1:1/synthetic_test?sslmode=disable",AuthoritySchema:"vm2api_node_"+strings.Repeat("a",32),NodeID:"synthetic-node"} + config := Config{Listen:"127.0.0.1:0",Root:root,Token:testToken,TLSDir:filepath.Join(root,"tls"),Fenced:true,AuthorityDir:filepath.Join(root,"authority"),Management:management} + server,err := New(config) + if err != nil { t.Fatal(err) } + server.authority.Close() + config.Management = nil + if reopened,err := New(config); err == nil { reopened.authority.Close();t.Fatal("initialized node accepted missing authorizer") } + copy := *management + copy.NodeID = "different-node" + config.Management = © + if reopened,err := New(config); err == nil { reopened.authority.Close();t.Fatal("initialized node accepted a different identity") } } From c05da6f2b3c71b8c27bd6257bf7d64778681edd6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:43:31 +0800 Subject: [PATCH 324/473] fix: await original remote slot file publication and propagate uncertainty --- src/lib/admin/panel-routes.mjs | 16 +++++---- src/lib/admin/routing-runtime.mjs | 26 ++++++++++----- src/lib/cluster/native-slot.mjs | 10 ++++-- src/lib/cluster/remote-slot-files.mjs | 17 +++++++--- src/lib/cluster/remote-slot.mjs | 13 ++++++-- src/lib/oauth/official-cc-bootstrap.mjs | 4 ++- src/lib/transport/rust-kernel-supervisor.mjs | 22 +++++++++--- src/lib/vm/slot-host.mjs | 5 ++- src/lib/vm/vm-runtime.mjs | 6 ++-- src/lib/vm/worker-telemetry.mjs | 4 +-- test/unit/remote-slot-publication.test.mjs | 35 +++++++++++++++++++- test/unit/routing-runtime-load.test.mjs | 12 ++++--- 12 files changed, 126 insertions(+), 44 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 6814a7cc7..4097b4066 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -524,7 +524,7 @@ async function syncInstalledKernels({ continue } if (resolveInferenceEngine(vm, routingConfig) !== 'rust') continue - writeKernelConfig(project, vm, { routing: routingConfig, timezone: vm.timezone }) + await writeKernelConfig(project, vm, { routing: routingConfig, timezone: vm.timezone })?.publication const exec = slotExec(project, vm) item.kernel = await effect(item.id, () => restartRustKernel(exec)).catch((e) => ({ ok: false, @@ -1212,7 +1212,7 @@ export function createPanelHandler(ctx) { return { ok: false, id, code: laid.code || 'dataplane_materialize_failed', error: laid.error } } } - writeKernelConfig(cfg.paths.project, saved, { routing: ctx.routingConfig }) + await writeKernelConfig(cfg.paths.project, saved, { routing: ctx.routingConfig })?.publication if (previousDataplane !== targetDataplane && saved.status === 'running') { const exec = slotExec(cfg.paths.project, saved) const restarted = await restartRustKernel(exec).catch((e) => ({ @@ -1253,13 +1253,15 @@ export function createPanelHandler(ctx) { } } - function projectSlotKernelConfig(vm) { + async function projectSlotKernelConfig(vm) { if (!vm?.id || isCodexVm(vm)) return null const full = getVm(cfg.paths.project, vm.id) || vm - return writeKernelConfig(cfg.paths.project, full, { + const written = writeKernelConfig(cfg.paths.project, full, { routing: ctx.routingConfig, timezone: full.timezone, }) + await written?.publication + return written } async function restoreRoutingRuntime(previous) { @@ -3145,14 +3147,14 @@ export function createPanelHandler(ctx) { if (hasTimezone) { const vm = persistVmTimezone(cfg.paths.project, id, timezoneSync.timezone, { source: 'manual' }) if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) - const kernel = projectSlotKernelConfig(vm) + const kernel = await projectSlotKernelConfig(vm) timezoneSync.applied = true timezoneSync.kernel_hot = kernel?.changed === true } else if (followProxyTz) { const synced = await syncVmTimezoneFromProxy(cfg.paths.project, proxyPool, id, { force: true }) timezoneSync = { applied: synced.applied, timezone: synced.timezone, source: 'proxy_geo' } if (synced.applied) { - const kernel = projectSlotKernelConfig({ id }) + const kernel = await projectSlotKernelConfig({ id }) timezoneSync.kernel_hot = kernel?.changed === true } } @@ -4070,7 +4072,7 @@ export function createPanelHandler(ctx) { } const wrap = materializeWrapCli(cfg.paths.project, vm) if (!wrap.ok) return json(res, 400, { ok: false, error: { code: wrap.code, message: wrap.error } }) - writeKernelConfig(cfg.paths.project, vm, { routing: ctx.routingConfig }) + await writeKernelConfig(cfg.paths.project, vm, { routing: ctx.routingConfig })?.publication let kernel = { ok: false, skipped: true, reason: 'engine_not_rust' } if (resolveInferenceEngine(vm, ctx.routingConfig) === 'rust') { diff --git a/src/lib/admin/routing-runtime.mjs b/src/lib/admin/routing-runtime.mjs index ac0a0b9d5..e2be3c479 100644 --- a/src/lib/admin/routing-runtime.mjs +++ b/src/lib/admin/routing-runtime.mjs @@ -521,8 +521,10 @@ export function createRoutingRuntime(ctx) { return Object.prototype.hasOwnProperty.call(inference, 'session_idle_ttl_seconds') } - function syncKernelPanelConfig(routingConfig) { - return syncClaudeKernelConfigs(ctx.cfg.paths.project, routingConfig) + async function syncKernelPanelConfig(routingConfig) { + const result = syncClaudeKernelConfigs(ctx.cfg.paths.project, routingConfig) + await result.publication + return result } // kernel.json hot-reloads, but the job watchdog reads its idle limit from env at start. @@ -539,8 +541,8 @@ export function createRoutingRuntime(ctx) { } // The sidecar re-reads worker.json on mtime change, so a rewrite applies without a restart. - // Node slots read a pushed copy; that push is queued and only logged on failure. - function syncTelemetryUploadInterval(routingConfig) { + // Node slots read a pushed copy; the original push must finish before applying returns. + async function syncTelemetryUploadInterval(routingConfig) { const project = ctx.cfg.paths.project const applied = { updated: 0, queued: 0, skipped: 0, failed: [] } for (const { id } of listVms(project)) { @@ -551,10 +553,14 @@ export function createRoutingRuntime(ctx) { applied.skipped += 1 continue } - if (slotHost(vm).queueSyncRun(vm, path.join(project, 'vms', vm.id))) applied.queued += 1 - else applied.updated += 1 - } catch { + const publication = slotHost(vm).queueSyncRun(vm, path.join(project, 'vms', vm.id)) + if (publication) { + await publication + applied.queued += 1 + } else applied.updated += 1 + } catch (error) { applied.failed.push(vm.id) + throw error } } return applied @@ -661,12 +667,14 @@ export function createRoutingRuntime(ctx) { inferenceTouchesKernel(body.inference) || idleChanged || requestBytesChanged - ? syncKernelPanelConfig(routingConfig) + ? await syncKernelPanelConfig(routingConfig) : null // Stream idle / body-limit still need an idle restart; CCH and sticky-slot TTL hot-read kernel.json. const kernelReload = idleChanged || requestBytesChanged ? reloadClaudeKernelsWhenIdle(routingConfig) : 0 const telemetrySync = - telemetryUploadInterval(routingConfig) !== prevUploadInterval ? syncTelemetryUploadInterval(routingConfig) : null + telemetryUploadInterval(routingConfig) !== prevUploadInterval + ? await syncTelemetryUploadInterval(routingConfig) + : null // Pool and failover settings hot-reload into the live scheduler and runner. // Rebuilding them would drop the seat book and queues while their requests // still run (the panel sends `failover` with every routing save). diff --git a/src/lib/cluster/native-slot.mjs b/src/lib/cluster/native-slot.mjs index 3c28be5dd..5a50274c5 100644 --- a/src/lib/cluster/native-slot.mjs +++ b/src/lib/cluster/native-slot.mjs @@ -49,7 +49,12 @@ export const LAUNCHD_GUI_REQUIRED = function failure(err, fallbackCode = 'native_slot_failed') { const code = err?.code && typeof err.code === 'string' ? err.code : fallbackCode - return { ok: false, code, error: String(err?.message || err).slice(0, 500) } + return { + ok: false, + code, + error: String(err?.message || err).slice(0, 500), + ...(err?.vmOperationUncertain ? { vmOperationUncertain: true } : {}), + } } export function nativeLabels(vmId) { @@ -400,7 +405,8 @@ async function configureSlot(vm, projectRoot, routing) { const slotDir = path.join(projectRoot, 'vms', vm.id) await prepareSlotTree(session, p) const relays = await clusterManager().ensureSlotRelays(nodeId, vm.id) - writeWorkerFiles(vm, projectRoot, { transparent: true, routing }) + const worker = writeWorkerFiles(vm, projectRoot, { transparent: true, routing }) + await worker?.publication ensureGuestMachineIdFile(projectRoot, vm) await pushSlotFiles(vm, slotDir, ['run', 'seed'], session) await reconcileSlotCredentials(vm, slotDir, session) diff --git a/src/lib/cluster/remote-slot-files.mjs b/src/lib/cluster/remote-slot-files.mjs index 03341716d..107374a2f 100644 --- a/src/lib/cluster/remote-slot-files.mjs +++ b/src/lib/cluster/remote-slot-files.mjs @@ -219,21 +219,28 @@ export async function ensureRemoteOfficialCredentialLink(vm, slotDir, session = const queues = new Map() -/** Fire-and-forget push, serialized per VM so a slow push never races the next one. */ +/** Serialized per VM. The returned original Promise is required for completion proof. */ export function queueSlotPush(vm, slotDir, sets, { logger = console } = {}) { const prev = queues.get(vm.id) || Promise.resolve() const next = prev .catch(() => {}) .then(() => pushSlotFiles(vm, slotDir, sets)) - .catch((err) => { - logger.warn( - JSON.stringify({ event: 'remote_slot_push_failed', vm_id: vm.id, sets, error: String(err?.message || err) }), - ) + .catch((error) => { + error.vmOperationUncertain = true + error.executionUncertain = true + throw error }) .finally(() => { if (queues.get(vm.id) === next) queues.delete(vm.id) }) queues.set(vm.id, next) + // Logging observes the rejection; it does not replace the caller's Promise + // with a fulfilled one or turn a transport failure into completion proof. + next.catch((error) => { + logger.warn( + JSON.stringify({ event: 'remote_slot_push_failed', vm_id: vm.id, sets, error: String(error?.message || error) }), + ) + }) return next } diff --git a/src/lib/cluster/remote-slot.mjs b/src/lib/cluster/remote-slot.mjs index 013402589..e0d23d35d 100644 --- a/src/lib/cluster/remote-slot.mjs +++ b/src/lib/cluster/remote-slot.mjs @@ -87,7 +87,12 @@ const LIVE_BIN_GUEST = '/opt/kin-guest/opt/kin-live' const LIVE_SERVICE_MOUNT_DESTINATIONS = Object.freeze([LIVE_BIN, LIVE_BIN_GUEST]) function failure(err, fallbackCode = 'remote_slot_failed') { - return { ok: false, code: err?.code || fallbackCode, error: String(err?.message || err) } + return { + ok: false, + code: err?.code || fallbackCode, + error: String(err?.message || err), + ...(err?.vmOperationUncertain ? { vmOperationUncertain: true } : {}), + } } const MIGRATION_CONFIG_KEYS = Object.freeze([ @@ -1065,7 +1070,8 @@ export async function startRemoteSlot(vm, projectRoot, { recreate = false, routi await removeContainer(session.docker, name) existing = null } - writeWorkerFiles(vm, projectRoot, { transparent: true, routing }) + const worker = writeWorkerFiles(vm, projectRoot, { transparent: true, routing }) + await worker?.publication ensureGuestMachineIdFile(projectRoot, vm) await pushSlotFiles(vm, slotDir, ['run', 'seed'], session) if (!existing) await migrateClaudeDir(session, vm, remoteDir) @@ -1123,7 +1129,8 @@ export async function reloadRemoteSlot(vm, projectRoot, { routing } = {}) { return startRemoteSlot(vm, projectRoot, { recreate: !!existing, routing }) } const relays = await clusterManager().ensureSlotRelays(nodeId, vm.id) - writeWorkerFiles(vm, projectRoot, { transparent: true, routing }) + const worker = writeWorkerFiles(vm, projectRoot, { transparent: true, routing }) + await worker?.publication await pushSlotFiles(vm, slotDir, ['run', 'seed'], session) await reconcileSlotCredentials(vm, slotDir, session) const running = !!existing.State?.Running diff --git a/src/lib/oauth/official-cc-bootstrap.mjs b/src/lib/oauth/official-cc-bootstrap.mjs index d294b38ca..a7a8e1d2d 100644 --- a/src/lib/oauth/official-cc-bootstrap.mjs +++ b/src/lib/oauth/official-cc-bootstrap.mjs @@ -989,7 +989,9 @@ export async function finalizeOfficialCcTelemetry( let touched = false if (tel.enabled) { try { - touched = !!touchTelemetrySession(projectRoot, vmId).wrote + const touch = touchTelemetrySession(projectRoot, vmId) + await touch.publication + touched = !!touch.wrote } catch { uncertain = true } diff --git a/src/lib/transport/rust-kernel-supervisor.mjs b/src/lib/transport/rust-kernel-supervisor.mjs index 873f3f918..54a8c6eff 100644 --- a/src/lib/transport/rust-kernel-supervisor.mjs +++ b/src/lib/transport/rust-kernel-supervisor.mjs @@ -779,7 +779,7 @@ export function writeKernelConfig( } const changed = writeKernelJsonAtomically(configPath, config, vm) // kin-kernel hot-reloads kernel.json: the change must reach wherever the slot reads it. - if (changed) host.queueSyncRun(record, path.join(projectRoot, 'vms', vm.id)) + const publication = changed ? host.queueSyncRun(record, path.join(projectRoot, 'vms', vm.id)) : null return { runDir, socketPath, @@ -788,6 +788,7 @@ export function writeKernelConfig( tokenPath, provider: config.provider, changed, + publication, persona_preset: config.persona_preset, } } @@ -799,6 +800,7 @@ export function syncClaudeKernelConfigs(projectRoot, routing) { if (!projectRoot) return { updated: 0, skipped: 0 } let updated = 0 let skipped = 0 + const publications = [] for (const { id } of listVms(projectRoot)) { const vm = getVm(projectRoot, id) if (!vm || isCodexVm(vm)) { @@ -807,6 +809,7 @@ export function syncClaudeKernelConfigs(projectRoot, routing) { } try { const written = writeKernelConfig(projectRoot, vm, { routing }) + if (written?.publication) publications.push(written.publication) if (written?.changed) updated += 1 } catch (error) { skipped += 1 @@ -819,7 +822,12 @@ export function syncClaudeKernelConfigs(projectRoot, routing) { ) } } - return { updated, skipped } + const publication = Promise.allSettled(publications).then((results) => { + const failed = results.find((result) => result.status === 'rejected') + if (failed) throw failed.reason + }) + publication.catch(() => {}) + return Object.defineProperty({ updated, skipped }, 'publication', { value: publication }) } /** @@ -837,7 +845,9 @@ export function syncClaudeKernelConfigsFromFile(projectRoot, routingFile) { } catch { return { updated: 0, skipped: true } } - if (routingPersonaMtime.get(routingFile) === mtimeMs) return { updated: 0, skipped: true } + const previous = routingPersonaMtime.get(routingFile) + if (previous?.mtimeMs === mtimeMs) + return Object.defineProperty({ updated: 0, skipped: true }, 'publication', { value: previous.publication }) let routing try { routing = readRoutingDocument(routingFile) @@ -845,6 +855,10 @@ export function syncClaudeKernelConfigsFromFile(projectRoot, routingFile) { return { updated: 0, skipped: true } } const result = syncClaudeKernelConfigs(projectRoot, routing) - routingPersonaMtime.set(routingFile, mtimeMs) + const current = { mtimeMs, publication: result.publication } + routingPersonaMtime.set(routingFile, current) + result.publication.catch(() => { + if (routingPersonaMtime.get(routingFile) === current) routingPersonaMtime.delete(routingFile) + }) return result } diff --git a/src/lib/vm/slot-host.mjs b/src/lib/vm/slot-host.mjs index fdb9d9f27..a27b631e9 100644 --- a/src/lib/vm/slot-host.mjs +++ b/src/lib/vm/slot-host.mjs @@ -20,7 +20,7 @@ * host's default route (or the deployment proxy of a Codex slot); a node * slot gets a loopback SOCKS5 that dials from the node itself * syncRun(vm, slotDir) publish run/ (kernel.json, worker.json, token) where the slot reads it - * queueSyncRun(vm, slotDir) fire-and-forget syncRun + * queueSyncRun(vm, slotDir) serialized syncRun; callers must await the returned Promise * onImport(vm, slotDir) an imported credential replaces the slot's copy * afterRefresh(vm, slotDir) the slot rotated its credential; mirror it back * touch(vm, slotDir) telemetry activity mark @@ -64,7 +64,6 @@ import { ClusterError } from '../cluster/ssh-link.mjs' import { pullSlotCredentials, pushSlotCredentials, - pushSlotFiles, pushTelemetryTouch, queueSlotPush, } from '../cluster/remote-slot-files.mjs' @@ -156,7 +155,7 @@ function nodeHost(nodeId) { destroy: (vm) => destroyRemoteSlot(vm), setProxyEgressEnabled: (vm, projectRoot, enabled) => setRemoteProxyEgressEnabled(vm, projectRoot, enabled), localExitProxyUrl: () => nodeEgressProxyUrl(nodeId), - syncRun: (vm, slotDir) => pushSlotFiles(vm, slotDir, ['run']), + syncRun: (vm, slotDir) => queueSlotPush(vm, slotDir, ['run']), queueSyncRun: (vm, slotDir) => queueSlotPush(vm, slotDir, ['run']), onImport: (vm, slotDir) => pushSlotCredentials(vm, slotDir), afterRefresh: (vm, slotDir) => pullSlotCredentials(vm, slotDir), diff --git a/src/lib/vm/vm-runtime.mjs b/src/lib/vm/vm-runtime.mjs index 551890bcc..694b1cb51 100644 --- a/src/lib/vm/vm-runtime.mjs +++ b/src/lib/vm/vm-runtime.mjs @@ -426,6 +426,7 @@ export async function reloadSlotWorker(vm, projectRoot, { routing } = {}) { } worker = paths } + await worker?.publication const existing = inspectContainer(name) if (!existing) return await startVmRuntime(vm, projectRoot, { recreate: false, routing }) // Before the KVM branch so a runner on the wrong kin-eg-* is recreated too. @@ -571,6 +572,7 @@ export function writeWorkerFiles(vm, projectRoot, { transparent, routing } = {}) replaceSlotOwnedFile(paths.config, JSON.stringify(workerConfig, null, 2) + '\n', vm) const allowed = assertCliHopAllowed(vm, resolvedRouting) if (!allowed.ok) throw new Error(allowed.error) + ensureSlotClaudeOwnership(path.join(projectRoot, 'vms', vm.id, 'cli-home'), uid, gid) const kernel = writeKernelConfig(projectRoot, vm, { token, proxyUrl, @@ -583,8 +585,7 @@ export function writeWorkerFiles(vm, projectRoot, { transparent, routing } = {}) chownSlotRuntimeFile(paths.token, vm) chownSlotRuntimeFile(paths.config, vm) if (kernel?.configPath) chownSlotRuntimeFile(kernel.configPath, vm) - ensureSlotClaudeOwnership(path.join(projectRoot, 'vms', vm.id, 'cli-home'), uid, gid) - return { ...paths, kernelSocket: kernel?.socketPath || paths.kernelSocket } + return { ...paths, kernelSocket: kernel?.socketPath || paths.kernelSocket, publication: kernel?.publication } } /** Running slots survive Node deploy. Only an explicit recreate or a crash-looping container may docker rm -f. */ @@ -981,6 +982,7 @@ export async function startVmRuntime(vm, projectRoot, { recreate = false, routin } catch (error) { return { ok: false, error: String(error.message || error) } } + await worker?.publication try { fs.chownSync(home, runtimeUidNum(vm), Number(GID)) } catch {} diff --git a/src/lib/vm/worker-telemetry.mjs b/src/lib/vm/worker-telemetry.mjs index 1e9c89264..ff678e6bc 100644 --- a/src/lib/vm/worker-telemetry.mjs +++ b/src/lib/vm/worker-telemetry.mjs @@ -74,8 +74,8 @@ export function touchTelemetrySession(projectRoot, vmId, vm = null) { } catch { return { wrote: false } } - slotHost(vm).touch(vm, path.join(projectRoot, 'vms', vmId)) - return { wrote: true } + const publication = slotHost(vm).touch(vm, path.join(projectRoot, 'vms', vmId)) + return { wrote: true, publication } } function officialInstallTime(homeDir) { if (!homeDir) return '' diff --git a/test/unit/remote-slot-publication.test.mjs b/test/unit/remote-slot-publication.test.mjs index d186b4d0b..1936e11cf 100644 --- a/test/unit/remote-slot-publication.test.mjs +++ b/test/unit/remote-slot-publication.test.mjs @@ -6,7 +6,7 @@ import path from 'node:path' import { EventEmitter } from 'node:events' import { bindPlacement, unbindPlacement } from '../../src/lib/cluster/placement.mjs' import { queueSlotPush } from '../../src/lib/cluster/remote-slot-files.mjs' -import { writeKernelConfig } from '../../src/lib/transport/rust-kernel-supervisor.mjs' +import { writeKernelConfig, syncClaudeKernelConfigs } from '../../src/lib/transport/rust-kernel-supervisor.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { getDb } from '../../src/lib/db/database.mjs' import { createRepositoryTestDatabase } from '../repository-database.mjs' @@ -86,6 +86,7 @@ async function fixture(t) { peer, started, finish, + finishFirst: (error) => waiting.shift()(error), writes, operations: new SharedVmOperations(db), other: new SharedVmOperations(peer), @@ -174,3 +175,35 @@ test('panel timezone publication completes before HTTP success and VM claim rele } assert.equal(response.status, 200) }) + +test('a fleet config publication waits for every original push before reporting one failure', async (t) => { + const f = await fixture(t) + const second = { ...f.vm, id: 'vm-second' } + fs.mkdirSync(path.join(f.root, 'vms', second.id, 'run'), { recursive: true }) + fs.writeFileSync(path.join(f.root, 'vms', second.id + '.json'), JSON.stringify(second)) + const result = syncClaudeKernelConfigs(f.root, {}) + let ended = false + const publication = result.publication.then( + () => { + ended = true + }, + (error) => { + ended = true + throw error + }, + ) + publication.catch(() => {}) + try { + await f.started + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(f.writes.length, 2) + f.finishFirst(new Error('synthetic first publication disconnect')) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(ended, false) + } finally { + f.finish() + } + await assert.rejects(publication, { code: 'sftp_failed' }) + assert.equal(ended, true) + assert.equal(JSON.stringify(result).includes('publication'), false) +}) diff --git a/test/unit/routing-runtime-load.test.mjs b/test/unit/routing-runtime-load.test.mjs index 414c5f9c1..75300847f 100644 --- a/test/unit/routing-runtime-load.test.mjs +++ b/test/unit/routing-runtime-load.test.mjs @@ -447,7 +447,7 @@ test('persistRoutingPatch merges a valid vm patch', async () => { } }) -test('persistRoutingPatch rewrites Claude worker.json when the upload interval changes', async (t) => { +test('persistRoutingPatch reports a failed remote publication after rewriting the upload interval', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-routing-telemetry-')) const vms = path.join(root, 'vms') const routingFile = path.join(root, 'routing.json') @@ -471,14 +471,16 @@ test('persistRoutingPatch rewrites Claude worker.json when the upload interval c t.mock.method(console, 'warn', (line) => warned.push(String(line))) try { const runtime = persistHarness(root, routingFile, routingConfig) - const changed = await runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }) - assert.deepEqual(changed.telemetry_sync, { updated: 1, queued: 1, skipped: 0, failed: [] }) + await assert.rejects(runtime.persistRoutingPatch({ telemetry: { upload_interval_seconds: 3600 } }), { + code: 'cluster_unavailable', + vmOperationUncertain: true, + executionUncertain: true, + }) assert.equal(JSON.parse(fs.readFileSync(routingFile, 'utf8')).telemetry.upload_interval_seconds, 3600) assert.equal(worker('vm-claude').upload_interval_seconds, 3600) assert.equal(worker('vm-node').upload_interval_seconds, 3600) assert.equal(worker('vm-codex'), undefined) - // No cluster in the test: the queued push for the node slot runs, fails, and is only logged. - for (let i = 0; i < 50 && !warned.length; i++) await new Promise((r) => setTimeout(r, 10)) + // The synthetic node has no cluster; logging does not consume the caller's rejection. const push = JSON.parse(warned.find((line) => line.includes('remote_slot_push_failed'))) assert.equal(push.vm_id, 'vm-node') assert.deepEqual(push.sets, ['run']) From 12088dde288b36887dfe290ef3e4797693afa76e Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:46:20 +0800 Subject: [PATCH 325/473] fix: coordinate manual credential refresh before VM mutations --- src/lib/admin/panel-routes.mjs | 152 +++++++++++------- .../panel-credential-refresh-claims.test.mjs | 103 +++++++++++- 2 files changed, 199 insertions(+), 56 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 4097b4066..88190ab44 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -85,6 +85,7 @@ import { resetGptModelPolicy, } from '../protocol/gpt-model-policy.mjs' import { refreshCodexAccessToken } from '../protocol/codex-models.mjs' +import { refreshSharedCodexCredential } from '../oauth/shared-codex-refresh.mjs' import { defaultSeedPolicy, isTelemetryEnabled, @@ -5257,62 +5258,99 @@ export function createPanelHandler(ctx) { if (req.method === 'POST' && /^\/api\/panel\/vms\/[^/]+\/oauth\/refresh$/.test(p)) { const id = p.split('/')[4] - const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) - if (!fs.existsSync(vmPath)) { - return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'VM not found' } }) - } - const vm = getVm(cfg.paths.project, id) - if (isCodexVm(vm)) { - const first = readCodexAccounts(cfg.paths.project, id)[0] || {} - const refreshToken = String(first.refresh_token || '').trim() - if (!refreshToken) { + return mutateVms([id], async (handles) => { + const vmPath = path.join(cfg.paths.project, 'vms', `${id}.json`) + if (!fs.existsSync(vmPath)) { + return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'VM not found' } }) + } + const vm = getVm(cfg.paths.project, id) + if (isCodexVm(vm)) { + const first = readCodexAccounts(cfg.paths.project, id)[0] || {} + const refreshToken = String(first.refresh_token || '').trim() + if (!refreshToken) { + return json(res, 400, { + ok: false, + error: { code: 'credential_mode_unsupported', message: 'GPT 槽没有 refresh_token，请重新导入 OAuth' }, + }) + } + if (isKubernetesMode()) { + if (!first.chatgpt_account_id || readCodexAccounts(cfg.paths.project, id).length !== 1) + return json(res, 400, { + ok: false, + error: { code: 'codex_credential_unavailable', message: 'GPT 槽需要完整的单账号 OAuth 凭证' }, + }) + const refreshed = await refreshSharedCodexCredential({ + projectRoot: cfg.paths.project, + vm, + refresh: ctx.vmMutationOps?.refreshCodexAccessToken || refreshCodexAccessToken, + proxyUrl: hostProxyUrlForVm(vm), + }) + if (refreshed.finished !== true || refreshed.unknown) { + await handles.get(id)?.markUncertain() + return json(res, 502, { + ok: false, + error: { code: 'credential_refresh_unknown', message: 'GPT OAuth 刷新结果未知，暂不能再次刷新' }, + }) + } + if (refreshed.value?.ok !== true) + return json(res, 502, { + ok: false, + error: { + code: refreshed.value?.error || 'refresh_failed', + message: 'GPT OAuth 刷新失败，请重新导入', + }, + }) + return json( + res, + 200, + panel.ok({ ok: true, refreshed: true, platform: 'openai', shared: refreshed.shared }), + ) + } + const tok = await refreshCodexAccessToken({ + refreshToken, + proxyUrl: hostProxyUrlForVm(vm), + }) + if (!tok.ok) { + return json(res, 502, { + ok: false, + error: { code: tok.error || 'refresh_failed', message: 'GPT OAuth 刷新失败，请重新导入' }, + }) + } + upsertCodexAccount(cfg.paths.project, id, { + access_token: tok.access_token, + refresh_token: tok.refresh_token, + id_token: tok.id_token || first.id_token, + expires_at: tok.expires_at || first.expires_at, + }) + return json(res, 200, panel.ok({ ok: true, refreshed: true, platform: 'openai' })) + } + if (isApiKeyMode(vm?.claude?.mode)) { return json(res, 400, { ok: false, - error: { code: 'credential_mode_unsupported', message: 'GPT 槽没有 refresh_token，请重新导入 OAuth' }, + error: { code: 'credential_mode_unsupported', message: 'Console API Key 不能刷新' }, }) } - const tok = await refreshCodexAccessToken({ - refreshToken, - proxyUrl: hostProxyUrlForVm(vm), - }) - if (!tok.ok) { - return json(res, 502, { + if ( + isOfficialSetupTokenMode(vm?.claude?.mode) || + (isSetupTokenMode(vm?.claude?.mode) && !vm?.claude?.has_refresh) + ) { + return json(res, 400, { ok: false, - error: { code: tok.error || 'refresh_failed', message: 'GPT OAuth 刷新失败，请重新导入' }, + error: { code: 'credential_mode_unsupported', message: '官方 Setup Token（无 refresh）不能刷新' }, }) } - upsertCodexAccount(cfg.paths.project, id, { - access_token: tok.access_token, - refresh_token: tok.refresh_token, - id_token: tok.id_token || first.id_token, - expires_at: tok.expires_at || first.expires_at, - }) - return json(res, 200, panel.ok({ ok: true, refreshed: true, platform: 'openai' })) - } - if (isApiKeyMode(vm?.claude?.mode)) { - return json(res, 400, { - ok: false, - error: { code: 'credential_mode_unsupported', message: 'Console API Key 不能刷新' }, - }) - } - if ( - isOfficialSetupTokenMode(vm?.claude?.mode) || - (isSetupTokenMode(vm?.claude?.mode) && !vm?.claude?.has_refresh) - ) { - return json(res, 400, { - ok: false, - error: { code: 'credential_mode_unsupported', message: '官方 Setup Token（无 refresh）不能刷新' }, + const data = await refreshWorkerCredentialForVm({ + vmId: id, + vmPath, + homeDir: path.join(cfg.paths.project, 'vms', id, 'cli-home'), + vm, + force: true, }) - } - const data = await refreshWorkerCredentialForVm({ - vmId: id, - vmPath, - homeDir: path.join(cfg.paths.project, 'vms', id, 'cli-home'), - vm, - force: true, + if (data.credentialRefreshUnknown || data.vmOperationUncertain || data.attemptFinished === false) + await handles.get(id)?.markUncertain() + if (!data.ok) return json(res, 502, { ok: false, error: data.error, data }) + return json(res, 200, panel.ok(data)) }) - if (!data.ok) return json(res, 502, { ok: false, error: data.error, data }) - return json(res, 200, panel.ok(data)) } if (req.method === 'GET' && p === '/api/panel/oauth') { return json(res, 200, panel.ok(await oauthStatusWithWorker())) @@ -6391,14 +6429,20 @@ export function createPanelHandler(ctx) { if (!id || !fs.existsSync(vmPath)) { return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'VM not found' } }) } - const data = await refreshWorkerCredentialForVm({ - vmId: id, - vmPath, - homeDir: path.join(cfg.paths.project, 'vms', id, 'cli-home'), - vm: getVm(cfg.paths.project, id), - force: true, + return mutateVms([id], async (handles) => { + const vm = getVm(cfg.paths.project, id) + if (!vm) return json(res, 404, { ok: false, error: { code: 'vm_not_found', message: 'VM not found' } }) + const data = await refreshWorkerCredentialForVm({ + vmId: id, + vmPath, + homeDir: path.join(cfg.paths.project, 'vms', id, 'cli-home'), + vm, + force: true, + }) + if (data.credentialRefreshUnknown || data.vmOperationUncertain || data.attemptFinished === false) + await handles.get(id)?.markUncertain() + return json(res, data.ok ? 200 : 502, data) }) - return json(res, data.ok ? 200 : 502, data) } // Claude CLI inference/update was removed; workers are built and deployed separately. diff --git a/test/unit/panel-credential-refresh-claims.test.mjs b/test/unit/panel-credential-refresh-claims.test.mjs index 18274dda2..84d2fff9b 100644 --- a/test/unit/panel-credential-refresh-claims.test.mjs +++ b/test/unit/panel-credential-refresh-claims.test.mjs @@ -6,10 +6,11 @@ import path from 'node:path' import https from 'node:https' import { getDb } from '../../src/lib/db/database.mjs' import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' -import { writeCodexAccounts } from '../../src/lib/vm/codex-slot.mjs' +import { writeCodexAccounts, readCodexAccounts, codexCredentialPath } from '../../src/lib/vm/codex-slot.mjs' async function fixture(t, { codex = false } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-refresh-claims-')) @@ -72,7 +73,11 @@ async function fixture(t, { codex = false } = {}) { }, ...extra, }) - return { response, request: (route) => handle({ method: 'POST' }, {}, new URL('http://localhost' + route)) } + return { + response, + request: (route) => + withDatabaseActivities([db], () => handle({ method: 'POST' }, {}, new URL('http://localhost' + route))), + } } return { db, peer, root, vm, handler, calls: () => calls } } @@ -112,3 +117,97 @@ test('an unknown manual Codex refresh is not resubmitted by the next HTTP reques assert.notEqual(operation.uncertain_at, null) assert.equal(operation.released_at, null) }) + +test('manual Codex refresh publishes the rotated credential before releasing its VM claim', async (t) => { + const f = await fixture(t, { codex: true }) + let calls = 0 + const h = f.handler({ + vmMutationOps: { + refreshCodexAccessToken: async ({ refreshToken }) => { + calls++ + assert.equal(refreshToken, 'synthetic-old-refresh') + const operation = await f.peer.prepare('SELECT account_id, released_at FROM vm_operations').get() + assert.equal(operation.account_id, 'codex:synthetic-account') + assert.equal(operation.released_at, null) + return { + ok: true, + attemptFinished: true, + access_token: 'synthetic-new-access', + refresh_token: 'synthetic-new-refresh', + } + }, + }, + }) + await h.request('/api/panel/vms/vm-synthetic/oauth/refresh') + assert.equal(h.response.status, 200, JSON.stringify(h.response.body)) + assert.equal(calls, 1) + assert.equal(f.calls(), 0) + assert.equal(readCodexAccounts(f.root, f.vm.id)[0].refresh_token, 'synthetic-new-refresh') + assert.notEqual((await f.peer.prepare('SELECT released_at FROM vm_operations').get()).released_at, null) + assert.notEqual((await f.peer.prepare('SELECT finished_at FROM background_task_runs').get()).finished_at, null) + assert.equal(JSON.stringify(h.response.body).includes('synthetic-new-refresh'), false) +}) + +test('a complete Codex token rejection releases the manual operation without changing its credential', async (t) => { + const f = await fixture(t, { codex: true }) + const h = f.handler({ + vmMutationOps: { + refreshCodexAccessToken: async () => ({ ok: false, attemptFinished: true, error: 'refresh_rejected' }), + }, + }) + await h.request('/api/panel/vms/vm-synthetic/oauth/refresh') + assert.equal(h.response.status, 502) + assert.equal(readCodexAccounts(f.root, f.vm.id)[0].refresh_token, 'synthetic-old-refresh') + const next = await new SharedVmOperations(f.peer).tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, true) + await next.release() +}) + +test('publication failure after token rotation keeps the manual operation and forbids another refresh', async (t) => { + const f = await fixture(t, { codex: true }) + const rename = fs.renameSync + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === codexCredentialPath(f.root, f.vm.id)) + throw Object.assign(new Error('synthetic publication failure'), { code: 'EACCES' }) + return rename(from, to) + }) + let calls = 0 + const h = f.handler({ + vmMutationOps: { + refreshCodexAccessToken: async () => { + calls++ + return { ok: true, attemptFinished: true, access_token: 'synthetic-new-access' } + }, + }, + }) + await assert.rejects(h.request('/api/panel/vms/vm-synthetic/oauth/refresh'), { code: 'EACCES' }) + const next = f.handler() + await next.request('/api/panel/vms/vm-synthetic/oauth/refresh') + assert.equal(next.response.status, 409) + assert.equal(calls, 1) + assert.equal((await f.peer.prepare('SELECT released_at FROM vm_operations').get()).released_at, null) +}) + +for (const route of ['/api/panel/vms/vm-synthetic/oauth/refresh', '/admin/vm/oauth/refresh']) { + test(`Claude manual refresh ${route} retains an explicit unknown response`, async (t) => { + const f = await fixture(t) + const h = f.handler({ + refreshWorkerCredentialForVm: async () => ({ ok: false, attemptFinished: false, credentialRefreshUnknown: true }), + }) + await h.request(route) + assert.equal(h.response.status, 502) + const operation = await f.peer.prepare('SELECT uncertain_at, released_at FROM vm_operations').get() + assert.notEqual(operation.uncertain_at, null) + assert.equal(operation.released_at, null) + }) +} + +test('Claude complete rejection can release its manual operation', async (t) => { + const f = await fixture(t) + const h = f.handler({ refreshWorkerCredentialForVm: async () => ({ ok: false, attemptFinished: true }) }) + await h.request('/api/panel/vms/vm-synthetic/oauth/refresh') + assert.equal(h.response.status, 502) + const next = await new SharedVmOperations(f.peer).tryAcquire({ vmId: f.vm.id }) + assert.equal(next.ok, true) + await next.release() +}) From f1a2e181a5a6fa6aabbd21a6c46a74b69f086e5d Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:46:23 +0800 Subject: [PATCH 326/473] test: retain uncertain publication before model submission --- .../failover-preparation-unknown.test.mjs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test/unit/failover-preparation-unknown.test.mjs diff --git a/test/unit/failover-preparation-unknown.test.mjs b/test/unit/failover-preparation-unknown.test.mjs new file mode 100644 index 000000000..9c13e41a4 --- /dev/null +++ b/test/unit/failover-preparation-unknown.test.mjs @@ -0,0 +1,44 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' + +test('an unknown remote publication during preparation stops failover and preserves the admission', async () => { + let selections = 0 + let marked = 0 + let calls = 0 + let released = 0 + let uncertain = false + const scheduler = { + async selectAndReserve() { + selections++ + return { + ok: true, + accountId: 'publication-account', + vmId: 'publication-vm', + bindExecution: async () => {}, + markUncertain: async () => { marked++; uncertain = true }, + release: async () => { if (!uncertain) released++ }, + } + }, + markCooldown() {}, + markSuccess() {}, + } + const runner = new FailoverRunner({ scheduler }) + const result = await runner.run({ + requestId: 'publication-unknown', + model: 'claude-test', + canonicalBody: { model: 'claude-test' }, + applyAttempt() { + throw Object.assign(new Error('synthetic SFTP connection lost after write'), { + code: 'sftp_failed', + executionUncertain: true, + }) + }, + callAttempt: async () => { calls++; return { ok: false } }, + }) + assert.equal(selections, 1, 'unknown preparation must not select another VM') + assert.equal(marked, 1) + assert.equal(calls, 0) + assert.equal(released, 0) + assert.equal(result.executionUncertain, true) +}) From 31717e6fda7a0ea1568e4dc1a7b21eb0366502b0 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:46:37 +0800 Subject: [PATCH 327/473] test: retain parent VM operations after nested uncertain work --- test/unit/shared-vm-operations.test.mjs | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/unit/shared-vm-operations.test.mjs b/test/unit/shared-vm-operations.test.mjs index 8e474882b..868ce46d4 100644 --- a/test/unit/shared-vm-operations.test.mjs +++ b/test/unit/shared-vm-operations.test.mjs @@ -280,6 +280,63 @@ test('a failed release remains persistent and retries PostgreSQL rather than cac await next.release() }) +test('nested unknown outcomes retain their parent operation even when the outer result omits the flag', async (t) => { + const other = await otherDatabase(t) + const outer = await operations(getDb(), 'owner-a') + const nested = await operations(other, 'owner-b') + await outer.run('vm-a', async () => { + const result = await nested.run('vm-a', async () => ({ vmOperationUncertain: true, value: 'synthetic unknown' })) + assert.equal(result.value, 'synthetic unknown') + return { ok: true } + }) + const row = await other.prepare('SELECT released_at,uncertain_at FROM vm_operations').get() + assert.equal(row.released_at, null) + assert.ok(row.uncertain_at) + assert.equal((await nested.tryAcquire({ vmId: 'vm-a' })).code, 'vm_operation_active') +}) + +test('a nested unknown exception remains persistent when the outer callback catches it', async (t) => { + const other = await otherDatabase(t) + const outer = await operations(getDb(), 'owner-a') + const nested = await operations(other, 'owner-b') + const failure = new Error('synthetic unknown after remote submission') + await outer.run('vm-a', async () => { + await assert.rejects( + nested.run('vm-a', async () => { + throw failure + }), + (error) => error === failure, + ) + return { ok: true } + }) + const row = await other.prepare('SELECT released_at,uncertain_at FROM vm_operations').get() + assert.equal(row.released_at, null) + assert.ok(row.uncertain_at) + assert.equal((await nested.tryAcquire({ vmId: 'vm-a' })).code, 'vm_operation_active') +}) + +test('a nested explicitly unstarted exception does not retain an otherwise finished parent operation', async (t) => { + const other = await otherDatabase(t) + const outer = await operations(getDb(), 'owner-a') + const nested = await operations(other, 'owner-b') + const failure = Object.assign(new Error('synthetic validation rejection'), { vmOperationNotStarted: true }) + await outer.run('vm-a', async () => { + await assert.rejects( + nested.run('vm-a', async () => { + throw failure + }), + (error) => error === failure, + ) + return { ok: true } + }) + const row = await other.prepare('SELECT released_at,uncertain_at FROM vm_operations').get() + assert.ok(row.released_at) + assert.equal(row.uncertain_at, null) + const next = await nested.tryAcquire({ vmId: 'vm-a' }) + assert.equal(next.ok, true) + await next.release() +}) + test('different schemas cannot reuse one another nested operation context', async (t) => { const db = getDb() const other = await createDatabase({ connectionString: db.connectionString, schema: db.schema + '_other' }) From 0d5231fcea46d3e40eccfc50f3951e0de42c2d66 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:46:55 +0800 Subject: [PATCH 328/473] test(vm): require immutable operation identity handles --- test/unit/vm-operation-identity.test.mjs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/unit/vm-operation-identity.test.mjs diff --git a/test/unit/vm-operation-identity.test.mjs b/test/unit/vm-operation-identity.test.mjs new file mode 100644 index 000000000..f12561fcb --- /dev/null +++ b/test/unit/vm-operation-identity.test.mjs @@ -0,0 +1,19 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' + +test('operation identity fields cannot be reassigned while closure state can become uncertain',async () => { + const operations = new SharedVmOperations(getDb(),{ownerId:'immutable-owner'}) + const operation = await operations.tryAcquire({vmId:'immutable-vm',kind:'node_allocate'}) + try { + assert.equal(operation.ok,true) + for (const field of ['operationId','ownerId','vmId','kind']) { + assert.throws(() => {operation[field] = 'different-identity'},TypeError) + } + assert.equal(operation.uncertain,false) + assert.equal((await operation.markUncertain()).uncertain,true) + assert.equal(operation.uncertain,true) + assert.equal((await operation.release()).released,false) + } finally {await operation.release()} +}) From ea18a21379d43b517e4933d5c13190205f5dea6a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:47:50 +0800 Subject: [PATCH 329/473] test: bound the unknown preparation retry fixture --- test/unit/failover-preparation-unknown.test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/failover-preparation-unknown.test.mjs b/test/unit/failover-preparation-unknown.test.mjs index 9c13e41a4..3f1f314f0 100644 --- a/test/unit/failover-preparation-unknown.test.mjs +++ b/test/unit/failover-preparation-unknown.test.mjs @@ -11,6 +11,7 @@ test('an unknown remote publication during preparation stops failover and preser const scheduler = { async selectAndReserve() { selections++ + if (selections > 1) return { ok: false, reason: 'no_eligible_accounts' } return { ok: true, accountId: 'publication-account', From 3485404806f916361d55ce2d70714aafdf6f1c6a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:50:02 +0800 Subject: [PATCH 330/473] feat(slotgate): authorize initial allocation with durable private PG grants --- .../2026-10-21-node-allocation-authority.md | 33 ++ .../2026-10-21-node-fencing-launch-adapter.md | 2 +- src/lib/cluster/node-management-grants.mjs | 157 ++++++++ src/lib/db/node-management-authority.mjs | 167 ++++++++ .../transport/slotgate-management-client.mjs | 110 ++++++ src/lib/vm/shared-vm-operations.mjs | 50 ++- test/fixtures/real-allocation-gate.mjs | 58 +++ test/fixtures/slotgate-allocation-node.go | 41 ++ test/unit/node-management-grants.test.mjs | 366 ++++++++++++++++-- test/unit/vm-operation-identity.test.mjs | 26 +- worker/go.mod | 3 + worker/go.sum | 12 + worker/internal/slotgate/allocate.go | 105 +++++ worker/internal/slotgate/allocate_test.go | 39 +- worker/internal/slotgate/authority.go | 23 +- .../internal/slotgate/docker_launch_test.go | 1 + worker/internal/slotgate/ksg2.go | 51 ++- worker/internal/slotgate/ksg2_test.go | 3 + worker/internal/slotgate/pg_authorizer.go | 160 ++++++++ worker/internal/slotgate/slotgate.go | 20 + 20 files changed, 1336 insertions(+), 91 deletions(-) create mode 100644 docs/plans/2026-10-21-node-allocation-authority.md create mode 100644 src/lib/cluster/node-management-grants.mjs create mode 100644 src/lib/db/node-management-authority.mjs create mode 100644 src/lib/transport/slotgate-management-client.mjs create mode 100644 test/fixtures/real-allocation-gate.mjs create mode 100644 test/fixtures/slotgate-allocation-node.go create mode 100644 worker/internal/slotgate/allocate.go create mode 100644 worker/internal/slotgate/pg_authorizer.go diff --git a/docs/plans/2026-10-21-node-allocation-authority.md b/docs/plans/2026-10-21-node-allocation-authority.md new file mode 100644 index 000000000..755ea472d --- /dev/null +++ b/docs/plans/2026-10-21-node-allocation-authority.md @@ -0,0 +1,33 @@ +# 初始槽位分配与节点独立授权 + +本批只分配普通 Claude Docker 槽位的初始逻辑身份。它不创建目录、文件、容器或 provider 请求。文件 stage/publish、launch grant、停止与回收、生产 createVM 路由和节点权限迁移不在本批。现有预编译 payload 保持不变，生产没有启用 HA。 + +## 授权来源与持久状态 + +`NodeManagementGrants.issueAllocation` 必须在同一数据库与 application schema 的当前 exact `SharedVmOperations.run` 回调内调用。节点、VM、独立 allocation ID 与当前 operation 的 owner、目标和 `node_allocate` 类型绑定。仅提交 operation ID 不能发行授权。operation 必须已独立提交且没有 released/uncertain。 + +授权存储在 `vm2api_node_` 私有 schema。`meta` 固定 `protocol_version=1` 与 application schema；`allocations` 保存 node、逻辑与物理 VM 唯一身份、operation/owner、maintenance epoch、intent digest、原 node receipt 和撤销时间。allocation、VM 与物理别名保留唯一记录，不会因旧业务备份恢复重新分配。 + +GlobalMaintenance 的私有 coordination schema 另保存 `node_authority_features` 启用标记。首次初始化同时创建 marker、meta、授权表、只读 view 与函数。业务迁移数与表数不因这些私有对象变化。该功能没有自动协议升级；已有 marker 后，缺 schema、表、函数、view、绑定不符或未知协议均拒绝。只有 marker 和 authority schema 均不存在时才保留从未启用的旧操作路径。 + +只读 view 绑定当前私有 maintenance epoch 与 OPEN 阶段，并通过固定 schema 的动态 SELECT 核对当前 operation。业务表不是 view 的静态依赖；删除/恢复业务表不会删除私有 view。缺业务表时拒绝，恢复旧 operation 行也不能绕过 epoch 或持久撤销记录。 + +## 节点检查与本地 receipt + +节点 `Config.management` 只能由可信初始化流程安装，包含 node ID、private authority schema 和节点专用只读 PostgreSQL DSN。RPC 不能提供或更换这些值。节点使用固定 pgx v5.11.0；远程数据库要求验证证书的 TLS，Unix socket 与 loopback 可用于本地验证。 + +节点 role 仅获得自身授权 view、schema USAGE 和指定只读函数的权限。`reader_identity` 拒绝超级用户、可创建角色/数据库、BYPASSRLS、角色成员关系、private schema CREATE 或底表写权限。`SECURITY DEFINER` 函数固定 search_path，显式限定 schema，默认撤销 PUBLIC EXECUTE。`readerRoleSql` 只生成不含凭据的授权 SQL；运行时不会创建角色。实际节点安装 role、view 或配置需要另行授权，本任务仅在隔离测试库执行。 + +KSG2 `allocate` 只接受 VM、allocation ID、operation ID 与 owner ID。它不能提交 generation、executor、路径、模式、命令或文件内容。节点独立核对 PG 授权与 intent digest，随后在共享授权锁持有期间写本地 journal。锁只跨短时本地 journal I/O，不跨 Docker 或外部请求。发行、release 与 uncertain 使用 operation 行锁后再取授权排他锁；节点使用 MVCC 读取 operation，不反向锁行。 + +初始 generation 固定由 node 创建为 1，incarnation 由 node 随机生成。journal 同时持久原 allocation receipt。同 ID、同 intent、同 epoch 返回原 receipt；不同 intent 冲突。已有未受此流程管理的槽位不被自动认领。replay 返回原 VACANT fence，不使用最新 State 将旧请求重新授权。 + +节点首次配置 management 时把 node ID、authority schema 与协议写入本地 journal。重开时缺少或更换该绑定会拒绝启动。journal 写盘失败不返回成功，daemon 拒绝后续授权。PG 查询失败无本地授权回退；PG commit 失败也不能确认本地分配成功。 + +`NodeManagementGrants.allocate` 组合已提交 grant、固定证书的 TLS typed client 和 PG receipt 保存。只有原 node receipt 与 grant 严格相同才保存。receipt 保存失败会保留 unknown VM operation；没有自动删除、重新分配或重试。 + +## 默认拒绝与范围 + +转发 token 仅认证传输，不授予管理权限。`Config.Fenced=true` 时，未接入独立 action grant 的 launch/revoke/advance 默认拒绝。allocate 未配置 PG authorizer 也拒绝。state、prepare/forward、command 保留原 captured Fence 推理协议。单元测试中的 management fake authorizer 仅为明确注入的测试适配器，不是可用的节点配置项。 + +SSH、raw Docker、SFTP、旧会话权限、KVM、原生 macOS 和控制面 Codex 子进程仍未迁移。节点文件发布接口也未实现。任何用户节点启用前都必须完成这些权限与调用链迁移。本批临时 TLS/Unix、合成身份和两条 PG 连接的协议验证不证明可信容器启动或多节点 HA。 diff --git a/docs/plans/2026-10-21-node-fencing-launch-adapter.md b/docs/plans/2026-10-21-node-fencing-launch-adapter.md index 449f60267..5c4644225 100644 --- a/docs/plans/2026-10-21-node-fencing-launch-adapter.md +++ b/docs/plans/2026-10-21-node-fencing-launch-adapter.md @@ -16,7 +16,7 @@ KSG2 增加 `launch` 操作。客户端只提供 `vm_id`、创建操作时捕获 | `network_id` | `none`，或节点预先批准的完整 bridge network ID；不接受 host、container 或 service 网络命名空间 | | `uid`、`gid` | 非 root 的正整数；不能由 RPC 调用者覆盖 | -`launch` policy 缺失时拒绝启动。当前接口要求目标 VM 已有 `VACANT` 状态，且可信节点组件已准备好该 captured incarnation 的私有文件树。本批没有提供初始 allocate、文件上传或文件发布 RPC，也不会自动接管 legacy executor。 +`launch` policy 缺失时拒绝启动。当前接口要求目标 VM 已有 `VACANT` 状态，且可信节点组件已准备好该 captured incarnation 的私有文件树。后续初始 allocate 已接入独立 PG 授权与 node receipt，见 [初始槽位分配与节点独立授权](2026-10-21-node-allocation-authority.md)。文件上传、文件发布与 launch action grant 仍未接入，也不会自动接管 legacy executor。KSG2 静态转发 token 不再授予 launch/revoke/advance 管理权限；这些动作默认拒绝。 启动顺序为： diff --git a/src/lib/cluster/node-management-grants.mjs b/src/lib/cluster/node-management-grants.mjs new file mode 100644 index 000000000..a3217c5e5 --- /dev/null +++ b/src/lib/cluster/node-management-grants.mjs @@ -0,0 +1,157 @@ +import crypto from 'node:crypto' +import { withTransaction, quoteIdentifier } from '../db/database.mjs' +import { coordinationSchemaFor } from '../db/global-maintenance.mjs' +import { currentVmOperationIds } from '../vm/shared-vm-operations.mjs' +import { physicalContainerName } from '../vm/vm-physical-target.mjs' +import { ClusterNodesRepo } from '../db/repos/cluster-nodes-repo.mjs' +import { allocateNodeSlot, normalizeAllocationReceipt } from '../transport/slotgate-management-client.mjs' +import { isDeepStrictEqual } from 'node:util' +import { + initializeNodeAuthority, + nodeAuthorityState, + nodeAuthorityError, + nodeAuthoritySchemaFor, + lockNodeGrant, +} from '../db/node-management-authority.mjs' + +export { nodeAuthoritySchemaFor } + +function identifier(value) { + if (typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(value)) throw nodeAuthorityError('node_grant_invalid') + return value +} + +function literal(value) { + return "'" + value.replaceAll("'", "''") + "'" +} + +export class NodeManagementGrants { + constructor(db, { resolveNodeLink } = {}) { + this.db = db + this.schema = nodeAuthoritySchemaFor(db.schema) + this.resolveNodeLink = resolveNodeLink || ((nodeId) => new ClusterNodesRepo(db).getDirectLink(nodeId)) + } + initialize() { + return initializeNodeAuthority(this.db) + } + + async issueAllocation({ operation, nodeId, vmId, allocationId }) { + identifier(nodeId) + identifier(vmId) + identifier(allocationId) + if (!operation || !currentVmOperationIds(this.db).has(operation.operationId)) + throw nodeAuthorityError('node_operation_scope_required') + if (this.db.isTransaction) throw nodeAuthorityError('node_grant_transaction_active') + return withTransaction(this.db, async () => { + const row = await this.db + .prepare('SELECT * FROM vm_operations WHERE id=$1 AND owner_id=$2 FOR UPDATE') + .get(operation.operationId, operation.ownerId) + if (!row || row.vm_id !== vmId || row.kind !== 'node_allocate' || row.released_at || row.uncertain_at) + throw nodeAuthorityError('node_operation_invalid') + identifier(row.id) + identifier(row.owner_id) + const state = await nodeAuthorityState(this.db) + if (!state.enabled) throw nodeAuthorityError() + await lockNodeGrant(this.db, this.schema, nodeId, vmId) + const epoch = ( + await this.db.query( + `SELECT epoch::text,phase FROM ${quoteIdentifier(coordinationSchemaFor(this.db.schema))}.control WHERE id=1`, + ) + ).rows[0] + if (epoch?.phase !== 'open') throw nodeAuthorityError() + const digest = crypto + .createHash('sha256') + .update(['slotgate-allocate-v1', nodeId, vmId, allocationId, row.id, row.owner_id].join('\0')) + .digest('hex') + const existing = ( + await this.db.query( + `SELECT * FROM ${state.q}.allocations WHERE allocation_id=$1 OR vm_id=$2 OR physical_target=$3`, + [allocationId, vmId, physicalContainerName(vmId)], + ) + ).rows[0] + if (existing) { + if ( + existing.allocation_id !== allocationId || + existing.intent_digest !== digest || + existing.revoked_at || + String(existing.epoch) !== epoch.epoch + ) + throw nodeAuthorityError('node_allocation_conflict') + } else { + await this.db.query( + `INSERT INTO ${state.q}.allocations + (allocation_id,node_id,vm_id,physical_target,operation_id,owner_id,action,epoch,intent_digest) + VALUES ($1,$2,$3,$4,$5,$6,'allocate',$7,$8)`, + [allocationId, nodeId, vmId, physicalContainerName(vmId), row.id, row.owner_id, epoch.epoch, digest], + ) + } + return { + action: 'allocate', + nodeId, + vmId, + allocationId, + operationId: row.id, + ownerId: row.owner_id, + epoch: epoch.epoch, + intentDigest: digest, + } + }) + } + + async allocate(input) { + const link = await this.resolveNodeLink(input.nodeId) + if (!link) throw nodeAuthorityError('node_link_unavailable') + const grant = await this.issueAllocation(input) + const receipt = await allocateNodeSlot({ link, grant }) + await withTransaction(this.db, async () => { + if (!currentVmOperationIds(this.db).has(grant.operationId)) + throw nodeAuthorityError('node_operation_scope_required') + const operation = await this.db + .prepare('SELECT * FROM vm_operations WHERE id=$1 AND owner_id=$2 FOR UPDATE') + .get(grant.operationId, grant.ownerId) + if ( + !operation || + operation.vm_id !== grant.vmId || + operation.kind !== 'node_allocate' || + operation.released_at || + operation.uncertain_at + ) + throw nodeAuthorityError('node_operation_invalid') + const state = await nodeAuthorityState(this.db) + if (!state.enabled) throw nodeAuthorityError() + await lockNodeGrant(this.db, this.schema, grant.nodeId, grant.vmId) + const row = ( + await this.db.query(`SELECT * FROM ${state.q}.allocations WHERE allocation_id=$1`, [grant.allocationId]) + ).rows[0] + const normalized = normalizeAllocationReceipt(receipt, grant) + if ( + !row || + row.revoked_at || + row.intent_digest !== grant.intentDigest || + String(row.epoch) !== grant.epoch || + (row.receipt && !isDeepStrictEqual(row.receipt, normalized)) + ) + throw nodeAuthorityError('node_allocation_conflict') + await this.db.query(`UPDATE ${state.q}.allocations SET receipt=$2::jsonb WHERE allocation_id=$1`, [ + grant.allocationId, + JSON.stringify(normalized), + ]) + }) + return receipt + } + + // Explicit node-role installation instructions only. Runtime initialization + // never creates/alters roles and this SQL contains no credentials. + readerRoleSql({ role, nodeId }) { + if (typeof role !== 'string' || !/^[a-z_][a-z0-9_]{0,62}$/.test(role)) + throw nodeAuthorityError('node_reader_invalid') + identifier(nodeId) + const q = quoteIdentifier(this.schema) + const r = quoteIdentifier(role) + return `INSERT INTO ${q}.reader_roles(role_name,node_id) VALUES (${literal(role)},${literal(nodeId)}); + GRANT USAGE ON SCHEMA ${q} TO ${r}; + GRANT SELECT ON ${q}.authorized_allocations TO ${r}; + GRANT EXECUTE ON FUNCTION ${q}.reader_identity() TO ${r}; + GRANT EXECUTE ON FUNCTION ${q}.allocation_current(text) TO ${r};` + } +} diff --git a/src/lib/db/node-management-authority.mjs b/src/lib/db/node-management-authority.mjs new file mode 100644 index 000000000..82761e964 --- /dev/null +++ b/src/lib/db/node-management-authority.mjs @@ -0,0 +1,167 @@ +import crypto from 'node:crypto' +import { quoteIdentifier, withTransaction } from './database.mjs' +import { coordinationSchemaFor } from './global-maintenance.mjs' + +export function nodeAuthoritySchemaFor(applicationSchema) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(applicationSchema)) throw new Error('invalid database schema') + return 'vm2api_node_' + crypto.createHash('sha256').update(applicationSchema).digest('hex').slice(0, 32) +} + +export function nodeAuthorityError(code = 'node_authority_unavailable') { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} + +export function nodeGrantResource(nodeId, vmId) { + return `node:${nodeId}:vm:${vmId}` +} +export function nodeGrantNamespace(schema) { + return `vm2api:node-management:${schema}` +} + +export async function lockNodeGrant(db, schema, nodeId, vmId) { + await db.query('SELECT pg_advisory_xact_lock(hashtext($1),hashtext($2))', [ + nodeGrantNamespace(schema), + nodeGrantResource(nodeId, vmId), + ]) +} + +export async function nodeAuthorityState(db) { + const schema = nodeAuthoritySchemaFor(db.schema) + const coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + const q = quoteIdentifier(schema) + const names = ( + await db.query( + `SELECT to_regnamespace($1) AS authority, + COALESCE((SELECT pg_get_userbyid(nspowner)=current_user FROM pg_namespace WHERE nspname=$1),false) AS owned, + to_regclass($2) AS marker`, + [schema, `${coord}.node_authority_features`], + ) + ).rows[0] + // This combination can only represent a feature that has never been enabled. + if (!names.authority && !names.marker) return { enabled: false, schema } + if (!names.authority || !names.marker || !names.owned) throw nodeAuthorityError() + try { + const marker = ( + await db.query(`SELECT * FROM ${coord}.node_authority_features WHERE application_schema=$1`, [db.schema]) + ).rows[0] + const meta = (await db.query(`SELECT * FROM ${q}.meta WHERE id=1`)).rows[0] + if ( + marker?.authority_schema !== schema || + marker.protocol_version !== 1 || + meta?.application_schema !== db.schema || + meta.protocol_version !== 1 + ) + throw nodeAuthorityError() + const relations = ( + await db.query( + `SELECT to_regclass($1) IS NOT NULL AND to_regclass($2) IS NOT NULL + AND to_regclass($3) IS NOT NULL AND to_regprocedure($4) IS NOT NULL AND to_regprocedure($5) IS NOT NULL AS ready`, + [ + `${q}.allocations`, + `${q}.reader_roles`, + `${q}.authorized_allocations`, + `${q}.reader_identity()`, + `${q}.allocation_current(text)`, + ], + ) + ).rows[0] + if (!relations.ready) throw nodeAuthorityError() + return { enabled: true, schema, q } + } catch { + throw nodeAuthorityError() + } +} + +export async function initializeNodeAuthority(db) { + return withTransaction(db, async () => { + const schema = nodeAuthoritySchemaFor(db.schema) + const q = quoteIdentifier(schema) + const coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:node-authority-bootstrap:${db.schema}`, + ]) + const existing = await nodeAuthorityState(db) + if (existing.enabled) return existing + const maintenance = (await db.query(`SELECT application_schema,phase FROM ${coord}.control WHERE id=1`)).rows[0] + if (maintenance?.application_schema !== db.schema || maintenance.phase !== 'open') throw nodeAuthorityError() + await db.query(`CREATE SCHEMA ${q} AUTHORIZATION CURRENT_USER; + REVOKE ALL ON SCHEMA ${q} FROM PUBLIC; + CREATE TABLE ${q}.meta ( + id integer PRIMARY KEY CHECK(id=1), application_schema text NOT NULL, protocol_version integer NOT NULL CHECK(protocol_version=1)); + CREATE TABLE ${coord}.node_authority_features ( + application_schema text PRIMARY KEY, authority_schema text NOT NULL, protocol_version integer NOT NULL CHECK(protocol_version=1)); + REVOKE ALL ON ${coord}.node_authority_features FROM PUBLIC; + CREATE TABLE ${q}.reader_roles (role_name name PRIMARY KEY, node_id text NOT NULL); + CREATE TABLE ${q}.allocations ( + allocation_id text PRIMARY KEY, node_id text NOT NULL, vm_id text NOT NULL UNIQUE, + physical_target text NOT NULL UNIQUE, operation_id text NOT NULL, owner_id text NOT NULL, + action text NOT NULL CHECK(action='allocate'), epoch bigint NOT NULL CHECK(epoch>0), + intent_digest text NOT NULL, receipt jsonb, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), revoked_at timestamptz); + CREATE INDEX allocations_operation ON ${q}.allocations(operation_id,owner_id) WHERE revoked_at IS NULL; + REVOKE ALL ON ALL TABLES IN SCHEMA ${q} FROM PUBLIC`) + await db.query(`INSERT INTO ${q}.meta VALUES (1,$1,1)`, [db.schema]) + await db.query(`INSERT INTO ${coord}.node_authority_features VALUES ($1,$2,1)`, [db.schema, schema]) + // No business table is a static dependency: a schema restore must not DROP + // CASCADE these functions/view or resurrect an old allocation authorization. + await db.query(`CREATE FUNCTION ${q}.reader_identity() RETURNS TABLE(node_id text,application_schema text,protocol_version integer) + LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $body$ + DECLARE owner_name name; + BEGIN + SELECT pg_get_userbyid(nspowner) INTO owner_name FROM pg_namespace WHERE nspname='${schema}'; + IF EXISTS(SELECT 1 FROM pg_roles WHERE rolname=session_user AND (rolsuper OR rolcreaterole OR rolcreatedb OR rolbypassrls)) + OR pg_has_role(session_user,owner_name,'MEMBER') + OR EXISTS(SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member WHERE r.rolname=session_user) + OR has_schema_privilege(session_user,'${schema}','CREATE') + OR has_table_privilege(session_user,'${schema}.allocations','INSERT,UPDATE,DELETE,TRUNCATE') THEN RETURN; END IF; + RETURN QUERY SELECT r.node_id,m.application_schema,m.protocol_version FROM ${q}.reader_roles r CROSS JOIN ${q}.meta m + WHERE r.role_name=session_user AND m.id=1 AND m.protocol_version=1; + END $body$; + CREATE FUNCTION ${q}.allocation_current(p_allocation text) RETURNS boolean + LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $body$ + DECLARE g record; app text; allowed boolean; + BEGIN + SELECT a.*,m.application_schema INTO g FROM ${q}.allocations a CROSS JOIN ${q}.meta m + JOIN ${q}.reader_roles r ON r.role_name=session_user + JOIN ${coord}.control c ON c.id=1 + JOIN ${coord}.node_authority_features f ON f.application_schema=m.application_schema + WHERE a.allocation_id=p_allocation AND a.node_id=r.node_id AND a.revoked_at IS NULL AND a.action='allocate' + AND m.id=1 AND m.protocol_version=1 AND f.protocol_version=1 AND f.authority_schema='${schema}' + AND c.application_schema=m.application_schema AND c.phase='open' AND c.epoch=a.epoch; + IF NOT FOUND OR NOT EXISTS(SELECT 1 FROM ${q}.reader_identity()) THEN RETURN false; END IF; + app:=g.application_schema; + EXECUTE format('SELECT EXISTS(SELECT 1 FROM %I.vm_operations WHERE id=$1 AND owner_id=$2 AND vm_id=$3 + AND kind=''node_allocate'' AND released_at IS NULL AND uncertain_at IS NULL)',app) + INTO allowed USING g.operation_id,g.owner_id,g.vm_id; + RETURN allowed; + EXCEPTION WHEN undefined_table OR invalid_schema_name THEN RETURN false; + END $body$; + REVOKE ALL ON FUNCTION ${q}.reader_identity() FROM PUBLIC; + REVOKE ALL ON FUNCTION ${q}.allocation_current(text) FROM PUBLIC; + CREATE VIEW ${q}.authorized_allocations WITH(security_barrier=true) AS + SELECT allocation_id,node_id,vm_id,operation_id,owner_id,action,epoch,intent_digest + FROM ${q}.allocations WHERE ${q}.allocation_current(allocation_id); + REVOKE ALL ON ${q}.authorized_allocations FROM PUBLIC`) + return nodeAuthorityState(db) + }) +} + +// Caller must already hold the exact VM operation row. This lock order also +// governs issuance; the node's shared-lock path uses MVCC reads only. +export async function revokeNodeOperationGrants(db, { operationId, ownerId }) { + if (!db.isTransaction) throw nodeAuthorityError() + const state = await nodeAuthorityState(db) + if (!state.enabled) return + const rows = ( + await db.query( + `SELECT DISTINCT node_id,vm_id FROM ${state.q}.allocations + WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL ORDER BY node_id,vm_id`, + [operationId, ownerId], + ) + ).rows + for (const row of rows) await lockNodeGrant(db, state.schema, row.node_id, row.vm_id) + await db.query( + `UPDATE ${state.q}.allocations SET revoked_at=clock_timestamp() + WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL`, + [operationId, ownerId], + ) +} diff --git a/src/lib/transport/slotgate-management-client.mjs b/src/lib/transport/slotgate-management-client.mjs new file mode 100644 index 000000000..4be95131f --- /dev/null +++ b/src/lib/transport/slotgate-management-client.mjs @@ -0,0 +1,110 @@ +import tls from 'node:tls' +import net from 'node:net' +import { isDeepStrictEqual } from 'node:util' + +function denied() { + return Object.assign(new Error('node allocation acknowledgement unavailable'), { + code: 'node_allocation_unconfirmed', + }) +} +function exact(value, keys) { + if ( + !value || + typeof value !== 'object' || + Object.keys(value).length !== keys.length || + keys.some((key) => !Object.hasOwn(value, key)) + ) + throw denied() +} + +export function normalizeAllocationReceipt(receipt, grant) { + exact(receipt, ['node_id', 'vm_id', 'allocation_id', 'operation_id', 'owner_id', 'epoch', 'intent_digest', 'fence']) + if ( + receipt.node_id !== grant.nodeId || + receipt.vm_id !== grant.vmId || + receipt.allocation_id !== grant.allocationId || + receipt.operation_id !== grant.operationId || + receipt.owner_id !== grant.ownerId || + receipt.epoch !== grant.epoch || + receipt.intent_digest !== grant.intentDigest + ) + throw denied() + const fence = receipt.fence + exact(fence, ['generation', 'incarnation', 'executor']) + if ( + String(fence.generation) !== '1' || + !/^[a-f0-9]{32}$/.test(fence.incarnation) || + !isDeepStrictEqual(fence.executor, { kind: '', node_boot_id: '' }) + ) + throw denied() + return { + ...receipt, + fence: { generation: '1', incarnation: fence.incarnation, executor: { kind: '', node_boot_id: '' } }, + } +} + +/** One typed management action; no retry, shell, launch or arbitrary path RPC. */ +export function allocateNodeSlot({ link, grant }) { + const values = [grant?.nodeId, grant?.vmId, grant?.allocationId, grant?.operationId, grant?.ownerId] + if ( + grant?.action !== 'allocate' || + values.some((value) => typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(value)) || + !link || + !net.isIP(link.address) || + !Number.isInteger(link.port) || + link.port < 1 || + link.port > 65535 || + typeof link.token !== 'string' || + !/^[\x21-\x7e]{32,256}$/.test(link.token) || + !/^[a-f0-9]{64}$/.test(link.certSha256) + ) + return Promise.reject(denied()) + const request = { + op: 'allocate', + vm_id: grant.vmId, + allocation_id: grant.allocationId, + operation_id: grant.operationId, + owner_id: grant.ownerId, + } + return new Promise((resolve, reject) => { + let settled = false + let buffer = Buffer.alloc(0) + const socket = tls.connect({ + host: link.address, + port: link.port, + rejectUnauthorized: false, + minVersion: 'TLSv1.3', + }) + const finish = (error, value) => { + if (settled) return + settled = true + clearTimeout(timer) + socket.destroy() + if (error) reject(error) + else resolve(value) + } + const timer = setTimeout(() => finish(denied()), 8000) + socket.once('error', () => finish(denied())) + socket.once('close', () => finish(denied())) + socket.once('secureConnect', () => { + const pin = socket.getPeerCertificate()?.fingerprint256?.replaceAll(':', '').toLowerCase() + if (pin !== link.certSha256) return finish(denied()) + socket.write(`KSG2 ${link.token}\n${JSON.stringify(request)}\n`) + }) + socket.on('data', (chunk) => { + buffer = Buffer.concat([buffer, chunk]) + if (buffer.length > 8192) return finish(denied()) + const newline = buffer.indexOf(10) + if (newline < 0) return + try { + const reply = JSON.parse(buffer.subarray(0, newline).toString(), (key, value, context) => + key === 'generation' && typeof value === 'number' ? context.source : value, + ) + if (reply.ok !== true) throw denied() + finish(null, normalizeAllocationReceipt(reply.allocation, grant)) + } catch { + finish(denied()) + } + }) + }) +} diff --git a/src/lib/vm/shared-vm-operations.mjs b/src/lib/vm/shared-vm-operations.mjs index 0ba23e329..74b97f2d6 100644 --- a/src/lib/vm/shared-vm-operations.mjs +++ b/src/lib/vm/shared-vm-operations.mjs @@ -3,6 +3,7 @@ import { AsyncLocalStorage } from 'node:async_hooks' import { withTransaction } from '../db/database.mjs' import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' import { physicalVmAliases } from './vm-physical-target.mjs' +import { revokeNodeOperationGrants } from '../db/node-management-authority.mjs' const callbacks = new AsyncLocalStorage() @@ -108,7 +109,7 @@ export class SharedVmOperations { let uncertain = false let marking = null let releasing = null - return { + return Object.freeze({ ok: true, ...identity, get uncertain() { @@ -149,25 +150,37 @@ export class SharedVmOperations { } return releasing }, - } + }) } async markUncertain({ operationId, ownerId }) { - const result = await this.db - .prepare(`UPDATE vm_operations - SET uncertain_at = COALESCE(uncertain_at, clock_timestamp()) - WHERE id = $1 AND owner_id = $2 AND released_at IS NULL`) - .run(operationId, ownerId) - return { uncertain: result.changes > 0 } + return withTransaction(this.db, async () => { + const row = await this.db + .prepare('SELECT id FROM vm_operations WHERE id=$1 AND owner_id=$2 AND released_at IS NULL FOR UPDATE') + .get(operationId, ownerId) + if (!row) return { uncertain: false } + await revokeNodeOperationGrants(this.db, { operationId, ownerId }) + const result = await this.db + .prepare(`UPDATE vm_operations SET uncertain_at=COALESCE(uncertain_at,clock_timestamp()) + WHERE id=$1 AND owner_id=$2 AND released_at IS NULL`) + .run(operationId, ownerId) + return { uncertain: result.changes > 0 } + }) } async release({ operationId, ownerId }) { - const result = await this.db - .prepare(`UPDATE vm_operations - SET released_at = COALESCE(released_at, clock_timestamp()) - WHERE id = $1 AND owner_id = $2 AND uncertain_at IS NULL`) - .run(operationId, ownerId) - return { released: result.changes > 0 } + return withTransaction(this.db, async () => { + const row = await this.db + .prepare('SELECT id FROM vm_operations WHERE id=$1 AND owner_id=$2 AND uncertain_at IS NULL FOR UPDATE') + .get(operationId, ownerId) + if (!row) return { released: false } + await revokeNodeOperationGrants(this.db, { operationId, ownerId }) + const result = await this.db + .prepare(`UPDATE vm_operations SET released_at=COALESCE(released_at,clock_timestamp()) + WHERE id=$1 AND owner_id=$2 AND uncertain_at IS NULL`) + .run(operationId, ownerId) + return { released: result.changes > 0 } + }) } async run(vmId, callback, { kind = 'mutation', accountId = null, admission = null } = {}) { @@ -207,7 +220,14 @@ export class SharedVmOperations { return { ok: true } }) if (!valid.ok) throw operationError(valid.code) - return callback(nested.operation) + try { + const result = await callback(nested.operation) + if (result?.vmOperationUncertain === true) await nested.operation.markUncertain() + return result + } catch (error) { + if (error?.vmOperationNotStarted !== true || nested.operation.uncertain) await nested.operation.markUncertain() + throw error + } } const operation = await this.tryAcquire({ vmId, kind, accountId, admission }) if (!operation.ok) throw operationError(operation.code) diff --git a/test/fixtures/real-allocation-gate.mjs b/test/fixtures/real-allocation-gate.mjs new file mode 100644 index 000000000..cfaa8b348 --- /dev/null +++ b/test/fixtures/real-allocation-gate.mjs @@ -0,0 +1,58 @@ +import { spawn, execFileSync } from 'node:child_process' +import { once } from 'node:events' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +export async function realAllocationGate(t, management) { + const root = fs.mkdtempSync(path.join(os.tmpdir(),'ag2-')) + const worker = path.join(root,'worker') + fs.cpSync(fileURLToPath(new URL('../../worker',import.meta.url)),worker,{recursive:true}) + const cmd = path.join(worker,'cmd','allocation-fixture') + fs.mkdirSync(cmd,{recursive:true}) + fs.copyFileSync(fileURLToPath(new URL('./slotgate-allocation-node.go',import.meta.url)),path.join(cmd,'main.go')) + const binary = path.join(root,'gate') + let child + let exited + let closed = false + const close = async () => { + if (closed) return + closed = true + try { + if (child && child.exitCode == null) { + child.kill('SIGTERM') + const timer = setTimeout(() => child.kill('SIGKILL'),3000) + try { await exited } finally { clearTimeout(timer) } + } + } finally { fs.rmSync(root,{recursive:true,force:true}) } + } + t.after(close) + try { + execFileSync('go',['build','-buildvcs=false','-o',binary,'./cmd/allocation-fixture'],{ + cwd:worker,env:{...process.env,GOCACHE:'/tmp/vm2api-648-go-cache',GOPROXY:'off'},stdio:'pipe',timeout:60000, + }) + const runtime = path.join(root,'runtime') + fs.mkdirSync(runtime,{mode:0o700}) + const configFile = path.join(root,'config.json') + fs.writeFileSync(configFile,JSON.stringify({listen:'127.0.0.1:0',root:runtime,token:'synthetic-node-allocation-token-000000',tls_dir:path.join(root,'tls'), + fenced:true,authority_dir:path.join(root,'authority'),docker_socket:path.join(root,'never-opened.sock'),management}),{mode:0o600}) + child = spawn(binary,[configFile],{stdio:['ignore','pipe','pipe']}) + exited = once(child,'exit') + const link = await new Promise((resolve,reject) => { + let buffer = '' + const timer = setTimeout(() => reject(new Error('synthetic allocation fixture startup timeout')),5000) + child.once('error',(error) => {clearTimeout(timer);reject(error)}) + child.once('exit',(code) => {clearTimeout(timer);if(code)reject(new Error(`synthetic allocation fixture exit ${code}`))}) + child.stderr.resume() + child.stdout.on('data',(chunk) => { + buffer += chunk + const newline = buffer.indexOf('\n') + if (newline < 0) return + clearTimeout(timer) + try {resolve(JSON.parse(buffer.slice(0,newline)))} catch(error){reject(error)} + }) + }) + return {link,root,close} + } catch(error) {await close();throw error} +} diff --git a/test/fixtures/slotgate-allocation-node.go b/test/fixtures/slotgate-allocation-node.go new file mode 100644 index 000000000..0f607d06e --- /dev/null +++ b/test/fixtures/slotgate-allocation-node.go @@ -0,0 +1,41 @@ +// Local authorization interoperability fixture. No Docker call or executor is created. +package main + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "encoding/json" + "net" + "os" + "os/signal" + "syscall" + + "github.com/dofastted/kin-gateway/worker/internal/slotgate" +) + +func check(err error) { + if err != nil { + panic(err) + } +} +func main() { + file, err := os.Open(os.Args[1]) + check(err) + var config slotgate.Config + check(json.NewDecoder(file).Decode(&config)) + check(file.Close()) + server, err := slotgate.New(config) + check(err) + listener, err := net.Listen("tcp", "127.0.0.1:0") + check(err) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + done := make(chan error, 1) + go func() { done <- server.ServeListener(ctx, tls.NewListener(listener, server.TLSConfig())) }() + cert := sha256.Sum256(server.TLSConfig().Certificates[0].Certificate[0]) + check(json.NewEncoder(os.Stdout).Encode(map[string]any{"address": "127.0.0.1", "port": listener.Addr().(*net.TCPAddr).Port, "token": config.Token, "certSha256": hex.EncodeToString(cert[:])})) + <-ctx.Done() + check(<-done) +} diff --git a/test/unit/node-management-grants.test.mjs b/test/unit/node-management-grants.test.mjs index bbb25d090..ee46be1ed 100644 --- a/test/unit/node-management-grants.test.mjs +++ b/test/unit/node-management-grants.test.mjs @@ -1,83 +1,363 @@ import test from 'node:test' import assert from 'node:assert/strict' import pg from 'pg' +import crypto from 'node:crypto' +import fs from 'node:fs' +import path from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { nodeGrantNamespace, nodeGrantResource } from '../../src/lib/db/node-management-authority.mjs' import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { NodeManagementGrants, nodeAuthoritySchemaFor } from '../../src/lib/cluster/node-management-grants.mjs' +import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' +import { allocateNodeSlot } from '../../src/lib/transport/slotgate-management-client.mjs' async function fixture(t) { const db = getDb() const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) const maintenance = new GlobalMaintenanceCoordinator(db) - await maintenance.start() - await maintenance.acknowledgeEpoch(1) const service = new NodeManagementGrants(db) - await service.initialize() const held = [] - t.after(async () => { - for (const operation of held) { operation.resume(); await operation.pending } + const roles = [] + const readers = [] + let closed = false + const close = async () => { + if (closed) return + closed = true + for (const operation of held) operation.resume() + await Promise.allSettled(held.map((operation) => operation.pending)) await maintenance.stop() await peer.close() + for (const client of readers) await client.end() const cleanup = new pg.Client({ connectionString: db.connectionString }) await cleanup.connect() try { + for (const role of roles) { + await cleanup.query(`DROP OWNED BY ${quoteIdentifier(role)}; DROP ROLE ${quoteIdentifier(role)}`) + } await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE`) await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(maintenance.schema)} CASCADE`) - } finally { await cleanup.end() } - }) + } finally { + await cleanup.end() + } + } + t.after(close) + try { + await maintenance.start() + await maintenance.acknowledgeEpoch(1) + await service.initialize() + } catch (error) { + await close() + throw error + } async function issue(allocationId = 'allocation-one', patch = {}) { let resume let ready - const waiting = new Promise((resolve) => { resume = resolve }) - const received = new Promise((resolve) => { ready = resolve }) - const pending = new SharedVmOperations(db, { ownerId: 'synthetic-controller' }).run('synthetic-vm', async (operation) => { - const grant = await service.issueAllocation({ operation, vmId: 'synthetic-vm', nodeId: 'synthetic-node', allocationId, ...patch }) - ready({ grant, operation }) - await waiting - }, { kind: 'node_allocate' }) + const waiting = new Promise((resolve) => { + resume = resolve + }) + const received = new Promise((resolve) => { + ready = resolve + }) + const pending = new SharedVmOperations(db, { ownerId: 'synthetic-controller' }).run( + 'synthetic-vm', + async (operation) => { + const grant = await service.issueAllocation({ + operation, + vmId: 'synthetic-vm', + nodeId: 'synthetic-node', + allocationId, + ...patch, + }) + ready({ grant, operation }) + await waiting + }, + { kind: 'node_allocate' }, + ) pending.catch((error) => ready({ error })) held.push({ pending, resume }) const value = await received if (value.error) throw value.error return { ...value, resume, pending } } - return { db, peer, service, maintenance, issue } + async function reader(nodeId = 'synthetic-node') { + const role = 'node_reader_' + crypto.randomBytes(8).toString('hex') + const password = crypto.randomBytes(24).toString('hex') + await db.pool.query( + `CREATE ROLE ${quoteIdentifier(role)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, + ) + roles.push(role) + await db.pool.query(service.readerRoleSql({ role, nodeId })) + const url = new URL(db.connectionString) + url.username = role + url.password = password + const client = new pg.Client({ connectionString: url.toString() }) + await client.connect() + readers.push(client) + return { client, connectionString: url.toString() } + } + return { db, peer, service, maintenance, issue, close, reader } } -test('allocation grants require the exact current operation scope and persist outside the application schema', async (t) => { +async function usingFixture(t, callback) { const fx = await fixture(t) - await assert.rejects(fx.service.issueAllocation({ operation: { operationId: 'forged', ownerId: 'synthetic-controller' }, - vmId: 'synthetic-vm', nodeId: 'synthetic-node', allocationId: 'forged' }), { code: 'node_operation_scope_required' }) - const { grant } = await fx.issue() - assert.equal(grant.action, 'allocate') - assert.equal(grant.nodeId, 'synthetic-node') - assert.notEqual(fx.service.schema, fx.db.schema) - assert.equal(fx.service.schema, nodeAuthoritySchemaFor(fx.db.schema)) - const row = (await fx.peer.pool.query(`SELECT * FROM ${quoteIdentifier(fx.service.schema)}.allocations`)).rows[0] - assert.equal(row.operation_id, grant.operationId) - assert.equal(row.revoked_at, null) + try { + return await callback(fx) + } finally { + await fx.close() + } +} + +test('allocation grants require the exact current operation scope and persist outside the application schema', async (t) => { + await usingFixture(t, async (fx) => { + await assert.rejects( + fx.service.issueAllocation({ + operation: { operationId: 'forged', ownerId: 'synthetic-controller' }, + vmId: 'synthetic-vm', + nodeId: 'synthetic-node', + allocationId: 'forged', + }), + { code: 'node_operation_scope_required' }, + ) + const { grant } = await fx.issue() + assert.equal(grant.action, 'allocate') + assert.equal(grant.nodeId, 'synthetic-node') + assert.notEqual(fx.service.schema, fx.db.schema) + assert.equal(fx.service.schema, nodeAuthoritySchemaFor(fx.db.schema)) + const row = (await fx.peer.pool.query(`SELECT * FROM ${quoteIdentifier(fx.service.schema)}.allocations`)).rows[0] + assert.equal(row.operation_id, grant.operationId) + assert.equal(row.revoked_at, null) + }) +}) + +test('the installed reader role sees only its node grants and cannot read or mutate business/base tables', async (t) => { + await usingFixture(t, async (fx) => { + const correct = await fx.reader() + const wrong = await fx.reader('different-node') + await fx.issue() + const q = quoteIdentifier(fx.service.schema) + assert.equal((await correct.client.query(`SELECT * FROM ${q}.authorized_allocations`)).rowCount, 1) + assert.equal((await wrong.client.query(`SELECT * FROM ${q}.authorized_allocations`)).rowCount, 0) + await assert.rejects(correct.client.query(`SELECT * FROM ${q}.allocations`), { code: '42501' }) + await assert.rejects(correct.client.query(`SELECT * FROM ${quoteIdentifier(fx.db.schema)}.vm_operations`), { + code: '42501', + }) + await assert.rejects(correct.client.query(`UPDATE ${q}.allocations SET revoked_at=NULL`), { code: '42501' }) + }) +}) + +test('the actual Go node independently authorizes allocate and preserves the original receipt across two controller connections', { + skip: process.platform !== 'linux', +}, async (t) => { + await usingFixture(t, async (fx) => { + const reader = await fx.reader() + const node = await realAllocationGate(t, { + connection_string: reader.connectionString, + authority_schema: fx.service.schema, + node_id: 'synthetic-node', + }) + try { + const held = await fx.issue() + const receipts = await Promise.all([ + allocateNodeSlot({ link: node.link, grant: held.grant }), + allocateNodeSlot({ link: node.link, grant: held.grant }), + ]) + assert.deepEqual(receipts[0], receipts[1]) + assert.equal(receipts[0].fence.generation, '1') + await assert.rejects(allocateNodeSlot({ link: node.link, grant: { ...held.grant, ownerId: 'other-owner' } }), { + code: 'node_allocation_unconfirmed', + }) + await held.operation.markUncertain() + await assert.rejects(allocateNodeSlot({ link: node.link, grant: held.grant }), { + code: 'node_allocation_unconfirmed', + }) + } finally { + await node.close() + } + }) }) test('operation release or uncertainty revokes its private allocation grant', async (t) => { - const fx = await fixture(t) - const held = await fx.issue() - await held.operation.markUncertain() - const row = (await fx.peer.pool.query(`SELECT revoked_at FROM ${quoteIdentifier(fx.service.schema)}.allocations`)).rows[0] - assert.ok(row.revoked_at) + await usingFixture(t, async (fx) => { + const held = await fx.issue() + await held.operation.markUncertain() + const row = (await fx.peer.pool.query(`SELECT revoked_at FROM ${quoteIdentifier(fx.service.schema)}.allocations`)) + .rows[0] + assert.ok(row.revoked_at) + }) }) test('once enabled, missing private grant state prevents release rather than becoming legacy mode', async (t) => { - const fx = await fixture(t) - const held = await fx.issue() - await fx.peer.pool.query(`DROP SCHEMA ${quoteIdentifier(fx.service.schema)} CASCADE`) - await assert.rejects(held.operation.release(), { code: 'node_authority_unavailable' }) - const row = await fx.peer.prepare('SELECT released_at FROM vm_operations WHERE id=$1').get(held.operation.operationId) - assert.equal(row.released_at, null) - // The callback deliberately leaves an unknown operation; cleanup must not retry a normal release. - await fx.peer.prepare('UPDATE vm_operations SET uncertain_at=clock_timestamp() WHERE id=$1').run(held.operation.operationId) - held.resume() - await assert.rejects(held.pending) - // Prevent the shared fixture cleanup from observing the already asserted rejection twice. - held.pending.catch(() => {}) + await usingFixture(t, async (fx) => { + const held = await fx.issue() + await fx.peer.pool.query(`DROP SCHEMA ${quoteIdentifier(fx.service.schema)} CASCADE`) + await assert.rejects(held.operation.release(), { code: 'node_authority_unavailable' }) + const row = await fx.peer + .prepare('SELECT released_at FROM vm_operations WHERE id=$1') + .get(held.operation.operationId) + assert.equal(row.released_at, null) + // The callback deliberately leaves an unknown operation; cleanup must not retry a normal release. + await fx.peer + .prepare('UPDATE vm_operations SET uncertain_at=clock_timestamp() WHERE id=$1') + .run(held.operation.operationId) + held.resume() + await assert.rejects(held.pending) + // Prevent the shared fixture cleanup from observing the already asserted rejection twice. + held.pending.catch(() => {}) + }) +}) + +test('release waits for the node shared authorization lock and then revokes before returning', async (t) => { + await usingFixture(t, async (fx) => { + const reader = await fx.reader() + const held = await fx.issue() + await reader.client.query('BEGIN READ ONLY') + await reader.client.query('SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))', [ + nodeGrantNamespace(fx.service.schema), + nodeGrantResource(held.grant.nodeId, held.grant.vmId), + ]) + const releasing = held.operation.release() + try { + assert.equal(await Promise.race([releasing.then(() => 'released'), delay(100).then(() => 'waiting')]), 'waiting') + } finally { + await reader.client.query('COMMIT') + } + assert.equal((await releasing).released, true) + assert.equal( + (await reader.client.query(`SELECT * FROM ${quoteIdentifier(fx.service.schema)}.authorized_allocations`)) + .rowCount, + 0, + ) + }) +}) + +test('dropping business state preserves the private view and a new maintenance epoch rejects restored old operation rows', async (t) => { + await usingFixture(t, async (fx) => { + const reader = await fx.reader() + const held = await fx.issue() + const q = quoteIdentifier(fx.service.schema) + const app = quoteIdentifier(fx.db.schema) + await fx.peer.pool.query(`DROP TABLE ${app}.vm_operations CASCADE`) + assert.ok( + (await fx.peer.pool.query('SELECT to_regclass($1) AS relation', [`${q}.authorized_allocations`])).rows[0] + .relation, + ) + assert.equal((await reader.client.query(`SELECT * FROM ${q}.authorized_allocations`)).rowCount, 0) + await fx.peer.pool.query(`CREATE TABLE ${app}.vm_operations(id text PRIMARY KEY,vm_id text,owner_id text,kind text,account_id text, + created_at timestamptz DEFAULT clock_timestamp(),released_at timestamptz,uncertain_at timestamptz)`) + await fx.peer.pool.query( + `INSERT INTO ${app}.vm_operations(id,vm_id,owner_id,kind) VALUES($1,$2,$3,'node_allocate')`, + [held.grant.operationId, held.grant.vmId, held.grant.ownerId], + ) + await fx.peer.pool.query(`UPDATE ${quoteIdentifier(fx.maintenance.schema)}.control SET epoch=epoch+1 WHERE id=1`) + assert.equal((await reader.client.query(`SELECT * FROM ${q}.authorized_allocations`)).rowCount, 0) + }) +}) + +test('the typed service persists the authenticated initial receipt before its VM operation is released', { + skip: process.platform !== 'linux', +}, async (t) => { + await usingFixture(t, async (fx) => { + const reader = await fx.reader() + const node = await realAllocationGate(t, { + connection_string: reader.connectionString, + authority_schema: fx.service.schema, + node_id: 'synthetic-node', + }) + try { + fx.service.resolveNodeLink = async () => node.link + const receipt = await new SharedVmOperations(fx.db, { ownerId: 'synthetic-typed-controller' }).run( + 'typed-vm', + (operation) => + fx.service.allocate({ + operation, + nodeId: 'synthetic-node', + vmId: 'typed-vm', + allocationId: 'typed-allocation', + }), + { kind: 'node_allocate' }, + ) + const row = ( + await fx.peer.pool.query( + `SELECT receipt,revoked_at FROM ${quoteIdentifier(fx.service.schema)}.allocations WHERE allocation_id='typed-allocation'`, + ) + ).rows[0] + assert.deepEqual(row.receipt, receipt) + assert.ok(row.revoked_at) + } finally { + await node.close() + } + }) +}) + +test('a node PG authorization failure leaves no local allocation and has no fallback', { + skip: process.platform !== 'linux', +}, async (t) => { + await usingFixture(t, async (fx) => { + const reader = await fx.reader() + const unreachable = new URL(reader.connectionString) + unreachable.port = '1' + const node = await realAllocationGate(t, { + connection_string: unreachable.toString(), + authority_schema: fx.service.schema, + node_id: 'synthetic-node', + }) + try { + const held = await fx.issue() + await assert.rejects(allocateNodeSlot({ link: node.link, grant: held.grant }), { + code: 'node_allocation_unconfirmed', + }) + const journal = JSON.parse(fs.readFileSync(path.join(node.root, 'authority', 'authority.json'), 'utf8')) + assert.deepEqual(journal.slots, {}) + assert.equal(fs.existsSync(path.join(node.root, 'runtime', 'vms')), false) + } finally { + await node.close() + } + }) +}) + +test('a PG receipt write failure retains the unknown operation after a local allocation', { + skip: process.platform !== 'linux', +}, async (t) => { + await usingFixture(t, async (fx) => { + const reader = await fx.reader() + const node = await realAllocationGate(t, { + connection_string: reader.connectionString, + authority_schema: fx.service.schema, + node_id: 'synthetic-node', + }) + try { + const q = quoteIdentifier(fx.service.schema) + await fx.db.pool.query(`CREATE FUNCTION ${q}.reject_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic receipt failure' USING ERRCODE='P0001'; END; $$; + CREATE TRIGGER reject_receipt BEFORE UPDATE OF receipt ON ${q}.allocations FOR EACH ROW EXECUTE FUNCTION ${q}.reject_receipt()`) + fx.service.resolveNodeLink = async () => node.link + await assert.rejects( + new SharedVmOperations(fx.db, { ownerId: 'synthetic-receipt-controller' }).run( + 'receipt-vm', + (operation) => + fx.service.allocate({ + operation, + nodeId: 'synthetic-node', + vmId: 'receipt-vm', + allocationId: 'receipt-allocation', + }), + { kind: 'node_allocate' }, + ), + { code: 'P0001' }, + ) + const operation = await fx.peer + .prepare("SELECT released_at,uncertain_at FROM vm_operations WHERE vm_id='receipt-vm'") + .get() + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + const allocation = (await fx.peer.pool.query(`SELECT receipt,revoked_at FROM ${q}.allocations`)).rows[0] + assert.equal(allocation.receipt, null) + assert.ok(allocation.revoked_at) + assert.equal(fs.existsSync(path.join(node.root, 'runtime', 'vms')), false) + } finally { + await node.close() + } + }) }) diff --git a/test/unit/vm-operation-identity.test.mjs b/test/unit/vm-operation-identity.test.mjs index f12561fcb..451dd8a4d 100644 --- a/test/unit/vm-operation-identity.test.mjs +++ b/test/unit/vm-operation-identity.test.mjs @@ -3,17 +3,21 @@ import assert from 'node:assert/strict' import { getDb } from '../../src/lib/db/database.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' -test('operation identity fields cannot be reassigned while closure state can become uncertain',async () => { - const operations = new SharedVmOperations(getDb(),{ownerId:'immutable-owner'}) - const operation = await operations.tryAcquire({vmId:'immutable-vm',kind:'node_allocate'}) +test('operation identity fields cannot be reassigned while closure state can become uncertain', async () => { + const operations = new SharedVmOperations(getDb(), { ownerId: 'immutable-owner' }) + const operation = await operations.tryAcquire({ vmId: 'immutable-vm', kind: 'node_allocate' }) try { - assert.equal(operation.ok,true) - for (const field of ['operationId','ownerId','vmId','kind']) { - assert.throws(() => {operation[field] = 'different-identity'},TypeError) + assert.equal(operation.ok, true) + for (const field of ['operationId', 'ownerId', 'vmId', 'kind']) { + assert.throws(() => { + operation[field] = 'different-identity' + }, TypeError) } - assert.equal(operation.uncertain,false) - assert.equal((await operation.markUncertain()).uncertain,true) - assert.equal(operation.uncertain,true) - assert.equal((await operation.release()).released,false) - } finally {await operation.release()} + assert.equal(operation.uncertain, false) + assert.equal((await operation.markUncertain()).uncertain, true) + assert.equal(operation.uncertain, true) + assert.equal((await operation.release()).released, false) + } finally { + await operation.release() + } }) diff --git a/worker/go.mod b/worker/go.mod index 61eaf2b39..5dd5c08a1 100644 --- a/worker/go.mod +++ b/worker/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/imroc/req/v3 v3.61.0 + github.com/jackc/pgx/v5 v5.11.0 golang.org/x/net v0.57.0 ) @@ -11,6 +12,8 @@ require ( github.com/andybalholm/brotli v1.2.2 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/icholy/digest v1.2.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.61.0 // indirect diff --git a/worker/go.sum b/worker/go.sum index b5a3f4260..a56109a8c 100644 --- a/worker/go.sum +++ b/worker/go.sum @@ -1,5 +1,6 @@ github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -11,6 +12,12 @@ github.com/icholy/digest v1.2.0 h1:oTbG4IsNOmidJ+421ehG7Ty93yt1yotq13kFMG569yw= github.com/icholy/digest v1.2.0/go.mod h1:1P1+LzUv48ybX7bu8tVpZ2QWdd+xRuePNuGawHjwRUE= github.com/imroc/req/v3 v3.61.0 h1:JTfaJFkTFAfrG0r1mB6NkkAG+rx+DU8a0UctB0RR8yU= github.com/imroc/req/v3 v3.61.0/go.mod h1:zbKPxxawfwa0RmYp7Nm8yh5cCBD/7W/ho3tp3tqEnPc= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -23,6 +30,9 @@ github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNv github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/xyproto/randomstring v1.2.0 h1:y7PXAEBM3XlwJjPG2JQg4voxBYZ4+hPgRdGKCfU8wik= @@ -37,6 +47,8 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= diff --git a/worker/internal/slotgate/allocate.go b/worker/internal/slotgate/allocate.go new file mode 100644 index 000000000..344e01e26 --- /dev/null +++ b/worker/internal/slotgate/allocate.go @@ -0,0 +1,105 @@ +package slotgate + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "strconv" + "strings" +) + +var ( + ErrManagementDenied = errors.New("management_denied") + ErrManagementUnavailable = errors.New("management_authority_unavailable") + ErrAllocationConflict = errors.New("allocation_conflict") +) + +type AllocationIntent struct { + NodeID string `json:"node_id"` + VMID string `json:"vm_id"` + AllocationID string `json:"allocation_id"` + OperationID string `json:"operation_id"` + OwnerID string `json:"owner_id"` +} + +type AllocationReceipt struct { + AllocationIntent + Epoch string `json:"epoch"` + IntentDigest string `json:"intent_digest"` + Fence Fence `json:"fence"` +} + +func (i AllocationIntent) digest() string { + digest := sha256.Sum256([]byte(strings.Join([]string{"slotgate-allocate-v1", i.NodeID, i.VMID, i.AllocationID, i.OperationID, i.OwnerID}, "\x00"))) + return hex.EncodeToString(digest[:]) +} + +func validAllocation(i AllocationIntent, epoch string) bool { + for _, value := range []string{i.NodeID, i.VMID, i.AllocationID, i.OperationID, i.OwnerID} { + if !vmIDRe.MatchString(value) { + return false + } + } + n, err := strconv.ParseUint(epoch, 10, 63) + return err == nil && n > 0 && strconv.FormatUint(n, 10) == epoch +} + +// Allocate changes only the node journal. The caller is a trusted authorizer +// holding a live grant lock; no directory, process or container is created. +func (a *Authority) Allocate(intent AllocationIntent, epoch string) (AllocationReceipt, error) { + a.mu.Lock() + defer a.mu.Unlock() + if err := a.available(); err != nil { + return AllocationReceipt{}, err + } + if !validAllocation(intent, epoch) { + return AllocationReceipt{}, ErrInvalidAuthority + } + for vm, state := range a.journal.Slots { + if state.Allocation != nil && state.Allocation.AllocationID == intent.AllocationID && vm != intent.VMID { + return AllocationReceipt{}, ErrAllocationConflict + } + } + if state, exists := a.journal.Slots[intent.VMID]; exists { + if state.Allocation == nil || state.Allocation.AllocationIntent != intent || state.Allocation.Epoch != epoch { + return AllocationReceipt{}, ErrAllocationConflict + } + // Return the original vacant fence, never relabel an old allocation + // with a later active executor or generation from State(). + return *state.Allocation, nil + } + incarnation, err := randomAuthorityID() + if err != nil { + return AllocationReceipt{}, err + } + receipt := AllocationReceipt{AllocationIntent: intent, Epoch: epoch, IntentDigest: intent.digest(), Fence: Fence{Generation: 1, Incarnation: incarnation}} + state := AuthorityState{Fence: receipt.Fence, Phase: PhaseVacant, Commands: map[string]CommandRecord{}, Allocation: &receipt} + if _, err = a.commit(intent.VMID, state); err != nil { + return AllocationReceipt{}, err + } + return receipt, nil +} + +type allocationAuthorizer func(context.Context, AllocationIntent, func(string) error) error +type allocationManager struct { + authority *Authority + nodeID string + authorize allocationAuthorizer +} + +func (m *allocationManager) Allocate(ctx context.Context, intent AllocationIntent) (AllocationReceipt, error) { + if m.authorize == nil || intent.NodeID != m.nodeID { + return AllocationReceipt{}, ErrManagementDenied + } + var receipt AllocationReceipt + err := m.authorize(ctx, intent, func(epoch string) error { + var err error + receipt, err = m.authority.Allocate(intent, epoch) + return err + }) + if err != nil { + return AllocationReceipt{}, err + } + return receipt, nil +} diff --git a/worker/internal/slotgate/allocate_test.go b/worker/internal/slotgate/allocate_test.go index d803ee1a6..05d0541a6 100644 --- a/worker/internal/slotgate/allocate_test.go +++ b/worker/internal/slotgate/allocate_test.go @@ -3,9 +3,10 @@ package slotgate import ( "context" "errors" + "os" "path/filepath" - "sync" "strings" + "sync" "testing" ) @@ -90,15 +91,39 @@ func TestKSG2ForwardTokenAloneCannotAuthorizeManagement(t *testing.T) { func TestInitializedManagementBindingCannotBeRemovedOrRelabeled(t *testing.T) { root := shortRoot(t) - management := &PGManagementConfig{ConnectionString:"postgres://synthetic@127.0.0.1:1/synthetic_test?sslmode=disable",AuthoritySchema:"vm2api_node_"+strings.Repeat("a",32),NodeID:"synthetic-node"} - config := Config{Listen:"127.0.0.1:0",Root:root,Token:testToken,TLSDir:filepath.Join(root,"tls"),Fenced:true,AuthorityDir:filepath.Join(root,"authority"),Management:management} - server,err := New(config) - if err != nil { t.Fatal(err) } + management := &PGManagementConfig{ConnectionString: "postgres://synthetic@127.0.0.1:1/synthetic_test?sslmode=disable", AuthoritySchema: "vm2api_node_" + strings.Repeat("a", 32), NodeID: "synthetic-node"} + config := Config{Listen: "127.0.0.1:0", Root: root, Token: testToken, TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: filepath.Join(root, "authority"), Management: management} + server, err := New(config) + if err != nil { + t.Fatal(err) + } server.authority.Close() config.Management = nil - if reopened,err := New(config); err == nil { reopened.authority.Close();t.Fatal("initialized node accepted missing authorizer") } + if reopened, err := New(config); err == nil { + reopened.authority.Close() + t.Fatal("initialized node accepted missing authorizer") + } copy := *management copy.NodeID = "different-node" config.Management = © - if reopened,err := New(config); err == nil { reopened.authority.Close();t.Fatal("initialized node accepted a different identity") } + if reopened, err := New(config); err == nil { + reopened.authority.Close() + t.Fatal("initialized node accepted a different identity") + } +} + +func TestAllocationJournalFailureNeverAcknowledgesOrResetsGeneration(t *testing.T) { + a := authorityForTest(t, filepath.Join(shortRoot(t), "authority"), nil) + if err := os.Rename(filepath.Join(a.root, "authority.json"), filepath.Join(a.root, "saved.json")); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(a.root, "authority.json"), 0700); err != nil { + t.Fatal(err) + } + if _, err := a.Allocate(allocationIntent("write-failure"), "1"); err == nil { + t.Fatal("acknowledged failed journal") + } + if _, err := a.State("v"); !errors.Is(err, ErrAuthorityUnavailable) { + t.Fatalf("not poisoned: %v", err) + } } diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index a271e495f..e53bcd51e 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -100,6 +100,7 @@ type DockerLaunchTarget struct { } type AuthorityState struct { + Allocation *AllocationReceipt `json:"allocation,omitempty"` Fence Fence `json:"fence"` Phase AuthorityPhase `json:"phase"` Commands map[string]CommandRecord `json:"commands"` @@ -128,8 +129,9 @@ type TerminationProof struct { type TerminationVerifier func(context.Context, ExecutorIdentity) (TerminationProof, error) type authorityJournal struct { - Version int `json:"version"` - Slots map[string]AuthorityState `json:"slots"` + Version int `json:"version"` + Slots map[string]AuthorityState `json:"slots"` + Management *managementBinding `json:"management,omitempty"` } type Authority struct { @@ -611,7 +613,7 @@ func (a *Authority) available() error { } func (a *Authority) commit(vm string, state AuthorityState) (AuthorityState, error) { - next := authorityJournal{Version: 1, Slots: make(map[string]AuthorityState, len(a.journal.Slots)+1)} + next := authorityJournal{Version: 1, Slots: make(map[string]AuthorityState, len(a.journal.Slots)+1), Management: a.journal.Management} for key, value := range a.journal.Slots { next.Slots[key] = value } @@ -664,6 +666,10 @@ func (a *Authority) persist(next authorityJournal) error { } func cloneState(state AuthorityState) AuthorityState { + if state.Allocation != nil { + receipt := *state.Allocation + state.Allocation = &receipt + } commands := make(map[string]CommandRecord, len(state.Commands)) for id, command := range state.Commands { if command.LaunchTarget != nil { @@ -749,7 +755,18 @@ func validateJournal(journal authorityJournal) error { if journal.Version != 1 || journal.Slots == nil { return ErrInvalidAuthority } + if binding := journal.Management; binding != nil && (binding.ProtocolVersion != 1 || !vmIDRe.MatchString(binding.NodeID) || !authoritySchemaRE.MatchString(binding.AuthoritySchema)) { + return ErrInvalidAuthority + } for vm, state := range journal.Slots { + if receipt := state.Allocation; receipt != nil { + if journal.Management != nil && receipt.NodeID != journal.Management.NodeID { + return ErrInvalidAuthority + } + if !validAllocation(receipt.AllocationIntent, receipt.Epoch) || receipt.VMID != vm || receipt.IntentDigest != receipt.AllocationIntent.digest() || receipt.Fence.Generation != 1 || !validAuthorityID(receipt.Fence.Incarnation) || receipt.Fence.Executor != (ExecutorIdentity{}) || (state.Fence.Generation == 1 && state.Fence.Incarnation != receipt.Fence.Incarnation) { + return ErrInvalidAuthority + } + } if !vmIDRe.MatchString(vm) || state.Fence.Generation == 0 || !validAuthorityID(state.Fence.Incarnation) || state.Commands == nil { return ErrInvalidAuthority } diff --git a/worker/internal/slotgate/docker_launch_test.go b/worker/internal/slotgate/docker_launch_test.go index e67c65203..4ff4126c9 100644 --- a/worker/internal/slotgate/docker_launch_test.go +++ b/worker/internal/slotgate/docker_launch_test.go @@ -428,6 +428,7 @@ func TestKSG2LaunchCreatesAndBindsWithNodePolicyAndRejectsRawSpec(t *testing.T) if count != 0 { t.Fatal("raw spec reached Engine") } + s.authorizeManagement = func(_ context.Context, _ gate2Request, dispatch func() error) error { return dispatch() } reply := gate2Control(t, ln.Addr().String(), gate2Request{Op: "launch", VMID: "v", Fence: state.Fence, CommandID: "trusted-node-launch"}) if !reply.OK || reply.State == nil || reply.State.Phase != PhaseActive || reply.State.Fence.Executor.ContainerID != engine.container { t.Fatalf("KSG2 launch: %+v", reply) diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index 7f03a8654..0e46a3043 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -28,12 +28,15 @@ var errGateTicketInvalid = errors.New("ticket_invalid") // commands continue with one parsed HTTP/1 request and an HTTP response after // OK\n. KSG1 raw splicing is never accepted when fenced mode is enabled. type gate2Request struct { - Op string `json:"op"` - VMID string `json:"vm_id"` - Fence Fence `json:"fence"` - CommandID string `json:"command_id,omitempty"` - Kind string `json:"kind,omitempty"` - Ticket string `json:"ticket,omitempty"` + Op string `json:"op"` + VMID string `json:"vm_id"` + Fence Fence `json:"fence"` + CommandID string `json:"command_id,omitempty"` + Kind string `json:"kind,omitempty"` + Ticket string `json:"ticket,omitempty"` + AllocationID string `json:"allocation_id,omitempty"` + OperationID string `json:"operation_id,omitempty"` + OwnerID string `json:"owner_id,omitempty"` } type gate2State struct { @@ -70,7 +73,7 @@ func gate2Line(reader *bufio.Reader) ([]byte, error) { } func gate2Code(err error) string { - for _, known := range []error{ErrAuthorityUnavailable, ErrAuthorityPhase, ErrFenceMismatch, ErrCommandExists, ErrCommandOwner, ErrCommandDispatched, ErrPendingMutation, ErrInvalidAuthority, ErrTerminationProof, errGateTicketInvalid} { + for _, known := range []error{ErrAuthorityUnavailable, ErrAuthorityPhase, ErrFenceMismatch, ErrCommandExists, ErrCommandOwner, ErrCommandDispatched, ErrPendingMutation, ErrInvalidAuthority, ErrTerminationProof, ErrManagementDenied, ErrManagementUnavailable, ErrAllocationConflict, errGateTicketInvalid} { if errors.Is(err, known) { return known.Error() } @@ -119,6 +122,16 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply := map[string]any{"ok": false} var state AuthorityState switch request.Op { + case "allocate": + if s.allocator == nil || request.Fence != (Fence{}) || request.CommandID != "" || request.Kind != "" || request.Ticket != "" { + err = ErrManagementDenied + break + } + var receipt AllocationReceipt + receipt, err = s.allocator.Allocate(ctx, AllocationIntent{NodeID: s.allocator.nodeID, VMID: request.VMID, AllocationID: request.AllocationID, OperationID: request.OperationID, OwnerID: request.OwnerID}) + if err == nil { + reply["allocation"] = receipt + } case "state": state, err = s.authority.State(request.VMID) case "prepare": @@ -132,15 +145,31 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["ticket"] = gate2Ticket(handle.token, request) } case "revoke": - state, err = s.authority.Revoke(request.VMID, request.Fence) + if s.authorizeManagement == nil { + err = ErrManagementDenied + break + } + err = s.authorizeManagement(ctx, request, func() error { var e error; state, e = s.authority.Revoke(request.VMID, request.Fence); return e }) case "advance": - state, err = s.authority.Advance(ctx, request.VMID, request.Fence) + if s.authorizeManagement == nil { + err = ErrManagementDenied + break + } + err = s.authorizeManagement(ctx, request, func() error { var e error; state, e = s.authority.Advance(ctx, request.VMID, request.Fence); return e }) case "launch": + if s.authorizeManagement == nil { + err = ErrManagementDenied + break + } if s.launcher == nil || request.Kind != "" || request.Ticket != "" { err = ErrInvalidAuthority break } - state, err = s.launcher.Launch(ctx, request.VMID, request.Fence, request.CommandID) + err = s.authorizeManagement(ctx, request, func() error { + var e error + state, e = s.launcher.Launch(ctx, request.VMID, request.Fence, request.CommandID) + return e + }) case "command": var handle CommandHandle handle, err = s.forwardHandle(request) @@ -168,7 +197,7 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["error"] = gate2Code(err) } else { reply["ok"] = true - if request.Op != "prepare" && request.Op != "command" { + if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" { reply["state"] = publicGateState(state) } } diff --git a/worker/internal/slotgate/ksg2_test.go b/worker/internal/slotgate/ksg2_test.go index 77917eb1f..621a2d56d 100644 --- a/worker/internal/slotgate/ksg2_test.go +++ b/worker/internal/slotgate/ksg2_test.go @@ -106,6 +106,9 @@ func fencedGate(t *testing.T) (string, string, *Server, AuthorityState, *fakeEng if err != nil { t.Fatal(err) } + // Explicit trusted adapter for these verifier-focused tests. Production + // Config.Fenced does not authorize management with a forwarding token. + s.authorizeManagement = func(_ context.Context, _ gate2Request, dispatch func() error) error { return dispatch() } state := launchForTest(t, s.authority, "v", executor) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { diff --git a/worker/internal/slotgate/pg_authorizer.go b/worker/internal/slotgate/pg_authorizer.go new file mode 100644 index 000000000..d0d2448a0 --- /dev/null +++ b/worker/internal/slotgate/pg_authorizer.go @@ -0,0 +1,160 @@ +package slotgate + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net" + "path/filepath" + "regexp" + "time" + + "github.com/jackc/pgx/v5" +) + +// This credential is installed by trusted node provisioning, not by RPC. +// The role can read only its node's authorization view and cannot issue grants. +type PGManagementConfig struct { + ConnectionString string `json:"connection_string"` + AuthoritySchema string `json:"authority_schema"` + NodeID string `json:"node_id"` +} + +type pgAllocationAuthorizer struct { + config *pgx.ConnConfig + schema string + nodeID string +} + +type managementBinding struct { + ProtocolVersion int `json:"protocol_version"` + NodeID string `json:"node_id"` + AuthoritySchema string `json:"authority_schema"` +} + +func (a *Authority) bindManagement(config *PGManagementConfig) error { + a.mu.Lock() + defer a.mu.Unlock() + if err := a.available(); err != nil { + return err + } + if config == nil { + if a.journal.Management != nil { + return ErrManagementDenied + } + for _, state := range a.journal.Slots { + if state.Allocation != nil { + return ErrManagementDenied + } + } + return nil + } + binding := managementBinding{ProtocolVersion: 1, NodeID: config.NodeID, AuthoritySchema: config.AuthoritySchema} + if !vmIDRe.MatchString(binding.NodeID) || !authoritySchemaRE.MatchString(binding.AuthoritySchema) { + return ErrManagementDenied + } + if a.journal.Management != nil { + if *a.journal.Management != binding { + return ErrManagementDenied + } + return nil + } + next := a.journal + next.Management = &binding + if err := a.persist(next); err != nil { + return err + } + a.journal = next + return nil +} + +var authoritySchemaRE = regexp.MustCompile(`^vm2api_node_[a-f0-9]{32}$`) + +func newPGAllocationAuthorizer(input PGManagementConfig) (*pgAllocationAuthorizer, error) { + if !authoritySchemaRE.MatchString(input.AuthoritySchema) || !vmIDRe.MatchString(input.NodeID) || input.ConnectionString == "" { + return nil, ErrManagementDenied + } + config, err := pgx.ParseConfig(input.ConnectionString) + if err != nil { + return nil, ErrManagementUnavailable + } + // Plaintext is limited to a node-local Unix socket/loopback test database. + local := filepath.IsAbs(config.Host) + if ip := net.ParseIP(config.Host); ip != nil { + local = local || ip.IsLoopback() + } + if !local && (config.TLSConfig == nil || config.TLSConfig.InsecureSkipVerify || len(config.Fallbacks) != 0) { + return nil, ErrManagementDenied + } + config.ConnectTimeout = 3 * time.Second + config.DefaultQueryExecMode = pgx.QueryExecModeExec + return &pgAllocationAuthorizer{config: config, schema: input.AuthoritySchema, nodeID: input.NodeID}, nil +} + +func (a *pgAllocationAuthorizer) Authorize(ctx context.Context, intent AllocationIntent, dispatch func(string) error) error { + if intent.NodeID != a.nodeID || !validAllocation(intent, "1") { + return ErrManagementDenied + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + connection, err := pgx.ConnectConfig(ctx, a.config.Copy()) + if err != nil { + return ErrManagementUnavailable + } + defer func() { + closed, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + connection.Close(closed) + }() + var directRole bool + if err = connection.QueryRow(ctx, "SELECT current_user=session_user").Scan(&directRole); err != nil || !directRole { + return ErrManagementDenied + } + q := pgx.Identifier{a.schema}.Sanitize() + var nodeID, applicationSchema string + var version int + if err = connection.QueryRow(ctx, "SELECT node_id,application_schema,protocol_version FROM "+q+".reader_identity()").Scan(&nodeID, &applicationSchema, &version); err != nil { + return ErrManagementUnavailable + } + expectedSchema := sha256.Sum256([]byte(applicationSchema)) + if version != 1 || nodeID != a.nodeID || a.schema != "vm2api_node_"+hex.EncodeToString(expectedSchema[:16]) { + return ErrManagementDenied + } + tx, err := connection.BeginTx(ctx, pgx.TxOptions{AccessMode: pgx.ReadOnly}) + if err != nil { + return ErrManagementUnavailable + } + defer func() { + rollback, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + tx.Rollback(rollback) + }() + if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))", + "vm2api:node-management:"+a.schema, "node:"+a.nodeID+":vm:"+intent.VMID); err != nil { + return ErrManagementUnavailable + } + var epoch, digest string + err = tx.QueryRow(ctx, "SELECT epoch::text,intent_digest FROM "+q+".authorized_allocations WHERE allocation_id=$1 AND node_id=$2 AND vm_id=$3 AND operation_id=$4 AND owner_id=$5 AND action='allocate'", + intent.AllocationID, intent.NodeID, intent.VMID, intent.OperationID, intent.OwnerID).Scan(&epoch, &digest) + if err == pgx.ErrNoRows { + return ErrManagementDenied + } + if err != nil { + return ErrManagementUnavailable + } + if !validAllocation(intent, epoch) || digest != intent.digest() { + return ErrManagementDenied + } + if ctx.Err() != nil { + return ErrManagementUnavailable + } + // Only bounded local journal I/O belongs in dispatch. Never Docker or an + // outbound RPC. A failed COMMIT cannot acknowledge a local receipt. + if err = dispatch(epoch); err != nil { + return err + } + if err = tx.Commit(ctx); err != nil { + return ErrManagementUnavailable + } + return nil +} diff --git a/worker/internal/slotgate/slotgate.go b/worker/internal/slotgate/slotgate.go index ebb038f92..c7c510cc2 100644 --- a/worker/internal/slotgate/slotgate.go +++ b/worker/internal/slotgate/slotgate.go @@ -36,6 +36,7 @@ type Config struct { AuthorityDir string `json:"authority_dir,omitempty"` DockerSocket string `json:"docker_socket,omitempty"` Launch *DockerLaunchPolicy `json:"launch,omitempty"` + Management *PGManagementConfig `json:"management,omitempty"` } // Preface: "KSG1 \n"; reply "OK\n" or "ERR \n". @@ -60,9 +61,16 @@ type Server struct { wg sync.WaitGroup authority *Authority launcher *dockerLauncher + allocator *allocationManager + // Internal injection for trusted adapters/tests. Config alone never grants + // unimplemented network mutations to a holder of the forwarding token. + authorizeManagement func(context.Context, gate2Request, func() error) error } func New(cfg Config) (*Server, error) { + if !cfg.Fenced && (cfg.Management != nil || cfg.AuthorityDir != "") { + return nil, ErrManagementDenied + } if strings.TrimSpace(cfg.Listen) == "" { return nil, errors.New("listen required") } @@ -106,6 +114,10 @@ func New(cfg Config) (*Server, error) { if err != nil { return nil, err } + if err = srv.authority.bindManagement(cfg.Management); err != nil { + srv.authority.Close() + return nil, err + } if cfg.Launch != nil { srv.launcher, err = newDockerLauncher(socket, cfg.Root, boot, srv.authority, *cfg.Launch) if err != nil { @@ -113,6 +125,14 @@ func New(cfg Config) (*Server, error) { return nil, err } } + if cfg.Management != nil { + authorizer, authErr := newPGAllocationAuthorizer(*cfg.Management) + if authErr != nil { + srv.authority.Close() + return nil, authErr + } + srv.allocator = &allocationManager{authority: srv.authority, nodeID: cfg.Management.NodeID, authorize: authorizer.Authorize} + } } return srv, nil } From a37c4c4951cf033d6d0f5cbbfa7d4e57dc9a0034 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:50:56 +0800 Subject: [PATCH 331/473] test: exercise protocol publication waiting and unknown occupancy --- test/fixtures/protocol-publication.mjs | 94 +++++++++++++++++++ .../failover-preparation-unknown.test.mjs | 14 ++- test/unit/protocol-publication.test.mjs | 24 +++++ 3 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/protocol-publication.mjs create mode 100644 test/unit/protocol-publication.test.mjs diff --git a/test/fixtures/protocol-publication.mjs b/test/fixtures/protocol-publication.mjs new file mode 100644 index 000000000..493cc018b --- /dev/null +++ b/test/fixtures/protocol-publication.mjs @@ -0,0 +1,94 @@ +import { test, mock } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Readable } from 'node:stream' +import { EventEmitter } from 'node:events' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { readBody } from '../../src/lib/http/respond.mjs' + +const kind = process.env.VM2API_PROTOCOL_PUBLICATION_CASE +const telemetry = await import('../../src/lib/vm/worker-telemetry.mjs') +const supervisor = await import('../../src/lib/transport/rust-kernel-supervisor.mjs') +let enter +let resolve +let reject +const entered = new Promise((done) => { enter = done }) +const publication = new Promise((done, fail) => { resolve = done; reject = fail }) +mock.module(new URL('../../src/lib/vm/worker-telemetry.mjs', import.meta.url), { + namedExports: { ...telemetry, touchTelemetrySession() { enter(); return { wrote: true, publication } } }, +}) +mock.module(new URL('../../src/lib/transport/rust-kernel-supervisor.mjs', import.meta.url), { + namedExports: { + ...supervisor, + syncClaudeKernelConfigsFromFile() { + if (kind === 'sync') { enter(); return { publication } } + return { updated: 0 } + }, + }, +}) +const { createHandleProtocol } = await import('../../src/lib/protocol/handle-protocol.mjs') + +test(`protocol waits for ${kind} publication and preserves unknown activity`, async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-protocol-publication-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const db = getDb() + const store = new ApiKeyStore({ db, sharedAdmission: true }) + const key = await store.create({ max_concurrency: 1 }) + let runnerCalls = 0 + let marked = 0 + let finished = false + const states = [] + const handler = createHandleProtocol({ + cfg: { + limits: { max_body_bytes: 1 << 20 }, rewrite: { enabled: false }, + intercept: { rules: [] }, distill: { enabled: false }, paths: { project: root, data: root }, + }, + json(res, status, body) { res.statusCode = status; res.body = body }, + readBody, + requireAuth(req) { req.apiKeyRecord = key; req.apiKeyKind = 'managed'; return true }, + requestLog: { start: () => ({ request_id: 'publication-test', t0: Date.now() }) }, + stats: { errors: 0, requests: 0, by_route: {} }, + groupsRepo: { rateMultiplier: () => 1 }, + apiKeyStore: store, + stickyRouter: { resolve: () => null }, + accountQuota: { ingestHeaders() {} }, + failoverRunner: { + async run({ applyAttempt, canonicalBody }) { + runnerCalls++ + if (kind === 'touch') await applyAttempt(canonicalBody, { + vmId: 'publication-vm', accountId: 'publication-account', vm: {}, exec: {}, + markUncertain: async () => { marked++ }, + }) + return { ok: false, status: 502, body: { type: 'error', error: { type: 'api_error', message: 'synthetic' } } } + }, + }, + routingConfig: { compatibility: { claude_code_only: false } }, + onExecutionState: async (_req, state) => states.push(state), + }) + const req = Readable.from([Buffer.from(JSON.stringify({ model: 'claude-sonnet-4-6', max_tokens: 20, messages: [{ role: 'user', content: 'synthetic' }] }))]) + req.headers = { 'user-agent': 'claude-cli/2.1.0 (external, cli)', 'anthropic-version': '2023-06-01' } + const res = new EventEmitter() + res.setHeader = () => {} + const request = handler.handleProtocol(req, res, 'anthropic.messages', '/v1/messages') + request.then(() => { finished = true }, () => { finished = true }) + await entered + await new Promise((done) => setImmediate(done)) + assert.equal(finished, false) + if (kind === 'sync') { + assert.equal(runnerCalls, 0) + resolve() + await request + assert.equal(runnerCalls, 1) + } else { + reject(Object.assign(new Error('synthetic publication failed'), { code: 'sftp_failed', executionUncertain: true })) + await assert.rejects(request, /synthetic publication failed/) + assert.equal(marked, 1) + assert.ok(states.some((state) => state.started === true && state.ended === false && !state.requestId)) + const row = await db.prepare('SELECT released_at FROM api_key_admissions WHERE key_id=$1').get(key.id) + assert.equal(row.released_at, null) + assert.equal((await store.reserve(key)).code, 'api_key_concurrency_limit') + } +}) diff --git a/test/unit/failover-preparation-unknown.test.mjs b/test/unit/failover-preparation-unknown.test.mjs index 3f1f314f0..77771760f 100644 --- a/test/unit/failover-preparation-unknown.test.mjs +++ b/test/unit/failover-preparation-unknown.test.mjs @@ -17,8 +17,13 @@ test('an unknown remote publication during preparation stops failover and preser accountId: 'publication-account', vmId: 'publication-vm', bindExecution: async () => {}, - markUncertain: async () => { marked++; uncertain = true }, - release: async () => { if (!uncertain) released++ }, + markUncertain: async () => { + marked++ + uncertain = true + }, + release: async () => { + if (!uncertain) released++ + }, } }, markCooldown() {}, @@ -35,7 +40,10 @@ test('an unknown remote publication during preparation stops failover and preser executionUncertain: true, }) }, - callAttempt: async () => { calls++; return { ok: false } }, + callAttempt: async () => { + calls++ + return { ok: false } + }, }) assert.equal(selections, 1, 'unknown preparation must not select another VM') assert.equal(marked, 1) diff --git a/test/unit/protocol-publication.test.mjs b/test/unit/protocol-publication.test.mjs new file mode 100644 index 000000000..dba307f1a --- /dev/null +++ b/test/unit/protocol-publication.test.mjs @@ -0,0 +1,24 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const fixture = fileURLToPath(new URL('../fixtures/protocol-publication.mjs', import.meta.url)) +const setup = fileURLToPath(new URL('../pg-setup.mjs', import.meta.url)) +for (const kind of ['sync', 'touch']) { + test(`protocol awaits its own ${kind} publication`, () => { + const env = { ...process.env, VM2API_PROTOCOL_PUBLICATION_CASE: kind } + delete env.NODE_TEST_CONTEXT + const result = spawnSync( + process.execPath, + ['--experimental-test-module-mocks', '--import', setup, '--test', fixture], + { + env, + encoding: 'utf8', + timeout: 20_000, + }, + ) + assert.equal(result.status, 0, result.stdout + result.stderr) + assert.match(result.stdout, /# tests 1\b/) + }) +} From 63e641b585691e0edd8ed94d4e7096322e4a0dc4 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:50:56 +0800 Subject: [PATCH 332/473] fix: await inference publications and retain uncertain preparation --- src/lib/pool/failover-runner.mjs | 4 ++++ src/lib/protocol/handle-protocol.mjs | 24 +++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index da903ecc0..0b3dc9af3 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -1164,6 +1164,10 @@ export class FailoverRunner { } budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) } catch (error) { + if (error?.executionUncertain === true && selected.bindExecution) { + if (!unknownExecution) await selected.markUncertain() + unknownExecution = true + } // A submitted job can outlive its transport. Never replay it or free // its shared capacity without proof that the execution ended. if (selected.bindExecution && callEntered) { diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 71556713c..bc2aee441 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -1079,7 +1079,14 @@ export function createHandleProtocol(deps) { let preserveCacheBreakpoints = false const cacheBreakpoints = cacheBreakpointsFromRoutingFile(routingConfigPath) const openaiCompat = String(protocol || '').startsWith('openai.') - syncClaudeKernelConfigsFromFile(cfg.paths?.project, routingConfigPath) + try { + await syncClaudeKernelConfigsFromFile(cfg.paths?.project, routingConfigPath)?.publication + } catch (error) { + admission.executionStarted = true + if (admission.lease?.shared) admission.lease.executionUncertain = true + await observeExecution(req, admission, { started: true, ended: false, reason: 'remote_publication_unknown' }) + throw error + } const personaMode = personaModeFromRoutingFile(routingConfigPath) if (!requestContext && !officialClient && !officialTraffic) { ctx.body = ensureClaudeWebSearch(ctx.body, { @@ -1347,8 +1354,19 @@ export function createHandleProtocol(deps) { } } try { - touchTelemetrySession(cfg.paths.project, selected.vmId, selected.exec?.vm) - } catch {} + await touchTelemetrySession(cfg.paths.project, selected.vmId, selected.exec?.vm)?.publication + } catch (error) { + error.executionUncertain = true + admission.executionStarted = true + if (admission.lease?.shared) admission.lease.executionUncertain = true + await selected.markUncertain?.() + await observeExecution(req, admission, { + started: true, + ended: false, + reason: 'remote_publication_unknown', + }) + throw error + } const identity = loadVmIdentity(selected.exec) const attemptStartedAt = extra.attemptStartedAt ?? Date.now() // The runner snapshots after this session's queue turn, before VM prebinding. From 370111b166bd49889dd1267d51cd47534c8d2a98 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:54:22 +0800 Subject: [PATCH 333/473] test: reproduce completed bootstrap failure retaining package claims --- test/unit/panel-proxy-vm-mutations.test.mjs | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/unit/panel-proxy-vm-mutations.test.mjs b/test/unit/panel-proxy-vm-mutations.test.mjs index 1140fb803..087e102b4 100644 --- a/test/unit/panel-proxy-vm-mutations.test.mjs +++ b/test/unit/panel-proxy-vm-mutations.test.mjs @@ -349,3 +349,36 @@ for (const auxiliary of ['profile', 'models', 'identity']) { assert.equal((await new SharedProxyOperations(f.peer).tryAcquire()).ok, false) }) } + +test('a completed bootstrap business failure releases package VM and proxy claims', async (t) => { + const f = await fixture(t) + const body = buildVmPackage({ + vm: { id: 'vm-new', timezone: 'UTC', claude: { mode: 'oauth' } }, + proxy: f.pool.getProxyByIdWithAuth(f.proxyId), + claudeCred: { mode: 'oauth', access_token: 'synthetic-access', refresh_token: 'synthetic-refresh' }, + }) + const h = f.handler(body, { + officialCcStatsHandler: () => async () => {}, + accountQuota: { + snapshot: () => ({ accounts: [] }), + ensure() {}, + repo: { get: () => ({ unified: {} }), observe: async () => {} }, + beginObservation: async () => 'synthetic', + ingestOAuthUsage() {}, + setAccountTier() {}, + flush: async () => {}, + }, + vmMutationOps: { + startSlotReady: async () => ({ ok: true }), + runOfficialCcBootstrap: async () => ({ ok: false, attemptFinished: true, vmOperationUncertain: false }), + }, + }) + await h.request('POST', '/api/panel/vms/package') + assert.equal(h.response.status, 201, JSON.stringify(h.response.body)) + assert.equal(h.response.body.data.official_cc_bootstrap.ok, false) + const nextVm = await f.otherOperations.tryAcquire({ vmId: 'vm-new' }) + if (nextVm.ok) await nextVm.release() + const nextProxy = await new SharedProxyOperations(f.peer).tryAcquire() + if (nextProxy.ok) await nextProxy.release() + assert.deepEqual({ vm: nextVm.ok, proxy: nextProxy.ok }, { vm: true, proxy: true }) +}) From a9961b9ef8afc2eac22a08016193bbc0019310a6 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:54:37 +0800 Subject: [PATCH 334/473] test: verify publication uncertainty against durable admissions --- test/fixtures/protocol-publication.mjs | 8 +++++++ .../failover-preparation-unknown.test.mjs | 23 ++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/test/fixtures/protocol-publication.mjs b/test/fixtures/protocol-publication.mjs index 493cc018b..e35badbe8 100644 --- a/test/fixtures/protocol-publication.mjs +++ b/test/fixtures/protocol-publication.mjs @@ -40,6 +40,8 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as let runnerCalls = 0 let marked = 0 let finished = false + let noteRunner + const runnerStarted = new Promise((done) => { noteRunner = done }) const states = [] const handler = createHandleProtocol({ cfg: { @@ -58,6 +60,7 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as failoverRunner: { async run({ applyAttempt, canonicalBody }) { runnerCalls++ + noteRunner() if (kind === 'touch') await applyAttempt(canonicalBody, { vmId: 'publication-vm', accountId: 'publication-account', vm: {}, exec: {}, markUncertain: async () => { marked++ }, @@ -78,6 +81,11 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as await new Promise((done) => setImmediate(done)) assert.equal(finished, false) if (kind === 'sync') { + const phase = await Promise.race([ + runnerStarted.then(() => 'started'), + new Promise((done) => setTimeout(() => done('blocked'), 200)), + ]) + assert.equal(phase, 'blocked', 'model scheduling waits for publication completion') assert.equal(runnerCalls, 0) resolve() await request diff --git a/test/unit/failover-preparation-unknown.test.mjs b/test/unit/failover-preparation-unknown.test.mjs index 77771760f..539efa951 100644 --- a/test/unit/failover-preparation-unknown.test.mjs +++ b/test/unit/failover-preparation-unknown.test.mjs @@ -1,28 +1,40 @@ import test from 'node:test' import assert from 'node:assert/strict' import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' test('an unknown remote publication during preparation stops failover and preserves the admission', async () => { let selections = 0 let marked = 0 let calls = 0 let released = 0 - let uncertain = false + const db = getDb() + const lease = await new SharedExecutionAdmissions(db).tryReserve({ + accountId: 'publication-account', + vmId: 'publication-vm', + accountLimit: 1, + vmLimit: 1, + fableLimit: 0, + rpmLimit: 0, + }) + assert.equal(lease.ok, true) const scheduler = { async selectAndReserve() { selections++ if (selections > 1) return { ok: false, reason: 'no_eligible_accounts' } return { + ...lease, ok: true, accountId: 'publication-account', vmId: 'publication-vm', bindExecution: async () => {}, markUncertain: async () => { marked++ - uncertain = true + await lease.markUncertain() }, release: async () => { - if (!uncertain) released++ + if ((await lease.release()).released) released++ }, } }, @@ -50,4 +62,9 @@ test('an unknown remote publication during preparation stops failover and preser assert.equal(calls, 0) assert.equal(released, 0) assert.equal(result.executionUncertain, true) + const row = await db + .prepare('SELECT released_at, uncertain_at FROM execution_admissions WHERE id=$1') + .get(lease.admissionId) + assert.equal(row.released_at, null) + assert.ok(row.uncertain_at) }) From 4c1c89f7a64e2864703b854a85a077c52bca9640 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:54:37 +0800 Subject: [PATCH 335/473] ci: download Go modules before offline protocol fixtures --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 72a137561..8302700c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,6 +42,9 @@ jobs: with: go-version: '1.25.x' cache-dependency-path: worker/go.sum + - name: download worker modules for offline Go protocol fixtures + working-directory: worker + run: go mod download - run: npm ci - name: PostgreSQL 17 backup clients run: | From da7700bdd431cdfeea7e3f620367eec524c23c94 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:55:08 +0800 Subject: [PATCH 336/473] fix: release completed package bootstrap failures without losing business result --- src/lib/admin/panel-routes.mjs | 5 +- .../panel-credential-refresh-claims.test.mjs | 74 ++++++++++++++++++- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 88190ab44..fdb9b8c6e 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -886,10 +886,11 @@ export function createPanelHandler(ctx) { credentialMode: credentialModeOfVm(vm), } if (isKubernetesMode()) { - const completed = await vmEffect(operation, async () => { + const execution = await vmEffect(operation, async () => { const result = await (ctx.vmMutationOps?.runOfficialCcBootstrap || runOfficialCcBootstrap)(options) - return officialCcBootstrapFinished(result, options) ? result : { ...result, vmOperationUncertain: true } + return { ok: officialCcBootstrapFinished(result, options), value: result } }) + const completed = execution.value bootstrap = { scheduled: true, vmId: id, completed: true, ok: completed?.ok === true } } else bootstrap = scheduleOfficialCcBootstrap(options) } diff --git a/test/unit/panel-credential-refresh-claims.test.mjs b/test/unit/panel-credential-refresh-claims.test.mjs index 84d2fff9b..678cd99e5 100644 --- a/test/unit/panel-credential-refresh-claims.test.mjs +++ b/test/unit/panel-credential-refresh-claims.test.mjs @@ -4,13 +4,14 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import https from 'node:https' -import { getDb } from '../../src/lib/db/database.mjs' +import { getDb, quoteIdentifier } from '../../src/lib/db/database.mjs' import { createRepositoryTestDatabase } from '../repository-database.mjs' import { withDatabaseActivities } from '../maintenance-fixture.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { writeCodexAccounts, readCodexAccounts, codexCredentialPath } from '../../src/lib/vm/codex-slot.mjs' +import { refreshSharedCodexCredential } from '../../src/lib/oauth/shared-codex-refresh.mjs' async function fixture(t, { codex = false } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-refresh-claims-')) @@ -211,3 +212,74 @@ test('Claude complete rejection can release its manual operation', async (t) => assert.equal(next.ok, true) await next.release() }) + +test('manual Codex refresh never resubmits an account with a pending or unknown refresh on another VM', async (t) => { + const f = await fixture(t, { codex: true }) + const winnerVm = { ...f.vm, id: 'vm-winner' } + const thirdVm = { ...f.vm, id: 'vm-third' } + for (const vm of [winnerVm, thirdVm]) { + fs.mkdirSync(path.join(f.root, 'vms', vm.id)) + fs.writeFileSync(path.join(f.root, 'vms', vm.id + '.json'), JSON.stringify(vm)) + writeCodexAccounts(f.root, vm.id, readCodexAccounts(f.root, f.vm.id)) + } + let entered + const started = new Promise((resolve) => { + entered = resolve + }) + let finish + const pending = new Promise((resolve) => { + finish = resolve + }) + let winnerCalls = 0 + const winner = withDatabaseActivities([f.peer], () => + refreshSharedCodexCredential({ + projectRoot: f.root, + vm: winnerVm, + db: f.peer, + refresh: async () => { + winnerCalls++ + entered() + await pending + return { ok: false, attemptFinished: false, error: 'synthetic_disconnect' } + }, + }), + ) + winner.catch(() => {}) + await started + let manualCalls = 0 + const extra = { + vmMutationOps: { + refreshCodexAccessToken: async () => { + manualCalls++ + throw new Error('must not resubmit the same account refresh') + }, + }, + } + const h = f.handler(extra) + const request = h.request('/api/panel/vms/vm-synthetic/oauth/refresh') + request.catch(() => {}) + try { + const deadline = Date.now() + 2000 + while (true) { + const row = await f.peer.pool.query( + `SELECT count(*) AS n FROM ${quoteIdentifier(f.peer.schema)}.vm_operations WHERE vm_id=$1 AND released_at IS NULL`, + [f.vm.id], + ) + if (Number(row.rows[0].n) === 1) break + assert.ok(Date.now() < deadline, 'manual route did not enter its VM operation') + await new Promise((resolve) => setTimeout(resolve, 5)) + } + assert.equal(manualCalls, 0) + } finally { + finish() + await winner + await request + } + assert.equal(h.response.status, 502) + const third = f.handler(extra) + await third.request('/api/panel/vms/vm-third/oauth/refresh') + assert.equal(third.response.status, 502) + assert.equal(winnerCalls, 1) + assert.equal(manualCalls, 0) + assert.equal(f.calls(), 0) +}) From 04f92c90361d5ae3e446e64fd95e782f2f73769a Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:55:45 +0800 Subject: [PATCH 337/473] test: retain unknown OAuth refresh attempts before retry or later observations --- test/unit/credential-refresh-proof.test.mjs | 7 ++ test/unit/slot-oauth-shared.test.mjs | 22 ++++ worker/internal/oauth/refresher_test.go | 106 ++++++++++++++++++++ worker/internal/oauthcmd/oauthcmd_test.go | 46 +++++++++ 4 files changed, 181 insertions(+) diff --git a/test/unit/credential-refresh-proof.test.mjs b/test/unit/credential-refresh-proof.test.mjs index 3d55b2ea6..c0c34ddc4 100644 --- a/test/unit/credential-refresh-proof.test.mjs +++ b/test/unit/credential-refresh-proof.test.mjs @@ -21,6 +21,13 @@ test('a complete successful worker command proves its credential attempt finishe assert.equal(result.attemptFinished, true) }) +test('worker refresh uncertainty takes priority over a final complete rejection', async () => { + const result = await runSlotOauth(exec, 'refresh', { + runDocker: worker({ ok: false, status: 401, refresh_unknown: true, body: { error: { code: 'refresh_failed' } } }), + }) + assert.equal(result.attemptFinished, false) +}) + test('worker timeout, nonzero exit and reported upstream truncation cannot prove completion', async () => { for (const [payload, extra] of [ [{ ok: true, status: 200 }, { timed_out: true }], diff --git a/test/unit/slot-oauth-shared.test.mjs b/test/unit/slot-oauth-shared.test.mjs index bf7e251ff..7a126f787 100644 --- a/test/unit/slot-oauth-shared.test.mjs +++ b/test/unit/slot-oauth-shared.test.mjs @@ -112,6 +112,28 @@ test('a slot refresh timeout retains its shared operation and task', async (t) = assert.ok(rows.rows[0].uncertain_at) }) +test('worker refresh uncertainty retains shared claims despite its final rejection status', async (t) => { + const f = await fixture(t) + const result = await runSlotOauth(f.exec, 'refresh', { + runDocker: async () => ({ + code: 0, + stdout: JSON.stringify({ + ok: false, + status: 400, + refresh_unknown: true, + body: { error: { code: 'refresh_failed' } }, + }), + }), + }) + assert.equal(result.attemptFinished, false) + const operation = await f.run(() => f.db.prepare('SELECT released_at,uncertain_at FROM vm_operations').get()) + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + const task = await f.run(() => f.db.prepare('SELECT finished_at,unknown_at FROM background_task_runs').get()) + assert.equal(task.finished_at, null) + assert.ok(task.unknown_at) +}) + test('ensure publishes its VM projection before the shared refresh and VM operation finish', async (t) => { const f = await fixture(t) let publications = 0 diff --git a/worker/internal/oauth/refresher_test.go b/worker/internal/oauth/refresher_test.go index 54ffd1453..d27f1f2d8 100644 --- a/worker/internal/oauth/refresher_test.go +++ b/worker/internal/oauth/refresher_test.go @@ -3,6 +3,8 @@ package oauth import ( "context" "encoding/json" + "fmt" + "io" "net/http" "net/http/httptest" "path/filepath" @@ -15,6 +17,110 @@ import ( "github.com/dofastted/kin-gateway/worker/internal/credential" ) +func TestRefreshNeverRetriesAnAcceptedUnknownAttempt(t *testing.T) { + for _, headers := range []bool{false, true} { + t.Run(fmt.Sprintf("response_headers_%v", headers), func(t *testing.T) { + var calls atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + if calls.Add(1) > 1 { + writer.WriteHeader(http.StatusBadRequest) + _, _ = writer.Write([]byte(`{"error":"invalid_grant"}`)) + return + } + conn, buffered, err := writer.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + if headers { + _, _ = buffered.WriteString("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 500\r\n\r\n{\"access_token\":\"synthetic-lost") + _ = buffered.Flush() + } + _ = conn.Close() + })) + defer upstream.Close() + store := credential.NewStore(filepath.Join(t.TempDir(), "credentials.json")) + original, err := store.Save(credential.Credential{AccessToken: "synthetic-old-at", RefreshToken: "synthetic-old-rt", ExpiresAt: 1}, nil) + if err != nil { + t.Fatal(err) + } + refresher := &Refresher{Store: store, Client: upstream.Client(), TokenURL: upstream.URL, MaxTries: 3} + if _, err = refresher.Ensure(context.Background(), true); err == nil { + t.Fatal("accepted truncated refresh must remain unknown") + } + if calls.Load() != 1 { + t.Fatalf("refresh calls = %d, want 1 after accepted unknown attempt", calls.Load()) + } + current, err := store.Status() + if err != nil { + t.Fatal(err) + } + if current.RefreshToken != original.RefreshToken || current.Generation != original.Generation { + t.Fatal("unknown refresh changed stored credential") + } + }) + } +} + +func TestRefreshMayRetryADefinitivelyCompleteRejection(t *testing.T) { + var calls atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "application/json") + if calls.Add(1) == 1 { + writer.WriteHeader(http.StatusServiceUnavailable) + _, _ = writer.Write([]byte(`{"error":"temporarily_unavailable"}`)) + return + } + _, _ = writer.Write([]byte(`{"access_token":"synthetic-new-at"}`)) + })) + defer upstream.Close() + store := credential.NewStore(filepath.Join(t.TempDir(), "credentials.json")) + if _, err := store.Save(credential.Credential{AccessToken: "synthetic-old-at", RefreshToken: "synthetic-old-rt", ExpiresAt: 1}, nil); err != nil { + t.Fatal(err) + } + refresher := &Refresher{Store: store, Client: upstream.Client(), TokenURL: upstream.URL, MaxTries: 2} + if _, err := refresher.Ensure(context.Background(), true); err != nil { + t.Fatal(err) + } + if calls.Load() != 2 { + t.Fatalf("refresh calls = %d, want 2 after complete rejection", calls.Load()) + } + current, err := store.Status() + if err != nil { + t.Fatal(err) + } + if current.RefreshToken != "synthetic-old-rt" { + t.Fatal("an access-only response must retain the existing refresh token") + } +} + +func TestUnknownRefreshCannotBecomeSuccessFromNewFileObservation(t *testing.T) { + store := credential.NewStore(filepath.Join(t.TempDir(), "credentials.json")) + original, err := store.Save(credential.Credential{AccessToken: "synthetic-old-at", RefreshToken: "synthetic-old-rt", ExpiresAt: 1}, nil) + if err != nil { + t.Fatal(err) + } + upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if _, err := store.Save(credential.Credential{AccessToken: "synthetic-observed-at", RefreshToken: "synthetic-observed-rt", ExpiresAt: time.Now().Add(time.Hour).UnixMilli(), Generation: original.Generation + 1}, nil); err != nil { + t.Error(err) + } + conn, buffered, err := writer.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _, _ = buffered.WriteString("HTTP/1.1 200 OK\r\nContent-Length: 500\r\n\r\n{\"access_token\":\"synthetic-lost") + _ = buffered.Flush() + _ = conn.Close() + })) + defer upstream.Close() + refresher := &Refresher{Store: store, Client: upstream.Client(), TokenURL: upstream.URL, MaxTries: 1} + if _, err = refresher.Ensure(context.Background(), true); err == nil { + t.Fatal("later credential observation replaced an unknown refresh result with success") + } +} + func TestEnsureDeduplicatesConcurrentRefresh(t *testing.T) { var calls atomic.Int32 upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { diff --git a/worker/internal/oauthcmd/oauthcmd_test.go b/worker/internal/oauthcmd/oauthcmd_test.go index 1431925b7..452533ff1 100644 --- a/worker/internal/oauthcmd/oauthcmd_test.go +++ b/worker/internal/oauthcmd/oauthcmd_test.go @@ -3,17 +3,63 @@ package oauthcmd import ( "bytes" "encoding/json" + "io" "net/http" "net/http/httptest" "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" "github.com/dofastted/kin-gateway/worker/internal/credential" ) +func TestUnknownRefreshMetadataSurvivesExplicitAndImplicitEnsure(t *testing.T) { + for _, op := range []string{"refresh", "models"} { + t.Run(op, func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + _, _ = io.Copy(io.Discard, request.Body) + if calls.Add(1) > 1 { + writer.WriteHeader(http.StatusUnauthorized) + _, _ = writer.Write([]byte(`{"error":"invalid_grant"}`)) + return + } + conn, buffered, err := writer.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _, _ = buffered.WriteString("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 500\r\n\r\n{\"access_token\":\"synthetic-lost") + _ = buffered.Flush() + _ = conn.Close() + })) + defer server.Close() + configPath, credPath := testConfig(t, server.URL, server.URL, nil) + saveCredential(t, credPath, credential.Credential{AccessToken: "synthetic-old-at", RefreshToken: "synthetic-old-rt", ExpiresAt: 1}) + var output bytes.Buffer + if code := Run([]string{op, "--config", configPath}, nil, &output); code != 0 { + t.Fatalf("exit code = %d", code) + } + var envelope map[string]any + if err := json.Unmarshal(output.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope["refresh_unknown"] != true { + t.Fatalf("refresh_unknown = %v, want true", envelope["refresh_unknown"]) + } + if calls.Load() != 1 { + t.Fatalf("refresh calls = %d, want 1", calls.Load()) + } + if strings.Contains(output.String(), "synthetic-old") || strings.Contains(output.String(), "synthetic-lost") { + t.Fatal("output leaked a synthetic token") + } + }) + } +} + func TestUsageSuccessForwardsUserAgentAndBeta(t *testing.T) { var gotUA, gotBeta string server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { From 08a182870b6334691a114c7b93a0f07032588ff7 Mon Sep 17 00:00:00 2001 From: liubing Date: Wed, 21 Oct 2026 23:58:06 +0800 Subject: [PATCH 338/473] fix: stop retrying OAuth refresh after an unknown accepted attempt --- src/lib/transport/slot-oauth.mjs | 1 + worker/internal/oauth/refresher.go | 42 ++++++++++++++++++++++------ worker/internal/oauthcmd/oauthcmd.go | 21 +++++++++----- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/lib/transport/slot-oauth.mjs b/src/lib/transport/slot-oauth.mjs index f58889820..679c95ea5 100644 --- a/src/lib/transport/slot-oauth.mjs +++ b/src/lib/transport/slot-oauth.mjs @@ -235,6 +235,7 @@ async function runSlotOauthRaw( via: 'slot-worker', attemptFinished: result?.code === 0 && + payload.refresh_unknown !== true && payload.body?.error?.code !== 'upstream_transport_error' && ((payload.ok && status >= 200 && status < 300) || status >= 400 || diff --git a/worker/internal/oauth/refresher.go b/worker/internal/oauth/refresher.go index fbe9dad22..a34f9d7bd 100644 --- a/worker/internal/oauth/refresher.go +++ b/worker/internal/oauth/refresher.go @@ -51,6 +51,19 @@ type RefreshError struct { Code string Message string Retryable bool + Unknown bool + Cause error +} + +func (e *RefreshError) Unwrap() error { return e.Cause } + +func IsRefreshUnknown(err error) bool { + var refreshErr *RefreshError + return errors.As(err, &refreshErr) && refreshErr.Unknown +} + +func unknownRefresh(status int, cause error) *RefreshError { + return &RefreshError{Status: status, Code: "refresh_outcome_unknown", Message: "OAuth refresh outcome is unknown", Unknown: true, Cause: cause} } func (e *RefreshError) Error() string { @@ -124,6 +137,11 @@ func (r *Refresher) ensureLocked(ctx context.Context, force bool) (Result, error } refreshed, refreshErr := r.requestWithRetry(ctx, current) if refreshErr != nil { + // Neither cancellation nor a later file observation proves what this + // accepted request did to the rotating refresh token. + if IsRefreshUnknown(refreshErr) { + return refreshErr + } if ctx.Err() != nil { return ctx.Err() } @@ -136,12 +154,12 @@ func (r *Refresher) ensureLocked(ctx context.Context, force bool) (Result, error return refreshErr } if ctx.Err() != nil { - return ctx.Err() + return unknownRefresh(0, ctx.Err()) } refreshed.Generation = current.Generation saved, saveErr := r.Store.Save(refreshed, document) if saveErr != nil { - return saveErr + return unknownRefresh(0, saveErr) } result.Credential = saved result.Refreshed = true @@ -174,7 +192,9 @@ func (r *Refresher) requestWithRetry(ctx context.Context, current credential.Cre } lastErr = err var refreshErr *RefreshError - if errors.As(err, &refreshErr) && !refreshErr.Retryable { + // Retry only a fully read, explicit provider rejection. Replaying a + // transport-unknown request can consume the same rotating token twice. + if !errors.As(err, &refreshErr) || refreshErr.Unknown || !refreshErr.Retryable { break } } @@ -200,15 +220,18 @@ func (r *Refresher) request(ctx context.Context, current credential.Credential) request.Header.Set("User-Agent", "axios/1.13.6") response, err := r.Client.Do(request) if err != nil { - return credential.Credential{}, fmt.Errorf("OAuth refresh transport: %w", err) + return credential.Credential{}, unknownRefresh(0, err) } defer response.Body.Close() - data, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + data, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) if err != nil { - return credential.Credential{}, fmt.Errorf("read OAuth refresh response: %w", err) + return credential.Credential{}, unknownRefresh(response.StatusCode, err) + } + if len(data) > 1<<20 { + return credential.Credential{}, unknownRefresh(response.StatusCode, errors.New("OAuth refresh response exceeds limit")) } var decoded tokenResponse - _ = json.Unmarshal(data, &decoded) + decodeErr := json.Unmarshal(data, &decoded) if response.StatusCode < 200 || response.StatusCode >= 300 { code := strings.TrimSpace(decoded.Error) message := strings.TrimSpace(decoded.Description) @@ -229,8 +252,11 @@ func (r *Refresher) request(ctx context.Context, current credential.Credential) Retryable: retryable, } } + if decodeErr != nil { + return credential.Credential{}, unknownRefresh(response.StatusCode, decodeErr) + } if strings.TrimSpace(decoded.AccessToken) == "" { - return credential.Credential{}, &RefreshError{Status: response.StatusCode, Code: "missing_access_token", Message: "refresh response has no access_token"} + return credential.Credential{}, &RefreshError{Status: response.StatusCode, Code: "missing_access_token", Message: "refresh response has no access_token", Unknown: true} } refreshToken := strings.TrimSpace(decoded.RefreshToken) if refreshToken == "" { diff --git a/worker/internal/oauthcmd/oauthcmd.go b/worker/internal/oauthcmd/oauthcmd.go index 9660c01d0..69d77e8b7 100644 --- a/worker/internal/oauthcmd/oauthcmd.go +++ b/worker/internal/oauthcmd/oauthcmd.go @@ -38,10 +38,11 @@ type resetRedeemInput struct { RequestID string `json:"request_id"` } type responseEnvelope struct { - OK bool `json:"ok"` - Status int `json:"status"` - Body any `json:"body"` - Headers map[string]string `json:"headers"` + OK bool `json:"ok"` + Status int `json:"status"` + Body any `json:"body"` + Headers map[string]string `json:"headers"` + RefreshUnknown bool `json:"refresh_unknown,omitempty"` } type countTokensInput struct { @@ -86,7 +87,7 @@ func Run(args []string, stdin io.Reader, stdout io.Writer) int { if op == "refresh" { result, err := refresher.Ensure(ctx, force) if err != nil { - return writeAndReturn(stdout, errorEnvelope(refreshErrorStatus(err), classifyError(err), errorMessage(classifyError(err))), 0) + return writeAndReturn(stdout, refreshErrorEnvelope(refreshErrorStatus(err), err), 0) } return writeAndReturn(stdout, responseEnvelope{ OK: true, @@ -157,8 +158,7 @@ func Run(args []string, stdin io.Reader, stdout io.Writer) int { response, err = client.Post(ctx, target, payload, headers) } if err != nil { - code := classifyError(err) - return writeAndReturn(stdout, errorEnvelope(0, code, errorMessage(code)), 0) + return writeAndReturn(stdout, refreshErrorEnvelope(0, err), 0) } return writeUpstreamResponse(stdout, response) } @@ -345,6 +345,13 @@ func errorEnvelope(status int, code, message string) responseEnvelope { } } +func refreshErrorEnvelope(status int, err error) responseEnvelope { + code := classifyError(err) + envelope := errorEnvelope(status, code, errorMessage(code)) + envelope.RefreshUnknown = kinoauth.IsRefreshUnknown(err) + return envelope +} + func classifyError(err error) string { if err == nil { return "" From 1f2299c27123c651983fa7b5c3ba0b8398794150 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:09:28 +0800 Subject: [PATCH 339/473] test: reproduce validation retaining VM claims and masking standalone errors --- .../panel-memory-shared-validation.test.mjs | 38 +++++++++++++++++++ test/unit/panel-vm-memory-raise.test.mjs | 11 ++++++ 2 files changed, 49 insertions(+) create mode 100644 test/unit/panel-memory-shared-validation.test.mjs diff --git a/test/unit/panel-memory-shared-validation.test.mjs b/test/unit/panel-memory-shared-validation.test.mjs new file mode 100644 index 000000000..0757114b1 --- /dev/null +++ b/test/unit/panel-memory-shared-validation.test.mjs @@ -0,0 +1,38 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' + +for (const [name, patch, record] of [ + ['invalid size', { memory: '3g' }, {}], + ['KVM', { memory: '1g' }, { runtime_type: 'kvm', runtime: { type: 'kvm', memory: '512m' } }], +]) { + test(`memory validation for ${name} returns its unused shared VM claim`, async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-memory-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + fs.mkdirSync(path.join(root, 'vms')) + fs.writeFileSync(path.join(root, 'vms/memory-vm.json'), JSON.stringify({ + id: 'memory-vm', status: 'running', machine: { memory: '512m' }, + runtime: { type: 'docker', memory: '512m' }, ...record, + })) + const operations = new SharedVmOperations(getDb()) + let status + const handler = createPanelHandler({ + cfg: { paths: { project: root } }, routingConfig: { vm: { memory: '512m' } }, + vmOperations: operations, + requireAuth(req) { req.apiKeyKind = 'master'; req.panelRole = 'admin'; return true }, + readBody: async () => patch, + json(_res, code) { status = code; return true }, + proxyPool: { unbindVm() {} }, accountQuota: { snapshot: () => ({ accounts: [] }) }, + }) + await handler({ method: 'PATCH' }, {}, new URL('http://localhost/api/panel/vms/memory-vm')) + assert.equal(status, 400) + const next = await new SharedVmOperations(getDb()).tryAcquire({ vmId: 'memory-vm' }) + assert.equal(next.ok, true, 'validation before Docker I/O must not leave an unknown operation') + await next.release() + }) +} diff --git a/test/unit/panel-vm-memory-raise.test.mjs b/test/unit/panel-vm-memory-raise.test.mjs index efa58354c..b68f790a7 100644 --- a/test/unit/panel-vm-memory-raise.test.mjs +++ b/test/unit/panel-vm-memory-raise.test.mjs @@ -108,3 +108,14 @@ test('PATCH vm memory rejects invalid values without persisting', async () => { fs.rmSync(root, { recursive: true, force: true }) } }) + +test('standalone memory errors preserve the original failure without a database fixture', async () => { + const root = tmpProject('memory-error') + const failure = new Error('synthetic memory failure') + try { + const { handlePanel } = handler(root, { memory: '1g' }, { + raiseDockerSlotMemory: () => { throw failure }, + }) + await assert.rejects(patchVm(handlePanel, 'memory-error'), (error) => error === failure) + } finally { fs.rmSync(root, { recursive: true, force: true }) } +}) From acf9ac384e7f9a36a9af8bd80f98842af175fc58 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:11:05 +0800 Subject: [PATCH 340/473] test: reproduce scheduled backup failure when discovery is refused during drain --- test/unit/maintenance-backup.test.mjs | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/test/unit/maintenance-backup.test.mjs b/test/unit/maintenance-backup.test.mjs index 5f1bb063e..ed816122e 100644 --- a/test/unit/maintenance-backup.test.mjs +++ b/test/unit/maintenance-backup.test.mjs @@ -4,9 +4,12 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import pg from 'pg' +import { AsyncResource } from 'node:async_hooks' import { getDb, registerActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { BackupService } from '../../src/lib/admin/backup-service.mjs' +import { VmTestScheduleRunner } from '../../src/lib/admin/vm-test-schedule.mjs' +import { SharedBackgroundTasks } from '../../src/lib/core/shared-background-tasks.mjs' async function fixture(t) { const db = getDb() @@ -73,6 +76,66 @@ test('BackupService scheduled checks finish their read activity before claiming assert.equal(await coordinator.isReady(), true) }) +async function discoveryTimer(t, db) { + const schedule = new VmTestScheduleRunner({ + projectRoot: '/missing-maintenance-fixture', + list: () => [], + sharedTasks: new SharedBackgroundTasks(db), + }) + let callback + const timer = t.mock.method(globalThis, 'setInterval', (fn) => { + callback = fn + return { unref() {} } + }) + try { + await schedule.start() + } finally { + timer.mock.restore() + } + const context = new AsyncResource('maintenance-discovery-timer') + t.after(async () => { + await schedule.stop() + context.emitDestroy() + }) + return { schedule, tick: () => context.runInAsyncScope(callback) } +} + +test('a scheduled backup drains when periodic discovery is refused before starting in DRAINING', async (t) => { + const { db, coordinator, service } = await fixture(t) + const { schedule, tick } = await discoveryTimer(t, db) + let discoveries = 0 + const listPlans = schedule.resultRepo.listPlans.bind(schedule.resultRepo) + schedule.resultRepo.listPlans = () => { + discoveries++ + return listPlans() + } + service.onMaintenance(async (enabled) => { + if (!enabled) return + assert.equal((await coordinator.getState()).phase, 'draining') + await tick() + await schedule.stop() + await schedule.flush() + }) + const record = await service.runScheduled() + assert.equal(record.status, 'ok') + assert.equal(discoveries, 0) + assert.equal(await coordinator.isReady(), true) +}) + +test('a discovery failure after admission remains a persistence failure even with the maintenance error code', async (t) => { + const { db } = await fixture(t) + const { schedule, tick } = await discoveryTimer(t, db) + const failure = Object.assign(new Error('synthetic admitted discovery failure'), { code: 'database_maintenance' }) + let discoveries = 0 + schedule.resultRepo.listPlans = async () => { + discoveries++ + throw failure + } + await tick() + assert.equal(discoveries, 1) + await assert.rejects(schedule.flush(), (error) => error === failure) +}) + test('a pre-restore refusal cancels only its own drain and leaves ordinary requests available', async (t) => { const { db, coordinator, service } = await fixture(t) fs.mkdirSync(service.dataDir, { recursive: true }) From f011a2277f63ce0f8e3775612ee436b008b4dde0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:11:56 +0800 Subject: [PATCH 341/473] fix: distinguish pre-admission discovery refusal during backup drain --- src/lib/admin/vm-test-schedule.mjs | 16 ++++++++++++--- test/e2e/maintenance-backup.e2e.test.mjs | 8 +++++--- test/unit/codex-model-mapping.test.mjs | 25 +++++++++++++----------- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/lib/admin/vm-test-schedule.mjs b/src/lib/admin/vm-test-schedule.mjs index 9002a9416..bfdcd7883 100644 --- a/src/lib/admin/vm-test-schedule.mjs +++ b/src/lib/admin/vm-test-schedule.mjs @@ -391,9 +391,19 @@ export class VmTestScheduleRunner { if (error.code !== 'maintenance_activity_closed') throw error } } - return coordinator - ? coordinator.runActivity({ kind }, async () => ({ ended: true, value: await callback() })) - : callback() + if (!coordinator) return callback() + let started = false + try { + return await coordinator.runActivity({ kind }, async () => { + started = true + return { ended: true, value: await callback() } + }) + } catch (error) { + // A timer can reach admission after DRAINING commits but before local stop. + // No discovery SQL started, so this refusal is not a failed persistence operation. + if (kind === 'vm_test_discovery' && !started && error.code === 'database_maintenance') return + throw error + } } _vm(vmId) { diff --git a/test/e2e/maintenance-backup.e2e.test.mjs b/test/e2e/maintenance-backup.e2e.test.mjs index 0c1cd4c5a..2a47bd010 100644 --- a/test/e2e/maintenance-backup.e2e.test.mjs +++ b/test/e2e/maintenance-backup.e2e.test.mjs @@ -46,9 +46,13 @@ test('Kubernetes manual backup transfers its HTTP activity to private maintenanc test('Kubernetes scheduled backup uses private claims and leaves no active business task in its snapshot', async (t) => { const { gw, q } = await fixture(t, true) let records = [] + let history = [] for (let i = 0; i < 100; i++) { records = (await gw.db.pool.query("SELECT id FROM backup_records WHERE kind='scheduled' AND status='ok'")).rows - if (records.length) break + if (records.length) { + history = (await gw.db.pool.query(`SELECT ended_at FROM ${q}.operations WHERE task_key='scheduled-backup'`)).rows + if (history.some((operation) => operation.ended_at)) break + } await new Promise((resolve) => setTimeout(resolve, 100)) } assert.equal(records.length, 1, gw.stderr()) @@ -57,8 +61,6 @@ test('Kubernetes scheduled backup uses private claims and leaves no active busin .n, 0, ) - const history = (await gw.db.pool.query(`SELECT ended_at FROM ${q}.operations WHERE task_key='scheduled-backup'`)) - .rows assert.equal(history.length, 1) assert.ok(history[0].ended_at) }) diff --git a/test/unit/codex-model-mapping.test.mjs b/test/unit/codex-model-mapping.test.mjs index 8afefe526..0e72c7fa7 100644 --- a/test/unit/codex-model-mapping.test.mjs +++ b/test/unit/codex-model-mapping.test.mjs @@ -10,11 +10,12 @@ import { } from '../../src/lib/protocol/codex-model-mapping.mjs' import { normalizeCodexRouting } from '../../src/lib/protocol/codex-route.mjs' import { resetGptModelPolicy, saveGptModelPolicy } from '../../src/lib/protocol/gpt-model-policy.mjs' +import { getDb } from '../../src/lib/db/database.mjs' import fs from 'node:fs' import path from 'node:path' -test('default map rewrites codex-auto-review before platform detect', () => { - resetGptModelPolicy() +test('default map rewrites codex-auto-review before platform detect', async () => { + await resetGptModelPolicy() const rewrite = applyCodexInboundModelRewrite({ model: 'codex-auto-review' }, {}) assert.equal(rewrite.ok, true) assert.equal(rewrite.mapped, true) @@ -30,16 +31,16 @@ test('literal detectInboundPlatform still fail-closes codex-auto-review', () => assert.equal(detectInboundPlatform('codex-auto-review').ok, false) }) -test('custom routing.model_mappings overrides default target', () => { - resetGptModelPolicy() +test('custom routing.model_mappings overrides default target', async () => { + await resetGptModelPolicy() const routing = { codex: { model_mappings: { 'codex-auto-review': 'gpt-6-sol' } } } const rewrite = applyCodexInboundModelRewrite({ model: 'codex-auto-review' }, routing) assert.equal(rewrite.upstreamModel, 'gpt-6-sol') assert.equal(detectInboundPlatform(rewrite.body.model).platform, 'openai') }) -test('empty configured target for codex-auto-review uses fallback sol model', () => { - resetGptModelPolicy() +test('empty configured target for codex-auto-review uses fallback sol model', async () => { + await resetGptModelPolicy() const mappings = normalizeCodexModelMappings({ 'codex-auto-review': '' }) assert.equal(mappings['codex-auto-review'], DEFAULT_CODEX_MODEL_MAPPING_FALLBACK) const rewrite = applyCodexInboundModelRewrite( @@ -49,8 +50,8 @@ test('empty configured target for codex-auto-review uses fallback sol model', () assert.equal(rewrite.upstreamModel, DEFAULT_CODEX_MODEL_MAPPING_FALLBACK) }) -test('invalid non-GPT mapping target returns codex_model_mapping_invalid', () => { - resetGptModelPolicy() +test('invalid non-GPT mapping target returns codex_model_mapping_invalid', async () => { + await resetGptModelPolicy() const rewrite = applyCodexInboundModelRewrite( { model: 'codex-auto-review' }, { codex: { model_mappings: { 'codex-auto-review': 'claude-sonnet-4-6' } } }, @@ -59,13 +60,15 @@ test('invalid non-GPT mapping target returns codex_model_mapping_invalid', () => assert.equal(rewrite.code, 'codex_model_mapping_invalid') }) -test('disabled GPT mapping target returns model_disabled', () => { - resetGptModelPolicy() - saveGptModelPolicy({ +test('disabled GPT mapping target returns model_disabled', async () => { + await resetGptModelPolicy() + await saveGptModelPolicy({ models: { 'gpt-6-luna': { id: 'gpt-6-luna', enabled: false }, }, }) + const stored = (await getDb().query("SELECT value FROM settings WHERE key='gpt_model_policy'")).rows[0] + assert.equal(JSON.parse(stored.value).models['gpt-6-luna'].enabled, false) const check = validateCodexModelMappingTarget('gpt-6-luna') assert.equal(check.ok, false) assert.equal(check.code, 'model_disabled') From 44e48fa6aa9cd69a417d33c063aceac121d47a08 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:14:29 +0800 Subject: [PATCH 342/473] test: reject mock gateway touching host Docker networks --- test/e2e/gateway-boundaries.e2e.test.mjs | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/e2e/gateway-boundaries.e2e.test.mjs diff --git a/test/e2e/gateway-boundaries.e2e.test.mjs b/test/e2e/gateway-boundaries.e2e.test.mjs new file mode 100644 index 000000000..caa47895b --- /dev/null +++ b/test/e2e/gateway-boundaries.e2e.test.mjs @@ -0,0 +1,49 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { startGateway, api } from '../harness.mjs' + +test('the mock gateway never invokes host Docker network or iptables commands', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gateway-boundary-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const bin = path.join(root, 'bin') + fs.mkdirSync(bin) + const trace = path.join(root, 'commands.log') + for (const command of ['docker', 'iptables']) { + const file = path.join(bin, command) + fs.writeFileSync( + file, + `#!/bin/sh\nprintf '${command} %s\\n' "$1" >> "$VM2API_TEST_COMMAND_TRACE"\necho 'test boundary refuses host operations' >&2\nexit 1\n`, + { mode: 0o700 }, + ) + } + const gw = await startGateway({ + project: path.join(root, 'project'), + env: { + PATH: bin + path.delimiter + process.env.PATH, + VM2API_TEST_COMMAND_TRACE: trace, + KIN_BACKUP_DISABLED: '1', + }, + }) + try { + const health = await fetch(gw.baseUrl + '/health') + assert.equal(health.status, 200) + const commands = [trace, gw.runtimeCallsFile].filter(Boolean).flatMap((file) => { + try { + return fs.readFileSync(file, 'utf8').trim().split('\n') + } catch { + return [] + } + }) + assert.deepEqual( + commands.filter((line) => /^(docker network|iptables\b)/.test(line)), + [], + ) + const vms = await api(gw, 'GET', '/admin/vms') + assert.equal(vms.status, 200, vms.text) + } finally { + await gw.stop() + } +}) From f52285d2c56839a1f61024436dbaabe323326748 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:15:48 +0800 Subject: [PATCH 343/473] test: release protocol capacity after confirmed publication failure --- test/fixtures/protocol-publication.mjs | 23 +++++++++++++++++------ test/unit/protocol-publication.test.mjs | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/test/fixtures/protocol-publication.mjs b/test/fixtures/protocol-publication.mjs index e35badbe8..e034112ad 100644 --- a/test/fixtures/protocol-publication.mjs +++ b/test/fixtures/protocol-publication.mjs @@ -61,7 +61,7 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as async run({ applyAttempt, canonicalBody }) { runnerCalls++ noteRunner() - if (kind === 'touch') await applyAttempt(canonicalBody, { + if (kind.startsWith('touch')) await applyAttempt(canonicalBody, { vmId: 'publication-vm', accountId: 'publication-account', vm: {}, exec: {}, markUncertain: async () => { marked++ }, }) @@ -91,12 +91,23 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as await request assert.equal(runnerCalls, 1) } else { - reject(Object.assign(new Error('synthetic publication failed'), { code: 'sftp_failed', executionUncertain: true })) + const finishedPublication = kind === 'touch-finished' + reject(Object.assign(new Error('synthetic publication failed'), { + code: 'sftp_failed', + ...(finishedPublication ? { publicationFinished: true } : { executionUncertain: true }), + })) await assert.rejects(request, /synthetic publication failed/) - assert.equal(marked, 1) - assert.ok(states.some((state) => state.started === true && state.ended === false && !state.requestId)) + assert.equal(marked, finishedPublication ? 0 : 1) + assert.equal(states.some((state) => state.started === true && state.ended === false && !state.requestId), !finishedPublication) const row = await db.prepare('SELECT released_at FROM api_key_admissions WHERE key_id=$1').get(key.id) - assert.equal(row.released_at, null) - assert.equal((await store.reserve(key)).code, 'api_key_concurrency_limit') + if (finishedPublication) { + assert.ok(row.released_at) + const next = await store.reserve(key) + assert.equal(next.ok, true) + await next.release() + } else { + assert.equal(row.released_at, null) + assert.equal((await store.reserve(key)).code, 'api_key_concurrency_limit') + } } }) diff --git a/test/unit/protocol-publication.test.mjs b/test/unit/protocol-publication.test.mjs index dba307f1a..19c33bd36 100644 --- a/test/unit/protocol-publication.test.mjs +++ b/test/unit/protocol-publication.test.mjs @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url' const fixture = fileURLToPath(new URL('../fixtures/protocol-publication.mjs', import.meta.url)) const setup = fileURLToPath(new URL('../pg-setup.mjs', import.meta.url)) -for (const kind of ['sync', 'touch']) { +for (const kind of ['sync', 'touch', 'touch-finished']) { test(`protocol awaits its own ${kind} publication`, () => { const env = { ...process.env, VM2API_PROTOCOL_PUBLICATION_CASE: kind } delete env.NODE_TEST_CONTEXT From 68d2b54dae22457818e18e6f3b2bd7d2b7309d36 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:16:12 +0800 Subject: [PATCH 344/473] fix: return unused VM claims after memory validation failures --- src/lib/admin/panel-routes.mjs | 6 ++-- src/lib/vm/docker-slot-memory.mjs | 2 +- .../panel-memory-shared-validation.test.mjs | 31 ++++++++++++++----- test/unit/panel-vm-memory-raise.test.mjs | 16 +++++++--- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index fdb9b8c6e..118b9748a 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -660,7 +660,8 @@ export function createPanelHandler(ctx) { } catch (error) { if (artifactCallbacks.getStore()) artifactCallbacks.getStore().uncertain = true await operation?.markUncertain?.() - await markCurrentProxyOperationUncertain(getDb()) + if (isKubernetesMode() || ctx.proxyOperations) + await markCurrentProxyOperationUncertain(ctx.proxyOperations?.db || getDb()) throw error } if ( @@ -671,7 +672,8 @@ export function createPanelHandler(ctx) { ) { if (artifactCallbacks.getStore()) artifactCallbacks.getStore().uncertain = true await operation?.markUncertain?.() - await markCurrentProxyOperationUncertain(getDb()) + if (isKubernetesMode() || ctx.proxyOperations) + await markCurrentProxyOperationUncertain(ctx.proxyOperations?.db || getDb()) } return result } diff --git a/src/lib/vm/docker-slot-memory.mjs b/src/lib/vm/docker-slot-memory.mjs index 9608dab84..72c61a3f1 100644 --- a/src/lib/vm/docker-slot-memory.mjs +++ b/src/lib/vm/docker-slot-memory.mjs @@ -179,7 +179,7 @@ export function validateDockerMemoryRaise(vm, requestedMemory, routing) { export function raiseDockerSlotMemory(projectRoot, vm, requestedMemory, { routing, execImpl = defaultExec } = {}) { const validated = validateDockerMemoryRaise(vm, requestedMemory, routing) - if (!validated.ok) return validated + if (!validated.ok) return { ...validated, vmOperationNotStarted: true } const mem = validated.memory const name = containerName(vm.id) diff --git a/test/unit/panel-memory-shared-validation.test.mjs b/test/unit/panel-memory-shared-validation.test.mjs index 0757114b1..ed33bd951 100644 --- a/test/unit/panel-memory-shared-validation.test.mjs +++ b/test/unit/panel-memory-shared-validation.test.mjs @@ -15,19 +15,34 @@ for (const [name, patch, record] of [ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-shared-memory-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) fs.mkdirSync(path.join(root, 'vms')) - fs.writeFileSync(path.join(root, 'vms/memory-vm.json'), JSON.stringify({ - id: 'memory-vm', status: 'running', machine: { memory: '512m' }, - runtime: { type: 'docker', memory: '512m' }, ...record, - })) + fs.writeFileSync( + path.join(root, 'vms/memory-vm.json'), + JSON.stringify({ + id: 'memory-vm', + status: 'running', + machine: { memory: '512m' }, + runtime: { type: 'docker', memory: '512m' }, + ...record, + }), + ) const operations = new SharedVmOperations(getDb()) let status const handler = createPanelHandler({ - cfg: { paths: { project: root } }, routingConfig: { vm: { memory: '512m' } }, + cfg: { paths: { project: root } }, + routingConfig: { vm: { memory: '512m' } }, vmOperations: operations, - requireAuth(req) { req.apiKeyKind = 'master'; req.panelRole = 'admin'; return true }, + requireAuth(req) { + req.apiKeyKind = 'master' + req.panelRole = 'admin' + return true + }, readBody: async () => patch, - json(_res, code) { status = code; return true }, - proxyPool: { unbindVm() {} }, accountQuota: { snapshot: () => ({ accounts: [] }) }, + json(_res, code) { + status = code + return true + }, + proxyPool: { unbindVm() {} }, + accountQuota: { snapshot: () => ({ accounts: [] }) }, }) await handler({ method: 'PATCH' }, {}, new URL('http://localhost/api/panel/vms/memory-vm')) assert.equal(status, 400) diff --git a/test/unit/panel-vm-memory-raise.test.mjs b/test/unit/panel-vm-memory-raise.test.mjs index b68f790a7..39e2a94c9 100644 --- a/test/unit/panel-vm-memory-raise.test.mjs +++ b/test/unit/panel-vm-memory-raise.test.mjs @@ -113,9 +113,17 @@ test('standalone memory errors preserve the original failure without a database const root = tmpProject('memory-error') const failure = new Error('synthetic memory failure') try { - const { handlePanel } = handler(root, { memory: '1g' }, { - raiseDockerSlotMemory: () => { throw failure }, - }) + const { handlePanel } = handler( + root, + { memory: '1g' }, + { + raiseDockerSlotMemory: () => { + throw failure + }, + }, + ) await assert.rejects(patchVm(handlePanel, 'memory-error'), (error) => error === failure) - } finally { fs.rmSync(root, { recursive: true, force: true }) } + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } }) From e0ad3f54e676342efaeba382898e705c96abf45b Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:16:16 +0800 Subject: [PATCH 345/473] test: distinguish completed remote preparation from unknown publication --- test/unit/remote-slot-publication.test.mjs | 159 +++++++++++++++++++-- 1 file changed, 148 insertions(+), 11 deletions(-) diff --git a/test/unit/remote-slot-publication.test.mjs b/test/unit/remote-slot-publication.test.mjs index 1936e11cf..187158cfa 100644 --- a/test/unit/remote-slot-publication.test.mjs +++ b/test/unit/remote-slot-publication.test.mjs @@ -6,13 +6,14 @@ import path from 'node:path' import { EventEmitter } from 'node:events' import { bindPlacement, unbindPlacement } from '../../src/lib/cluster/placement.mjs' import { queueSlotPush } from '../../src/lib/cluster/remote-slot-files.mjs' +import { runRemote } from '../../src/lib/cluster/remote-fs.mjs' import { writeKernelConfig, syncClaudeKernelConfigs } from '../../src/lib/transport/rust-kernel-supervisor.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' import { getDb } from '../../src/lib/db/database.mjs' import { createRepositoryTestDatabase } from '../repository-database.mjs' import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' -async function fixture(t) { +async function fixture(t, { preparationCode = 0 } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'remote-publication-')) const vm = { id: 'vm-synthetic', node_id: 'node-synthetic', timezone: 'UTC', status: 'stopped' } const slotDir = path.join(root, 'vms', vm.id) @@ -27,6 +28,7 @@ async function fixture(t) { let failure = null const waiting = [] const writes = [] + let sftpOpens = 0 const sftp = new EventEmitter() Object.assign(sftp, { open: (_file, _flags, _attrs, callback) => callback(null, Buffer.from('synthetic-handle')), @@ -42,25 +44,27 @@ async function fixture(t) { }, }) const client = { - sftp: (callback) => callback(null, sftp), - exec: (_command, callback) => { + sftp: (callback) => { + sftpOpens++ + callback(null, sftp) + }, + exec: (command, callback) => { const stream = new EventEmitter() stream.stderr = new EventEmitter() stream.close = () => {} callback(null, stream) queueMicrotask(() => { - stream.emit('exit', 0) + const code = typeof preparationCode === 'function' ? preparationCode(command) : preparationCode + if (code !== null) stream.emit('exit', code) stream.emit('close') }) }, } - bindPlacement({ - projectRoot: root, - manager: { - client: () => client, - remoteHost: async () => ({ root: '/synthetic-node', uid: 1000, gid: 1000 }), - }, - }) + const manager = { + client: () => client, + remoteHost: async () => ({ root: '/synthetic-node', uid: 1000, gid: 1000 }), + } + bindPlacement({ projectRoot: root, manager }) const finish = (error = null) => { completed = true failure = error @@ -88,11 +92,91 @@ async function fixture(t) { finish, finishFirst: (error) => waiting.shift()(error), writes, + client, + manager, + sftpOpens: () => sftpOpens, operations: new SharedVmOperations(db), other: new SharedVmOperations(peer), } } +test('a nonzero remote command exit is confirmed only after its channel closes', async () => { + const stream = new EventEmitter() + stream.stderr = new EventEmitter() + stream.close = () => {} + let settled = false + const command = runRemote({ exec: (_command, callback) => callback(null, stream) }, 'synthetic command').finally( + () => { + settled = true + }, + ) + command.catch(() => {}) + stream.emit('exit', 7) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(settled, false) + stream.emit('close') + await assert.rejects(command, { code: 'remote_exec_failed', remoteCommandEnded: true }) +}) + +test('a command timeout after exit without channel close has no terminal proof', async () => { + const stream = new EventEmitter() + stream.stderr = new EventEmitter() + stream.close = () => {} + const command = runRemote({ exec: (_command, callback) => callback(null, stream) }, 'synthetic command', { + timeoutMs: 5, + }) + stream.emit('exit', 7) + await assert.rejects(command, (error) => { + assert.equal(error.code, 'exec_timeout') + assert.notEqual(error.remoteCommandEnded, true) + return true + }) +}) + +test('a confirmed initial directory preparation failure ends the publication before SFTP starts', async (t) => { + const f = await fixture(t, { preparationCode: 1 }) + const publication = queueSlotPush(f.vm, f.slotDir, ['run'], { logger: { warn() {} } }) + await assert.rejects(publication, (error) => { + assert.equal(error.code, 'remote_exec_failed') + assert.equal(error.remoteCommandEnded, true) + assert.equal(error.publicationFinished, true) + assert.notEqual(error.vmOperationUncertain, true) + assert.notEqual(error.executionUncertain, true) + return true + }) + assert.equal(f.sftpOpens(), 0) +}) + +for (const code of [null, '1', Number.NaN]) { + test(`preparation close without a safe integer exit (${String(code)}) retains unknown publication`, async (t) => { + const f = await fixture(t, { preparationCode: code }) + await assert.rejects(queueSlotPush(f.vm, f.slotDir, ['run'], { logger: { warn() {} } }), (error) => { + assert.notEqual(error.remoteCommandEnded, true) + assert.notEqual(error.publicationFinished, true) + assert.equal(error.vmOperationUncertain, true) + assert.equal(error.executionUncertain, true) + return true + }) + assert.equal(f.sftpOpens(), 0) + }) +} + +test('contradictory completed publication metadata cannot clear an existing unknown marker', async (t) => { + const f = await fixture(t) + const failure = Object.assign(new Error('synthetic contradictory completion proof'), { + publicationFinished: true, + vmOperationUncertain: true, + }) + f.manager.client = () => { + throw failure + } + await assert.rejects(queueSlotPush(f.vm, f.slotDir, ['run'], { logger: { warn() {} } }), { + publicationFinished: true, + vmOperationUncertain: true, + executionUncertain: true, + }) +}) + test('queued SFTP errors reject the exact publication promise after logging', async (t) => { const f = await fixture(t) const logged = [] @@ -103,6 +187,20 @@ test('queued SFTP errors reject the exact publication promise after logging', as assert.equal(logged.length, 1) }) +test('a later SFTP failure cannot become a completed publication from command-level metadata', async (t) => { + const f = await fixture(t) + const publication = queueSlotPush(f.vm, f.slotDir, ['run'], { logger: { warn() {} } }) + await f.started + f.finish(Object.assign(new Error('synthetic interrupted SFTP'), { remoteCommandEnded: true })) + await assert.rejects(publication, (error) => { + assert.equal(error.code, 'sftp_failed') + assert.notEqual(error.publicationFinished, true) + assert.equal(error.vmOperationUncertain, true) + assert.equal(error.executionUncertain, true) + return true + }) +}) + test('kernel config publication keeps its VM claim until the original SFTP write completes', async (t) => { const f = await fixture(t) let ended = false @@ -207,3 +305,42 @@ test('a fleet config publication waits for every original push before reporting assert.equal(ended, true) assert.equal(JSON.stringify(result).includes('publication'), false) }) + +test('a fleet publication prioritizes an unknown failure after an earlier confirmed preparation failure', async (t) => { + const f = await fixture(t, { preparationCode: (command) => (command.includes('/vm-synthetic/') ? 1 : 0) }) + const second = { ...f.vm, id: 'vm-z-unknown' } + fs.mkdirSync(path.join(f.root, 'vms', second.id, 'run'), { recursive: true }) + fs.writeFileSync(path.join(f.root, 'vms', second.id + '.json'), JSON.stringify(second)) + const result = syncClaudeKernelConfigs(f.root, {}) + let settled = false + const publication = result.publication.finally(() => { + settled = true + }) + publication.catch(() => {}) + await f.started + assert.equal(settled, false) + f.finish(new Error('synthetic later SFTP disconnect')) + await assert.rejects(publication, (error) => { + assert.equal(error.code, 'sftp_failed') + assert.notEqual(error.publicationFinished, true) + assert.equal(error.vmOperationUncertain, true) + assert.equal(error.executionUncertain, true) + return true + }) +}) + +test('a fleet with only confirmed preparation failures retains complete publication proof', async (t) => { + const f = await fixture(t, { preparationCode: 1 }) + const second = { ...f.vm, id: 'vm-z-ended' } + fs.mkdirSync(path.join(f.root, 'vms', second.id, 'run'), { recursive: true }) + fs.writeFileSync(path.join(f.root, 'vms', second.id + '.json'), JSON.stringify(second)) + const result = syncClaudeKernelConfigs(f.root, {}) + await assert.rejects(result.publication, (error) => { + assert.equal(error.code, 'remote_exec_failed') + assert.equal(error.publicationFinished, true) + assert.notEqual(error.vmOperationUncertain, true) + assert.notEqual(error.executionUncertain, true) + return true + }) + assert.equal(f.sftpOpens(), 0) +}) From 04dde1cd68b51139361580fc6be29f12f61a70e6 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:17:02 +0800 Subject: [PATCH 346/473] fix: preserve confirmed remote preparation completion without masking unknown pushes --- src/lib/cluster/remote-fs.mjs | 4 +++- src/lib/cluster/remote-slot-files.mjs | 20 +++++++++++++++++--- src/lib/transport/rust-kernel-supervisor.mjs | 9 ++++++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/lib/cluster/remote-fs.mjs b/src/lib/cluster/remote-fs.mjs index 293a9162e..91e741dba 100644 --- a/src/lib/cluster/remote-fs.mjs +++ b/src/lib/cluster/remote-fs.mjs @@ -116,7 +116,9 @@ export async function runRemote(client, script, { timeoutMs = 60_000, code = 're const r = await execCollect(client, `sh -c ${shellQuote(script)}`, { timeoutMs }) if (r.code !== 0) { const detail = (r.stderr || r.stdout || `exit ${r.code}`).trim().slice(-500) - throw new ClusterError(502, code, detail) + const error = new ClusterError(502, code, detail) + if (Number.isSafeInteger(r.code)) error.remoteCommandEnded = true + throw error } return r.stdout } diff --git a/src/lib/cluster/remote-slot-files.mjs b/src/lib/cluster/remote-slot-files.mjs index 107374a2f..e82115b89 100644 --- a/src/lib/cluster/remote-slot-files.mjs +++ b/src/lib/cluster/remote-slot-files.mjs @@ -94,7 +94,15 @@ function ownerFor(session, vm) { /** Push the named sets (`run`, `seed`, `touch`). Missing local files are skipped. */ export async function pushSlotFiles(vm, slotDir, sets = ['run', 'seed'], session = null) { const s = session || (await nodeSession(vmNodeId(vm))) - await prepareRemoteDir(s, vm) + try { + await prepareRemoteDir(s, vm) + } catch (error) { + // This is the first remote step. Its confirmed failure precedes every SFTP operation. + if (error.remoteCommandEnded === true && error.vmOperationUncertain !== true && error.executionUncertain !== true) { + error.publicationFinished = true + } + throw error + } const sftp = await openSftp(s.client) const remote = remoteSlotDir(s.host, vm.id) const owner = ownerFor(s, vm) @@ -226,8 +234,14 @@ export function queueSlotPush(vm, slotDir, sets, { logger = console } = {}) { .catch(() => {}) .then(() => pushSlotFiles(vm, slotDir, sets)) .catch((error) => { - error.vmOperationUncertain = true - error.executionUncertain = true + if ( + error.publicationFinished !== true || + error.vmOperationUncertain === true || + error.executionUncertain === true + ) { + error.vmOperationUncertain = true + error.executionUncertain = true + } throw error }) .finally(() => { diff --git a/src/lib/transport/rust-kernel-supervisor.mjs b/src/lib/transport/rust-kernel-supervisor.mjs index 54a8c6eff..bccb9ed0d 100644 --- a/src/lib/transport/rust-kernel-supervisor.mjs +++ b/src/lib/transport/rust-kernel-supervisor.mjs @@ -823,7 +823,14 @@ export function syncClaudeKernelConfigs(projectRoot, routing) { } } const publication = Promise.allSettled(publications).then((results) => { - const failed = results.find((result) => result.status === 'rejected') + const failures = results.filter((result) => result.status === 'rejected') + const unknown = failures.find( + ({ reason }) => + reason?.publicationFinished !== true || + reason.vmOperationUncertain === true || + reason.executionUncertain === true, + ) + const failed = unknown || failures[0] if (failed) throw failed.reason }) publication.catch(() => {}) From a5843c0f56288aecb787cb2899dd3e0ef7683bc0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:18:20 +0800 Subject: [PATCH 347/473] test: keep Codex catalog refresh fixtures independent of registry requests --- test/unit/vm-test-chat-codex.test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/vm-test-chat-codex.test.mjs b/test/unit/vm-test-chat-codex.test.mjs index 9777f891b..b0ab43312 100644 --- a/test/unit/vm-test-chat-codex.test.mjs +++ b/test/unit/vm-test-chat-codex.test.mjs @@ -230,6 +230,7 @@ test('resolveTestModels refresh merges ChatGPT ids and ignores 401', async (t) = projectRoot: root, vmId: 'vm-codex-01', refresh: true, + catalogClientVersion: '0.158.0', fetchImpl: async () => ({ ok: true, status: 200, From 43076583817af19d3e0f5e15a0a097e03a89bd81 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:18:32 +0800 Subject: [PATCH 348/473] test: initialize PostgreSQL for Codex catalog policy persistence --- test/unit/vm-test-chat-codex.test.mjs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/unit/vm-test-chat-codex.test.mjs b/test/unit/vm-test-chat-codex.test.mjs index b0ab43312..e6cea7494 100644 --- a/test/unit/vm-test-chat-codex.test.mjs +++ b/test/unit/vm-test-chat-codex.test.mjs @@ -4,6 +4,7 @@ import http from 'node:http' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' import { listTestableModels, resolveTestModels, @@ -240,6 +241,8 @@ test('resolveTestModels refresh merges ChatGPT ids and ignores 401', async (t) = assert.equal(merged.source, 'chatgpt') assert.ok(merged.items.some((m) => m.id === 'gpt-5.6')) assert.equal(merged.protocol, 'openai.responses') + const stored = (await getDb().query("SELECT value FROM settings WHERE key='gpt_model_policy'")).rows[0] + assert.equal(JSON.parse(stored.value).models['gpt-5.6'].id, 'gpt-5.6') const denied = await resolveTestModels({ projectRoot: root, @@ -285,6 +288,10 @@ test('syncCodexCatalog refreshes OAuth on 401 then merges GPT ids', async (t) => assert.equal(result.ok, true) assert.ok(result.ids.includes('gpt-5.6')) assert.ok(result.ids.includes('gpt-5.5')) + const stored = (await getDb().query("SELECT value FROM settings WHERE key='gpt_model_policy'")).rows[0] + const models = JSON.parse(stored.value).models + assert.equal(models['gpt-5.6'].id, 'gpt-5.6') + assert.equal(models['gpt-5.5'].id, 'gpt-5.5') const saved = JSON.parse(fs.readFileSync(path.join(root, 'vms', 'vm-codex-01', 'codex-credentials.json'), 'utf8')) assert.equal(saved.accounts[0].access_token, 'at-rotated') assert.equal(saved.accounts[0].refresh_token, 'rt-rotated') From f9f396dd3186216117f6a877d329b3ad7ea95114 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:18:48 +0800 Subject: [PATCH 349/473] test: isolate mock gateways from host Docker and preserve egress receipt fixtures --- test/fixtures/mock-gateway-runtime-loader.mjs | 17 +++++ test/fixtures/mock-gateway-runtime.mjs | 70 +++++++++++++++++++ test/harness.mjs | 34 +++++++-- test/unit/egress.test.mjs | 27 +++++-- 4 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 test/fixtures/mock-gateway-runtime-loader.mjs create mode 100644 test/fixtures/mock-gateway-runtime.mjs diff --git a/test/fixtures/mock-gateway-runtime-loader.mjs b/test/fixtures/mock-gateway-runtime-loader.mjs new file mode 100644 index 000000000..c7ff3ae1d --- /dev/null +++ b/test/fixtures/mock-gateway-runtime-loader.mjs @@ -0,0 +1,17 @@ +import { registerHooks } from 'node:module' + +if (process.env.KIN_CRS_MOCK !== '1') throw new Error('mock gateway runtime requires KIN_CRS_MOCK=1') + +const mocked = new Set([ + '../../src/lib/vm/egress.mjs', + '../../src/lib/vm/proxy-geo.mjs', + '../../src/lib/cluster/docker-remote.mjs', +].map((file) => new URL(file, import.meta.url).href)) +const fixture = new URL('./mock-gateway-runtime.mjs', import.meta.url).href + +registerHooks({ + resolve(specifier, context, nextResolve) { + const resolved = nextResolve(specifier, context) + return mocked.has(resolved.url) ? { ...resolved, url: fixture } : resolved + }, +}) diff --git a/test/fixtures/mock-gateway-runtime.mjs b/test/fixtures/mock-gateway-runtime.mjs new file mode 100644 index 000000000..a0279473d --- /dev/null +++ b/test/fixtures/mock-gateway-runtime.mjs @@ -0,0 +1,70 @@ +// Only harness-spawned gateways load this module. Pure policy/formatting logic +// stays real; egress effects use an in-memory registry, not the host daemon. +export * from '../../src/lib/vm/egress.mjs?mock-gateway-original' +export * from '../../src/lib/vm/proxy-geo.mjs?mock-gateway-original' +export * from '../../src/lib/cluster/docker-remote.mjs?mock-gateway-original' +import { + boundProxyUrl, + isLocalEgressProxy, + networkName, +} from '../../src/lib/vm/egress.mjs?mock-gateway-original' +import { proxyBlockedReason } from '../../src/lib/vm/proxy-policy.mjs' + +const networks = new Set() + +function ensure(proxy) { + const blocked = proxyBlockedReason(proxy) + if (blocked) return { ok: false, error: blocked, vmOperationNotStarted: true } + if (!proxy?.id || (!isLocalEgressProxy(proxy) && !boundProxyUrl(proxy))) + return { ok: false, error: 'bound SOCKS5 id and url required', vmOperationNotStarted: true } + const name = networkName(proxy.id) + const reused = networks.has(name) + networks.add(name) + return { ok: true, mock: true, reused, network: name, proxy_id: proxy.id } +} + +export async function ensureProxyEgress(_projectRoot, proxy) { + return ensure(proxy) +} + +export function ensureLocalProxyEgress(proxy) { + return ensure(proxy) +} + +export async function startProxyEgress(_projectRoot, proxy) { + return ensure(proxy) +} + +export async function stopProxyEgress(_projectRoot, proxyId) { + networks.delete(networkName(proxyId)) + return { ok: true, ended: true, mock: true } +} + +export function proxyEgressReady(proxy) { + const blocked = proxyBlockedReason(proxy) + if (blocked) return { ok: false, reason: blocked } + if (isLocalEgressProxy(proxy)) return { ok: true, mode: 'direct' } + return networks.has(networkName(proxy?.id)) ? { ok: true, mock: true } : { ok: false, reason: 'not_running' } +} + +export function localEgressStatus(proxyOrId) { + const proxy = typeof proxyOrId === 'string' ? { id: proxyOrId } : proxyOrId + if (!isLocalEgressProxy(proxy)) return null + const name = networkName(proxy.id) + return networks.has(name) ? { ok: true, mode: 'local', name, network: name } : { ok: false, reason: 'local_network_missing' } +} + +// Geo remains unavailable instead of inventing a successful provider response. +export async function lookupProxyGeo() { + return { ok: false, error: 'mock_geo_unavailable' } +} + +export async function lookupProxyGeoV6() { + return { ok: false, error: 'mock_geo_unavailable' } +} + +export function localDocker() { + return async () => { + throw Object.assign(new Error('mock gateway has no Docker daemon'), { code: 'docker_unavailable' }) + } +} diff --git a/test/harness.mjs b/test/harness.mjs index 82609f8d4..b127f70ce 100644 --- a/test/harness.mjs +++ b/test/harness.mjs @@ -119,11 +119,34 @@ export async function startGateway(opts = {}) { } if (opts.env) Object.assign(env, opts.env) - const child = spawn(process.execPath, [path.join(GATEWAY_ROOT, 'server.mjs')], { - env, - cwd: GATEWAY_ROOT, - stdio: ['ignore', 'pipe', 'pipe'], - }) + // Mock HTTP gateways must never create host bridges, change iptables, or exec + // a real slot. The loader supplies virtual egress; other Docker calls fail closed. + const boundaryDir = path.join(project, 'mock-runtime') + fs.mkdirSync(boundaryDir, { recursive: true }) + const runtimeCallsFile = path.join(boundaryDir, 'commands.log') + for (const command of ['docker', 'iptables']) { + fs.writeFileSync( + path.join(boundaryDir, command), + `#!${process.execPath}\nconst fs = require('node:fs');\nfs.appendFileSync(${JSON.stringify(runtimeCallsFile)}, ${JSON.stringify(command + ' ')} + process.argv.slice(2, 4).join(' ') + '\\n');\nprocess.stderr.write('Error: No such object: mock gateway runtime\\n');\nprocess.exit(1);\n`, + { mode: 0o700 }, + ) + } + env.PATH = boundaryDir + path.delimiter + env.PATH + env.KIN_DOCKER_BIN = path.join(boundaryDir, 'docker') + + const child = spawn( + process.execPath, + [ + '--import', + path.join(REPO_ROOT, 'test/fixtures/mock-gateway-runtime-loader.mjs'), + path.join(GATEWAY_ROOT, 'server.mjs'), + ], + { + env, + cwd: GATEWAY_ROOT, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ) let stdout = '' let stderr = '' child.stdout.on('data', (d) => { @@ -163,6 +186,7 @@ export async function startGateway(opts = {}) { port, project, traceFile, + runtimeCallsFile, apiKey: env.KIN_API_KEY, child, stdout: () => stdout, diff --git a/test/unit/egress.test.mjs b/test/unit/egress.test.mjs index e6000a45d..b7566f3a8 100644 --- a/test/unit/egress.test.mjs +++ b/test/unit/egress.test.mjs @@ -29,6 +29,7 @@ import { slotNetworkForVm, startEgressProcess, stopEgressProcess, + stopEgressProcessConfirmed, waitEgressExit, egressPidActive, ensureProxyEgress, @@ -539,25 +540,39 @@ test('DNS upstream validation rejects unsafe or malformed URL boundaries', () => } }) -test('egress config carries dns_upstream only when configured', async () => { +test('egress config carries dns_upstream only when configured', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'egress-dns-')) + const bin = path.join(root, 'kin-egress-config-fixture') + fs.writeFileSync(bin, `#!${process.execPath}\nsetInterval(() => {}, 1000)\n`, { mode: 0o700 }) + const proxies = [] + t.after(async () => { + for (const proxyId of proxies) { + const stopped = await stopEgressProcessConfirmed(root, proxyId, { termTimeoutMs: 500, killGraceMs: 500 }) + assert.equal(stopped.ok, true, JSON.stringify(stopped)) + } + fs.rmSync(root, { recursive: true, force: true }) + }) const base = { projectRoot: root, proxyUrl: 'socks5h://127.0.0.1:1', - tcpPort: 20000, - dnsPort: 20001, + tcpPort: 0, + dnsPort: 0, listenHost: '127.0.0.1', - bin: '/bin/true', + bin, } const a = await startEgressProcess({ ...base, proxyId: 'px-a', dnsUpstream: '' }) - assert.equal(a.ok, true) + if (a.pid) proxies.push('px-a') + assert.equal(a.ok, true, JSON.stringify(a)) assert.equal(JSON.parse(fs.readFileSync(a.configPath, 'utf8')).dns_upstream, undefined) const b = await startEgressProcess({ ...base, proxyId: 'px-b', dnsUpstream: '8.8.8.8:53,1.1.1.1:53' }) + if (b.pid) proxies.push('px-b') + assert.equal(b.ok, true, JSON.stringify(b)) assert.equal(JSON.parse(fs.readFileSync(b.configPath, 'utf8')).dns_upstream, '8.8.8.8:53,1.1.1.1:53') const c = await startEgressProcess({ ...base, proxyId: 'px-c', domainForward: true }) + if (c.pid) proxies.push('px-c') + assert.equal(c.ok, true, JSON.stringify(c)) assert.equal(JSON.parse(fs.readFileSync(c.configPath, 'utf8')).domain_forward, true) assert.equal(JSON.parse(fs.readFileSync(a.configPath, 'utf8')).domain_forward, undefined) - fs.rmSync(root, { recursive: true, force: true }) }) test('egress readiness checks the exact listener without opening a connection', async (t) => { From 1b6deff1e518d42b52a8f516f998bfa69214f0ca Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:21:19 +0800 Subject: [PATCH 350/473] test: cover completed and uncertain protocol publication capacity --- test/fixtures/protocol-publication.mjs | 10 ++++++---- test/unit/protocol-publication.test.mjs | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/fixtures/protocol-publication.mjs b/test/fixtures/protocol-publication.mjs index e034112ad..b9cfcf8fa 100644 --- a/test/fixtures/protocol-publication.mjs +++ b/test/fixtures/protocol-publication.mjs @@ -24,7 +24,7 @@ mock.module(new URL('../../src/lib/transport/rust-kernel-supervisor.mjs', import namedExports: { ...supervisor, syncClaudeKernelConfigsFromFile() { - if (kind === 'sync') { enter(); return { publication } } + if (kind.startsWith('sync')) { enter(); return { publication } } return { updated: 0 } }, }, @@ -91,13 +91,15 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as await request assert.equal(runnerCalls, 1) } else { - const finishedPublication = kind === 'touch-finished' + const finishedPublication = kind.endsWith('-finished') reject(Object.assign(new Error('synthetic publication failed'), { code: 'sftp_failed', - ...(finishedPublication ? { publicationFinished: true } : { executionUncertain: true }), + ...(kind.includes('-finished') ? { publicationFinished: true } : {}), + ...(!finishedPublication ? { executionUncertain: true } : {}), })) await assert.rejects(request, /synthetic publication failed/) - assert.equal(marked, finishedPublication ? 0 : 1) + assert.equal(marked, finishedPublication || kind.startsWith('sync') ? 0 : 1) + if (kind.startsWith('sync')) assert.equal(runnerCalls, 0) assert.equal(states.some((state) => state.started === true && state.ended === false && !state.requestId), !finishedPublication) const row = await db.prepare('SELECT released_at FROM api_key_admissions WHERE key_id=$1').get(key.id) if (finishedPublication) { diff --git a/test/unit/protocol-publication.test.mjs b/test/unit/protocol-publication.test.mjs index 19c33bd36..0ce3d13c0 100644 --- a/test/unit/protocol-publication.test.mjs +++ b/test/unit/protocol-publication.test.mjs @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url' const fixture = fileURLToPath(new URL('../fixtures/protocol-publication.mjs', import.meta.url)) const setup = fileURLToPath(new URL('../pg-setup.mjs', import.meta.url)) -for (const kind of ['sync', 'touch', 'touch-finished']) { +for (const kind of ['sync', 'sync-finished', 'sync-finished-unknown', 'touch', 'touch-finished', 'touch-finished-unknown']) { test(`protocol awaits its own ${kind} publication`, () => { const env = { ...process.env, VM2API_PROTOCOL_PUBLICATION_CASE: kind } delete env.NODE_TEST_CONTEXT From 9d06160d485112fb19e0de2e9c17edcc3296b6d4 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:21:43 +0800 Subject: [PATCH 351/473] test: check fleet publication before API key reservation --- test/fixtures/protocol-publication.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/fixtures/protocol-publication.mjs b/test/fixtures/protocol-publication.mjs index b9cfcf8fa..cf7afd797 100644 --- a/test/fixtures/protocol-publication.mjs +++ b/test/fixtures/protocol-publication.mjs @@ -102,7 +102,9 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as if (kind.startsWith('sync')) assert.equal(runnerCalls, 0) assert.equal(states.some((state) => state.started === true && state.ended === false && !state.requestId), !finishedPublication) const row = await db.prepare('SELECT released_at FROM api_key_admissions WHERE key_id=$1').get(key.id) - if (finishedPublication) { + if (kind.startsWith('sync')) { + assert.equal(row, undefined, 'fleet publication precedes API key reservation') + } else if (finishedPublication) { assert.ok(row.released_at) const next = await store.reserve(key) assert.equal(next.ok, true) From 06e8008be84f1cb261d8589dab7af8bc0a7d5333 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:26:16 +0800 Subject: [PATCH 352/473] test: exercise publication lifecycle through the real shared failover runner --- test/fixtures/protocol-publication.mjs | 145 ++++++++++++++++++++---- test/unit/protocol-publication.test.mjs | 11 +- 2 files changed, 133 insertions(+), 23 deletions(-) diff --git a/test/fixtures/protocol-publication.mjs b/test/fixtures/protocol-publication.mjs index cf7afd797..b33cb9f89 100644 --- a/test/fixtures/protocol-publication.mjs +++ b/test/fixtures/protocol-publication.mjs @@ -5,8 +5,13 @@ import os from 'node:os' import path from 'node:path' import { Readable } from 'node:stream' import { EventEmitter } from 'node:events' +import pg from 'pg' import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' -import { getDb } from '../../src/lib/db/database.mjs' +import { getDb, registerActivityCoordinator, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' import { readBody } from '../../src/lib/http/respond.mjs' const kind = process.env.VM2API_PROTOCOL_PUBLICATION_CASE @@ -31,18 +36,88 @@ mock.module(new URL('../../src/lib/transport/rust-kernel-supervisor.mjs', import }) const { createHandleProtocol } = await import('../../src/lib/protocol/handle-protocol.mjs') -test(`protocol waits for ${kind} publication and preserves unknown activity`, async (t) => { +test(`protocol uses the real shared runner for ${kind} publication`, async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-protocol-publication-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) const db = getDb() const store = new ApiKeyStore({ db, sharedAdmission: true }) const key = await store.create({ max_concurrency: 1 }) + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const otherStore = new ApiKeyStore({ db: peer, sharedAdmission: true }) + const executions = new SharedExecutionAdmissions(db) + const otherExecutions = new SharedExecutionAdmissions(peer) + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + registerActivityCoordinator(db, coordinator) + t.after(async () => { + await coordinator.stop() + await peer.close() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + const activity = await coordinator.admit({ kind: 'http', requestId: 'publication-test' }) let runnerCalls = 0 + let selections = 0 + let modelCalls = 0 let marked = 0 let finished = false let noteRunner const runnerStarted = new Promise((done) => { noteRunner = done }) const states = [] + const plan = { + vmId: 'publication-vm', accountId: 'publication-account', + accountLimit: 1, vmLimit: 1, fableLimit: 0, rpmLimit: 0, + } + const scheduler = { + executionAdmissions: executions, + async selectAndReserve() { + selections++ + if (selections > 1) return { ok: false, reason: 'no_eligible_accounts' } + const lease = await executions.tryReserve(plan) + assert.equal(lease.ok, true) + return { + ...lease, + ...plan, + vm: { id: plan.vmId, claude: {} }, + exec: { vmId: plan.vmId, homeDir: path.join(root, 'synthetic-home'), vm: {} }, + markUncertain: async () => { + marked++ + return lease.markUncertain() + }, + } + }, + markCooldown() {}, + markSuccess() {}, + } + const runner = new FailoverRunner({ + scheduler, + config: { max_total_attempts: 2, max_account_switches: 1, max_same_account_retries: 0 }, + }) + const run = runner.run.bind(runner) + // Keep the production runner's preparation/catch/finally path. Only backend execution is synthetic. + t.mock.method(runner, 'run', async (args) => { + runnerCalls++ + noteRunner() + return run({ + ...args, + callAttempt: async ({ candidate }) => { + modelCalls++ + const execution = { requestId: 'synthetic-backend', started: true, ended: true, reason: 'synthetic_response' } + await candidate.bindExecution(execution) + await candidate.completeExecution(execution) + return { + ok: false, status: 400, terminalState: 'rejected', execution, + body: { type: 'error', error: { type: 'invalid_request_error', message: 'synthetic backend rejection' } }, + } + }, + }) + }) const handler = createHandleProtocol({ cfg: { limits: { max_body_bytes: 1 << 20 }, rewrite: { enabled: false }, @@ -57,29 +132,27 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as apiKeyStore: store, stickyRouter: { resolve: () => null }, accountQuota: { ingestHeaders() {} }, - failoverRunner: { - async run({ applyAttempt, canonicalBody }) { - runnerCalls++ - noteRunner() - if (kind.startsWith('touch')) await applyAttempt(canonicalBody, { - vmId: 'publication-vm', accountId: 'publication-account', vm: {}, exec: {}, - markUncertain: async () => { marked++ }, - }) - return { ok: false, status: 502, body: { type: 'error', error: { type: 'api_error', message: 'synthetic' } } } - }, - }, + failoverRunner: runner, routingConfig: { compatibility: { claude_code_only: false } }, - onExecutionState: async (_req, state) => states.push(state), + onExecutionState: async (_req, state) => { + states.push(state) + activity.noteExecution(state) + }, }) const req = Readable.from([Buffer.from(JSON.stringify({ model: 'claude-sonnet-4-6', max_tokens: 20, messages: [{ role: 'user', content: 'synthetic' }] }))]) req.headers = { 'user-agent': 'claude-cli/2.1.0 (external, cli)', 'anthropic-version': '2023-06-01' } const res = new EventEmitter() res.setHeader = () => {} - const request = handler.handleProtocol(req, res, 'anthropic.messages', '/v1/messages') + const request = activity.run(() => handler.handleProtocol(req, res, 'anthropic.messages', '/v1/messages')) request.then(() => { finished = true }, () => { finished = true }) await entered await new Promise((done) => setImmediate(done)) assert.equal(finished, false) + assert.equal(modelCalls, 0) + if (kind.startsWith('touch')) { + assert.equal((await otherExecutions.tryReserve(plan)).ok, false, 'another connection cannot reuse pending capacity') + assert.equal((await otherStore.reserve(key)).code, 'api_key_concurrency_limit') + } if (kind === 'sync') { const phase = await Promise.race([ runnerStarted.then(() => 'started'), @@ -90,6 +163,9 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as resolve() await request assert.equal(runnerCalls, 1) + assert.equal(modelCalls, 1) + const keyAdmission = await peer.prepare('SELECT released_at FROM api_key_admissions WHERE key_id=$1').get(key.id) + assert.ok(keyAdmission.released_at) } else { const finishedPublication = kind.endsWith('-finished') reject(Object.assign(new Error('synthetic publication failed'), { @@ -97,21 +173,48 @@ test(`protocol waits for ${kind} publication and preserves unknown activity`, as ...(kind.includes('-finished') ? { publicationFinished: true } : {}), ...(!finishedPublication ? { executionUncertain: true } : {}), })) - await assert.rejects(request, /synthetic publication failed/) - assert.equal(marked, finishedPublication || kind.startsWith('sync') ? 0 : 1) - if (kind.startsWith('sync')) assert.equal(runnerCalls, 0) + if (kind.startsWith('sync')) { + await assert.rejects(request, /synthetic publication failed/) + assert.equal(runnerCalls, 0) + } else { + await request + assert.equal(runnerCalls, 1) + assert.ok(res.statusCode >= 400) + assert.ok(selections <= 2, 'the scheduler provides at most one candidate') + } + assert.equal(modelCalls, 0, 'publication preparation failure never invokes the model backend') + assert.equal(marked > 0, !finishedPublication && kind.startsWith('touch')) assert.equal(states.some((state) => state.started === true && state.ended === false && !state.requestId), !finishedPublication) - const row = await db.prepare('SELECT released_at FROM api_key_admissions WHERE key_id=$1').get(key.id) + const row = await peer.prepare('SELECT released_at FROM api_key_admissions WHERE key_id=$1').get(key.id) if (kind.startsWith('sync')) { assert.equal(row, undefined, 'fleet publication precedes API key reservation') } else if (finishedPublication) { assert.ok(row.released_at) - const next = await store.reserve(key) + const next = await otherStore.reserve(key) assert.equal(next.ok, true) await next.release() } else { assert.equal(row.released_at, null) - assert.equal((await store.reserve(key)).code, 'api_key_concurrency_limit') + assert.equal((await otherStore.reserve(key)).code, 'api_key_concurrency_limit') } } + const unknown = states.some((state) => state.started === true && state.ended === false) + assert.equal(unknown, kind !== 'sync' && !kind.endsWith('-finished')) + assert.equal((await peer.prepare('SELECT count(*) AS n FROM execution_jobs').get()).n, modelCalls) + await activity.complete({ ended: !unknown, reason: unknown ? 'remote_publication_unknown' : 'request_finished' }) + const durableActivity = await peer.pool.query( + `SELECT completed_at, uncertain_at FROM ${quoteIdentifier(coordinator.schema)}.activities WHERE id=$1`, + [activity.activityId], + ) + assert.equal(Boolean(durableActivity.rows[0].completed_at), !unknown) + assert.equal(Boolean(durableActivity.rows[0].uncertain_at), unknown) + const execution = await peer.prepare('SELECT released_at, uncertain_at FROM execution_admissions').get() + if (kind.startsWith('sync') && kind !== 'sync') assert.equal(execution, undefined) + else { + assert.equal(Boolean(execution.released_at), !unknown) + assert.equal(Boolean(execution.uncertain_at), unknown) + const next = await otherExecutions.tryReserve(plan) + assert.equal(next.ok, !unknown, 'another connection sees the final execution capacity') + if (next.ok) await next.release() + } }) diff --git a/test/unit/protocol-publication.test.mjs b/test/unit/protocol-publication.test.mjs index 0ce3d13c0..400a07427 100644 --- a/test/unit/protocol-publication.test.mjs +++ b/test/unit/protocol-publication.test.mjs @@ -5,8 +5,15 @@ import { fileURLToPath } from 'node:url' const fixture = fileURLToPath(new URL('../fixtures/protocol-publication.mjs', import.meta.url)) const setup = fileURLToPath(new URL('../pg-setup.mjs', import.meta.url)) -for (const kind of ['sync', 'sync-finished', 'sync-finished-unknown', 'touch', 'touch-finished', 'touch-finished-unknown']) { - test(`protocol awaits its own ${kind} publication`, () => { +for (const kind of [ + 'sync', + 'sync-finished', + 'sync-finished-unknown', + 'touch', + 'touch-finished', + 'touch-finished-unknown', +]) { + test(`protocol awaits ${kind} publication with real shared execution admissions`, () => { const env = { ...process.env, VM2API_PROTOCOL_PUBLICATION_CASE: kind } delete env.NODE_TEST_CONTEXT const result = spawnSync( From e9076922a4af1f43840e6f9db8381790c809ee21 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:26:47 +0800 Subject: [PATCH 353/473] fix: release protocol activity after confirmed preparation failure --- src/lib/protocol/handle-protocol.mjs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index bc2aee441..2793b8047 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -1082,6 +1082,12 @@ export function createHandleProtocol(deps) { try { await syncClaudeKernelConfigsFromFile(cfg.paths?.project, routingConfigPath)?.publication } catch (error) { + if ( + error?.publicationFinished === true && + error?.executionUncertain !== true && + error?.vmOperationUncertain !== true + ) + throw error admission.executionStarted = true if (admission.lease?.shared) admission.lease.executionUncertain = true await observeExecution(req, admission, { started: true, ended: false, reason: 'remote_publication_unknown' }) @@ -1356,6 +1362,12 @@ export function createHandleProtocol(deps) { try { await touchTelemetrySession(cfg.paths.project, selected.vmId, selected.exec?.vm)?.publication } catch (error) { + if ( + error?.publicationFinished === true && + error?.executionUncertain !== true && + error?.vmOperationUncertain !== true + ) + throw error error.executionUncertain = true admission.executionStarted = true if (admission.lease?.shared) admission.lease.executionUncertain = true From e5b91e69da9d23c85a45ccd4fe1cfe3a06217bcd Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:32:58 +0800 Subject: [PATCH 354/473] test: define explicit Kubernetes HA mode distinction --- test/unit/control-plane-ha-mode.test.mjs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test/unit/control-plane-ha-mode.test.mjs diff --git a/test/unit/control-plane-ha-mode.test.mjs b/test/unit/control-plane-ha-mode.test.mjs new file mode 100644 index 000000000..9abde7aac --- /dev/null +++ b/test/unit/control-plane-ha-mode.test.mjs @@ -0,0 +1,14 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import * as mode from '../../src/lib/core/control-plane-mode.mjs' + +test('HA is an explicit Kubernetes mode; existing standalone and singleton modes remain distinct', () => { + const ha = { VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha' } + assert.equal(mode.controlPlaneMode(ha), 'kubernetes-ha') + assert.equal(mode.isKubernetesMode(ha), true) + assert.equal(mode.isKubernetesHaMode(ha), true) + assert.equal(mode.isKubernetesHaMode({ VM2API_CONTROL_PLANE_MODE: 'kubernetes' }), false) + assert.equal(mode.isKubernetesHaMode({}), false) + assert.equal(mode.isKubernetesMode({}), false) + assert.throws(() => mode.controlPlaneMode({ VM2API_CONTROL_PLANE_MODE: 'ha' }), /CONTROL_PLANE_MODE/) +}) From dca06118b4f3e107d6974b2737f2086ca5f825f2 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:33:07 +0800 Subject: [PATCH 355/473] feat: distinguish opt-in Kubernetes HA from singleton mode --- src/lib/core/control-plane-mode.mjs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/core/control-plane-mode.mjs b/src/lib/core/control-plane-mode.mjs index 5ec005939..1f41d625f 100644 --- a/src/lib/core/control-plane-mode.mjs +++ b/src/lib/core/control-plane-mode.mjs @@ -1,13 +1,17 @@ export function controlPlaneMode(env = process.env) { const mode = env.VM2API_CONTROL_PLANE_MODE || 'standalone' - if (mode !== 'standalone' && mode !== 'kubernetes') { - throw new Error('VM2API_CONTROL_PLANE_MODE must be standalone or kubernetes') + if (mode !== 'standalone' && mode !== 'kubernetes' && mode !== 'kubernetes-ha') { + throw new Error('VM2API_CONTROL_PLANE_MODE must be standalone, kubernetes or kubernetes-ha') } return mode } export function isKubernetesMode(env = process.env) { - return controlPlaneMode(env) === 'kubernetes' + return controlPlaneMode(env) !== 'standalone' +} + +export function isKubernetesHaMode(env = process.env) { + return controlPlaneMode(env) === 'kubernetes-ha' } export function shutdownTimeoutMs(env = process.env) { From 60d635960cce980392f43a00d097ac1f23ff52e0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:35:56 +0800 Subject: [PATCH 356/473] test: verify discovery admission refusal without persistence failure --- test/unit/maintenance-monitor-startup.test.mjs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/unit/maintenance-monitor-startup.test.mjs b/test/unit/maintenance-monitor-startup.test.mjs index 02c4276b2..ce2e5eb80 100644 --- a/test/unit/maintenance-monitor-startup.test.mjs +++ b/test/unit/maintenance-monitor-startup.test.mjs @@ -210,6 +210,7 @@ test('a monitor can replace the closed initialization scope inherited by an asyn test('DRAINING rejects monitor configuration and VM discovery before any callback or claim', async (t) => { const { db, root, coordinator, tasks } = await fixture(t) let calls = 0 + let discoveryReads = 0 const callback = async () => { calls++ return { ok: true } @@ -224,8 +225,14 @@ test('DRAINING rejects monitor configuration and VM discovery before any callbac const runner = new VmTestScheduleRunner({ projectRoot: root, sharedTasks: tasks, - settings: () => new RuntimeSettings(db), - list: () => [vm], + settings: () => { + discoveryReads++ + return new RuntimeSettings(db) + }, + list: () => { + discoveryReads++ + return [vm] + }, runTest: callback, }) const operation = await coordinator.beginMaintenance({ kind: 'backup' }) @@ -233,7 +240,11 @@ test('DRAINING rejects monitor configuration and VM discovery before any callbac await assert.rejects(credential.runOnce(), { code: 'database_maintenance' }) await assert.rejects(health.runRealProbe(), { code: 'database_maintenance' }) await assert.rejects(usage.runOnce(), { code: 'database_maintenance' }) - await assert.rejects(runner.start(), { code: 'database_maintenance' }) + await runner.start() + await runner.flush() + assert.equal(discoveryReads, 0, 'a discovery admission refusal never reads settings or VMs') + assert.equal(runner.jobs.size, 0) + assert.equal(runner.writeFailure, null, 'maintenance refusal is not a persistence failure') assert.equal(calls, 0) assert.equal((await db.pool.query('SELECT count(*)::integer AS n FROM background_task_runs')).rows[0].n, 0) } finally { From da899ce4c4dd281d99b869479b272470c202b264 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:35:56 +0800 Subject: [PATCH 357/473] test: require PostgreSQL VM authority for HA controllers --- test/unit/ha-vm-catalog.test.mjs | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 test/unit/ha-vm-catalog.test.mjs diff --git a/test/unit/ha-vm-catalog.test.mjs b/test/unit/ha-vm-catalog.test.mjs new file mode 100644 index 000000000..c08c744a2 --- /dev/null +++ b/test/unit/ha-vm-catalog.test.mjs @@ -0,0 +1,64 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { getVm, listVms } from '../../src/lib/vm/vm-registry.mjs' + +test('HA reads VM metadata from PostgreSQL and never trusts or rebuilds Pod VM files', async (t) => { + const { HaVmCatalog, bindHaVmCatalog } = await import('../../src/lib/vm/ha-vm-catalog.mjs') + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ha-vm-catalog-')) + t.after(() => { + bindHaVmCatalog(null) + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + fs.rmSync(root, { recursive: true, force: true }) + }) + const db = getDb() + const repo = new VmsRepo(db) + const original = { + id: 'ha-catalog-vm', + node_id: 'ha-node', + status: 'running', + claude: { account_uuid: 'database-account', access_token: 'synthetic-secret', has_access: true }, + } + await repo.upsertFromVmJson(original) + fs.mkdirSync(path.join(root, 'vms')) + const file = path.join(root, 'vms', 'ha-catalog-vm.json') + const forged = JSON.stringify({ ...original, claude: { account_uuid: 'forged-account' } }) + fs.writeFileSync(file, forged) + const catalog = new HaVmCatalog(db) + bindHaVmCatalog(catalog) + assert.throws(() => getVm(root, original.id), { code: 'ha_vm_catalog_unavailable' }) + await catalog.refresh() + assert.equal(getVm(root, original.id).claude.account_uuid, 'database-account') + assert.equal(getVm(root, original.id).claude.access_token, undefined) + assert.equal(listVms(root).find((vm) => vm.id === original.id).account_uuid, 'database-account') + const copy = getVm(root, original.id) + copy.claude.account_uuid = 'mutated-copy' + assert.equal(getVm(root, original.id).claude.account_uuid, 'database-account') + assert.equal(fs.readFileSync(file, 'utf8'), forged) + await repo.upsertFromVmJson({ ...original, name: 'changed-in-database' }) + await catalog.refresh() + assert.equal(getVm(root, original.id).name, 'changed-in-database') + await repo.remove(original.id) + await catalog.refresh() + assert.equal(getVm(root, original.id), null, 'deletion cannot fall back to an old Pod file') + assert.equal(fs.readFileSync(file, 'utf8'), forged) +}) + +test('HA catalog failure refuses cached metadata and does not restore legacy file authority', async (t) => { + const { HaVmCatalog } = await import('../../src/lib/vm/ha-vm-catalog.mjs') + const catalog = new HaVmCatalog(getDb()) + await catalog.refresh() + t.mock.method(catalog.repo, 'list', async () => { + throw new Error('synthetic PostgreSQL unavailable') + }) + await assert.rejects(catalog.refresh(), /PostgreSQL unavailable/) + assert.throws(() => catalog.list(), { code: 'ha_vm_catalog_unavailable' }) + assert.throws(() => catalog.get('ha-catalog-vm'), { code: 'ha_vm_catalog_unavailable' }) +}) From 8f8c606b0b2f4bf51f5c1bf7bd6332259231bbe3 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:36:48 +0800 Subject: [PATCH 358/473] feat: read HA VM metadata exclusively from PostgreSQL --- src/lib/vm/ha-vm-catalog.mjs | 62 ++++++++++++++++++++++++++++++++++++ src/lib/vm/vm-registry.mjs | 7 ++++ 2 files changed, 69 insertions(+) create mode 100644 src/lib/vm/ha-vm-catalog.mjs diff --git a/src/lib/vm/ha-vm-catalog.mjs b/src/lib/vm/ha-vm-catalog.mjs new file mode 100644 index 000000000..28772bf7c --- /dev/null +++ b/src/lib/vm/ha-vm-catalog.mjs @@ -0,0 +1,62 @@ +import { VmsRepo } from '../db/repos/vms-repo.mjs' +import { stripCredentialSecrets } from '../oauth/oauth-credentials.mjs' +import { isCodexVm } from './vm-kind.mjs' +import { isKvmRuntime } from './runtime-kind.mjs' +import { isValidVmId } from './vm-file.mjs' + +function unavailable() { + return Object.assign(new Error('HA VM catalog is unavailable'), { code: 'ha_vm_catalog_unavailable', status: 503 }) +} + +/** HA metadata is a read projection of PostgreSQL. Node credentials never live here. */ +export class HaVmCatalog { + constructor(db) { + this.repo = new VmsRepo(db) + this.records = null + this.pending = Promise.resolve() + } + + refresh() { + const read = async () => { + try { + const rows = await this.repo.list() + const next = new Map() + for (const row of rows) { + const vm = row.vm + if (!vm || vm.id !== row.id || !isValidVmId(vm.id)) throw unavailable() + // This HA preset serves ordinary Claude Docker slots only. + if (isCodexVm(vm) || isKvmRuntime(vm)) continue + next.set(vm.id, { ...vm, claude: stripCredentialSecrets(vm.claude || {}) }) + } + this.records = next + } catch (error) { + this.records = null + throw error + } + } + // Serialize discovery so a slow older SELECT cannot replace a newer snapshot. + this.pending = this.pending.then(read, read) + return this.pending + } + + get(id) { + if (!this.records) throw unavailable() + return structuredClone(this.records.get(id) || null) + } + + list() { + if (!this.records) throw unavailable() + return structuredClone([...this.records.values()]) + } +} + +let bound = null + +export function bindHaVmCatalog(catalog) { + bound = catalog +} + +export function haVmCatalog() { + if (!bound) throw unavailable() + return bound +} diff --git a/src/lib/vm/vm-registry.mjs b/src/lib/vm/vm-registry.mjs index ccf9ea6c1..de4b9c9a0 100644 --- a/src/lib/vm/vm-registry.mjs +++ b/src/lib/vm/vm-registry.mjs @@ -38,10 +38,16 @@ import { effectiveOpenAIPolicy, normalizeOpenAIQuotaPolicy } from '../pool/opena import { runtimeKind } from './runtime-kind.mjs' import { hasBoundExit } from './egress.mjs' import { ensureSlotSmbios } from './machine-spec.mjs' +import { isKubernetesHaMode } from '../core/control-plane-mode.mjs' +import { haVmCatalog } from './ha-vm-catalog.mjs' export { isCodexVm, normalizeVmKind } from './vm-kind.mjs' export function listVms(projectRoot, routing = null) { + if (isKubernetesHaMode()) + return haVmCatalog() + .list() + .map((vm) => summarizeVm(vm, null, routing)) const dir = path.join(projectRoot, 'vms') if (!fs.existsSync(dir)) return [] const files = listVmRecordFiles(dir) @@ -59,6 +65,7 @@ export function listVms(projectRoot, routing = null) { } export function getVm(projectRoot, id) { + if (isKubernetesHaMode()) return haVmCatalog().get(id) return readJsonCached(path.join(projectRoot, 'vms', `${id}.json`)) } From a6dc87aea38db7bc846bfdc7d70f6a69880a208a Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:33:36 +0800 Subject: [PATCH 359/473] test: require trusted node capability capture and typed health --- test/fixtures/fake-slotgate.mjs | 12 ++ .../unit/node-execution-capabilities.test.mjs | 168 ++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 test/unit/node-execution-capabilities.test.mjs diff --git a/test/fixtures/fake-slotgate.mjs b/test/fixtures/fake-slotgate.mjs index 300bf7093..71eec6671 100644 --- a/test/fixtures/fake-slotgate.mjs +++ b/test/fixtures/fake-slotgate.mjs @@ -56,6 +56,18 @@ export async function fakeSlotgate(t, options = {}) { key === 'generation' && typeof value === 'number' ? context.source : value) buffer = buffer.subarray(second + 1) requests.push(command) + if (command.op === 'health') { + const reply = typeof options.health === 'function' ? options.health(command) : options.health + if (options.holdHealth) return + return socket.end(wire(reply || { + ok: true, fence: command.fence, + manifest_revision: command.manifest_revision, manifest_digest: command.manifest_digest, + health: { ok: true, engine: 'rust', healthy: true, ready_slots: 1, + credential: { has_access: true, has_refresh: true, generation: 'credential-1', + expires_at: 1981929600000, account_uuid: 'synthetic-account', + org_uuid: 'synthetic-org', email: 'synthetic@example.invalid' } }, + }) + '\n') + } if (command.op === 'prepare') { commands.set(command.command_id, { ticket: `ticket-${command.command_id}`, command }) return socket.end(JSON.stringify({ ok: true, ticket: `ticket-${command.command_id}` }) + '\n') diff --git a/test/unit/node-execution-capabilities.test.mjs b/test/unit/node-execution-capabilities.test.mjs new file mode 100644 index 000000000..107c0c562 --- /dev/null +++ b/test/unit/node-execution-capabilities.test.mjs @@ -0,0 +1,168 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { SlotgateClient } from '../../src/lib/transport/slotgate-client.mjs' +import { workerHealth, ensureWorkerCredential, callGoWorker } from '../../src/lib/transport/go-worker-client.mjs' +import { clearRustHealthCache, dispatchCallInference } from '../../src/lib/transport/kernel-router.mjs' +import { fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' + +function manifest(config) { + config.manifest = { revision: '1', digest: 'b'.repeat(64) } + return config +} + +function receipt(config) { + return { + intent: { action: 'launch', node_id: config.nodeId, vm_id: config.vmId, + allocation_id: 'synthetic-allocation', operation_id: 'synthetic-operation', + owner_id: 'synthetic-owner', command_id: 'synthetic-launch', epoch: '1', + fence: { generation: 1, incarnation: config.fence.incarnation, executor: { kind: '', node_boot_id: '' } }, + expected_manifest_revision: '1', manifest_revision: '1', payload_digest: '', + preset_id: 'claude-docker-v1', policy_digest: 'c'.repeat(64), manifest_digest: config.manifest.digest }, + intent_digest: 'd'.repeat(64), manifest_revision: '1', manifest_digest: config.manifest.digest, + fence: structuredClone(config.fence), + } +} + +async function capture(config, patch = {}) { + const { captureNodeExecution } = await import('../../src/lib/cluster/node-execution-capabilities.mjs') + return captureNodeExecution({ + db: { schema: 'synthetic_capability' }, vmId: config.vmId, accountId: 'synthetic-account', + trustedNodes: { [config.nodeId]: { ...config.link, presetId: 'claude-docker-v1', policyDigest: 'c'.repeat(64) } }, + readLaunchReceipt: async () => receipt(config), ...patch, + }) +} + +test('capture obtains the original ACTIVE fence from the PG receipt and confirms the same node account', async (t) => { + const fx = await fakeSlotgate(t) + manifest(fx.config) + const result = await capture(fx.config) + assert.deepEqual(result.slotgate.fence, syntheticFence()) + assert.equal(result.workerStatus.ok, true) + assert.equal(result.accountId, 'synthetic-account') + assert.equal(result.nodeManifest.epoch, '1') + assert.equal(Object.isFrozen(result.slotgate.link), true) + assert.equal(Object.isFrozen(result.slotgate.fence.executor), true) + assert.deepEqual(fx.requests.map((value) => value.op), ['health']) + assert.equal(fx.requests[0].manifest_revision, '1') + assert.equal(fx.requests[0].manifest_digest, 'b'.repeat(64)) +}) + +test('capture rejects missing PG proof and untrusted links before a node request', async (t) => { + const fx = await fakeSlotgate(t) + manifest(fx.config) + await assert.rejects(capture(fx.config, { readLaunchReceipt: async () => null }), { code: 'node_capability_unavailable' }) + await assert.rejects(capture(fx.config, { trustedNodes: {} }), { code: 'node_capability_unavailable' }) + await assert.rejects(capture(fx.config, { nodeId: 'other-node' }), { code: 'node_capability_unavailable' }) + await assert.rejects(capture(fx.config, { trustedNodes: { [fx.config.nodeId]: { ...fx.config.link, presetId: 'codex', policyDigest: 'c'.repeat(64) } } }), { code: 'node_capability_unavailable' }) + assert.equal(fx.requests.length, 0) +}) + +test('capture does not adopt a fresh PG receipt after its original node health check', async (t) => { + const fx = await fakeSlotgate(t) + manifest(fx.config) + let reads = 0 + await assert.rejects(capture(fx.config, { readLaunchReceipt: async () => { + const value = receipt(fx.config) + if (++reads > 1) value.intent.epoch = '2' + return value + } }), { code: 'node_capability_changed' }) + assert.equal(reads, 2) + assert.deepEqual(fx.requests.map((value) => value.op), ['health']) +}) + +test('capture refuses a different account without preparing inference', async (t) => { + const fx = await fakeSlotgate(t) + manifest(fx.config) + await assert.rejects(capture(fx.config, { accountId: 'other-account' }), { code: 'node_account_mismatch' }) + assert.deepEqual(fx.requests.map((value) => value.op), ['health']) +}) + +test('typed health returns only public fields and never needs a Pod kernel token', async (t) => { + const fx = await fakeSlotgate(t, { health: (command) => ({ + ok: true, fence: command.fence, manifest_revision: command.manifest_revision, manifest_digest: command.manifest_digest, + health: { ok: true, engine: 'rust', ready_slots: 2, healthy: true, internal_token: 'must-not-escape', + credential: { has_access: true, has_refresh: true, generation: 'g1', expires_at: 1981929600000, + account_uuid: 'synthetic-account', org_uuid: 'synthetic-org', email: 'synthetic@example.invalid', + access_token: 'must-not-escape', refresh_token: 'must-not-escape', provider_extension: { secret: 'must-not-escape' } } }, + }) }) + const config = manifest(fx.config) + const value = await workerHealth({ vmId: config.vmId, slotgate: config }) + assert.equal(value.ok, true) + assert.equal(value.credential.account_uuid, 'synthetic-account') + assert.equal(JSON.stringify(value).includes('must-not-escape'), false) + assert.deepEqual(fx.requests.map((entry) => entry.op), ['health']) +}) + +for (const kind of ['fence', 'manifest', 'missing']) { + test(`typed health rejects ${kind} proof and never prepares inference`, async (t) => { + const fx = await fakeSlotgate(t, { health: (command) => kind === 'missing' ? { ok: true, health: { ok: true } } : { + ok: true, fence: kind === 'fence' ? syntheticFence('2') : command.fence, + manifest_revision: kind === 'manifest' ? '2' : command.manifest_revision, + manifest_digest: command.manifest_digest, health: { ok: true, healthy: true, ready_slots: 1 }, + } }) + manifest(fx.config) + const value = await workerHealth({ vmId: fx.config.vmId, slotgate: fx.config }) + assert.equal(value.ok, false) + assert.equal(value.code, 'slotgate_health_unconfirmed') + assert.deepEqual(fx.requests.map((entry) => entry.op), ['health']) + }) +} + +test('typed health observes cancellation without a legacy cancel request', async (t) => { + const fx = await fakeSlotgate(t, { holdHealth: true }) + const client = new SlotgateClient(manifest(fx.config)) + const controller = new AbortController() + const pending = client.health({ signal: controller.signal, timeoutMs: 1000 }) + setTimeout(() => controller.abort(), 40) + await assert.rejects(pending, { code: 'ABORT_ERR' }) + assert.equal(fx.requests.every((value) => value.op === 'health'), true) +}) + +test('fenced router never creates Pod credential links or reads a host kernel binary', async (t) => { + clearRustHealthCache() + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-node-router-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const fx = await fakeSlotgate(t) + const config = manifest(fx.config) + const homeDir = path.join(root, 'forbidden-home') + const result = await dispatchCallInference({ + exec: { vmId: config.vmId, homeDir, slotgate: config, vm: { id: config.vmId } }, + envelope: { body: {} }, + onExecutionStart: async (value) => ({ bound: true, requestId: value.requestId, nodeBinding: value.nodeBinding }), + }) + assert.equal(result.ok, true) + assert.equal(result.execution.ended, true) + assert.equal(fs.existsSync(homeDir), false) + assert.deepEqual(fx.requests.map((value) => value.op), ['health', 'prepare', 'forward', 'command']) +}) + +test('fenced credential ensure fails before touching a Pod credential file', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-node-credential-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const fx = await fakeSlotgate(t) + const homeDir = path.join(root, 'forbidden-home') + const result = await ensureWorkerCredential({ vmId: fx.config.vmId, homeDir, slotgate: manifest(fx.config) }) + assert.equal(result.ok, false) + assert.equal(result.error.code, 'slotgate_route_unavailable') + assert.equal(result.credentialRefreshNotStarted, true) + assert.equal(fs.existsSync(homeDir), false) + assert.equal(fx.requests.length, 0) +}) + +test('HA transport refuses a missing trusted capability before mock or legacy execution', async () => { + const before = process.env.VM2API_CONTROL_PLANE_MODE + const beforeMock = process.env.KIN_CRS_MOCK + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + process.env.KIN_CRS_MOCK = '1' + try { + await assert.rejects(callGoWorker({ exec: { vmId: 'legacy-vm' }, envelope: { body: {} } }), { code: 'slotgate_capability_required' }) + } finally { + if (before == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = before + if (beforeMock == null) delete process.env.KIN_CRS_MOCK + else process.env.KIN_CRS_MOCK = beforeMock + } +}) From ebecaff7f55dced41a71fdaead4907f4ffc650fa Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:36:07 +0800 Subject: [PATCH 360/473] test: pin trusted TLS settings before asynchronous capability capture --- .../unit/node-execution-capabilities.test.mjs | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/test/unit/node-execution-capabilities.test.mjs b/test/unit/node-execution-capabilities.test.mjs index 107c0c562..dcdf6f2c0 100644 --- a/test/unit/node-execution-capabilities.test.mjs +++ b/test/unit/node-execution-capabilities.test.mjs @@ -4,8 +4,8 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { SlotgateClient } from '../../src/lib/transport/slotgate-client.mjs' -import { workerHealth, ensureWorkerCredential, callGoWorker } from '../../src/lib/transport/go-worker-client.mjs' -import { clearRustHealthCache, dispatchCallInference } from '../../src/lib/transport/kernel-router.mjs' +import { workerHealth, ensureWorkerCredential, callGoWorker, streamGoWorker, callWorkerGet, countTokensViaWorker } from '../../src/lib/transport/go-worker-client.mjs' +import { clearRustHealthCache, dispatchCallInference, rememberRustHealth, peekRustHealth } from '../../src/lib/transport/kernel-router.mjs' import { fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' function manifest(config) { @@ -80,6 +80,58 @@ test('capture refuses a different account without preparing inference', async (t assert.deepEqual(fx.requests.map((value) => value.op), ['health']) }) +test('capture snapshots fixed TLS configuration before awaiting the PG receipt', async (t) => { + const fx = await fakeSlotgate(t) + manifest(fx.config) + const trustedNodes = new Map([[fx.config.nodeId, { ...fx.config.link, presetId: 'claude-docker-v1', policyDigest: 'c'.repeat(64) }]]) + let resume + const blocked = new Promise((resolve) => { resume = resolve }) + let reads = 0 + const pending = capture(fx.config, { trustedNodes, readLaunchReceipt: async () => { + if (++reads === 1) await blocked + return receipt(fx.config) + } }) + // Wait until capture has entered its original lookup, not until node IO. + await new Promise((resolve) => setImmediate(resolve)) + trustedNodes.get(fx.config.nodeId).certSha256 = '0'.repeat(64) + resume() + const result = await pending + assert.equal(result.slotgate.link.certSha256, fx.config.link.certSha256) +}) + +for (const account of [undefined, null, '', 'other-account']) { + test(`capture does not authorize account by display email or conflicting identity (${account})`, async (t) => { + const fx = await fakeSlotgate(t, { health: (command) => ({ + ok: true, fence: command.fence, manifest_revision: command.manifest_revision, manifest_digest: command.manifest_digest, + health: { ok: true, credential: { has_access: true, account_uuid: account, email: 'synthetic-account' }, + identity: { account_uuid: 'synthetic-account' } }, + }) }) + manifest(fx.config) + await assert.rejects(capture(fx.config), { code: 'node_account_mismatch' }) + assert.deepEqual(fx.requests.map((value) => value.op), ['health']) + }) +} + +test('a PG receipt failure has no TLS or legacy fallback', async (t) => { + const fx = await fakeSlotgate(t) + manifest(fx.config) + await assert.rejects(capture(fx.config, { readLaunchReceipt: async () => { throw Object.assign(new Error('synthetic PG failure'), { code: 'synthetic_pg_failure' }) } }), { code: 'synthetic_pg_failure' }) + assert.equal(fx.requests.length, 0) +}) + +test('health cache separates captured fences and manifests and VM invalidation clears each', async () => { + clearRustHealthCache() + const first = { vmId: 'cache-vm', slotgate: { fence: syntheticFence(), manifest: { revision: '1', digest: 'a'.repeat(64) } } } + rememberRustHealth(first, { ok: true, health: { ok: true } }) + assert.equal(peekRustHealth(first, 1000).health.ok, true) + const next = { ...first, slotgate: { ...first.slotgate, fence: syntheticFence('2') } } + assert.equal(peekRustHealth(next, 1000), null) + const changedManifest = { ...first, slotgate: { ...first.slotgate, manifest: { revision: '2', digest: 'b'.repeat(64) } } } + assert.equal(peekRustHealth(changedManifest, 1000), null) + clearRustHealthCache('cache-vm') + assert.equal(peekRustHealth(first, 1000), null) +}) + test('typed health returns only public fields and never needs a Pod kernel token', async (t) => { const fx = await fakeSlotgate(t, { health: (command) => ({ ok: true, fence: command.fence, manifest_revision: command.manifest_revision, manifest_digest: command.manifest_digest, @@ -159,6 +211,9 @@ test('HA transport refuses a missing trusted capability before mock or legacy ex process.env.KIN_CRS_MOCK = '1' try { await assert.rejects(callGoWorker({ exec: { vmId: 'legacy-vm' }, envelope: { body: {} } }), { code: 'slotgate_capability_required' }) + await assert.rejects(streamGoWorker({ exec: { vmId: 'legacy-vm' }, envelope: { body: {} } }), { code: 'slotgate_capability_required' }) + await assert.rejects(callWorkerGet({ vmId: 'legacy-vm' }, '/internal/v1/models'), { code: 'slotgate_capability_required' }) + await assert.rejects(countTokensViaWorker({ vmId: 'legacy-vm' }, { body: {} }), { code: 'slotgate_capability_required' }) } finally { if (before == null) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = before From 11c7606659371db35971bad331d08c2256b84380 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:37:34 +0800 Subject: [PATCH 361/473] feat: capture trusted node capabilities and require fenced health --- .../2026-10-22-node-capability-contract.md | 107 ++++++++ .../cluster/node-execution-capabilities.mjs | 130 ++++++++++ src/lib/transport/go-worker-client.mjs | 63 ++++- src/lib/transport/kernel-router.mjs | 27 +- src/lib/transport/slotgate-client.mjs | 113 ++++++++- test/fixtures/fake-slotgate.mjs | 2 +- .../unit/node-execution-capabilities.test.mjs | 237 ++++++++++++++---- test/unit/slotgate-client.test.mjs | 2 +- 8 files changed, 614 insertions(+), 67 deletions(-) create mode 100644 docs/plans/2026-10-22-node-capability-contract.md create mode 100644 src/lib/cluster/node-execution-capabilities.mjs diff --git a/docs/plans/2026-10-22-node-capability-contract.md b/docs/plans/2026-10-22-node-capability-contract.md new file mode 100644 index 000000000..47f2112e1 --- /dev/null +++ b/docs/plans/2026-10-22-node-capability-contract.md @@ -0,0 +1,107 @@ +# Trusted node execution capability contract + +This change supplies a production reader and transport boundary for the explicit +`kubernetes-ha` mode. It does not enable that mode, start nodes, or provide the +Go health RPC or node credential refresh. Native, KVM and Codex HA are unsupported. +The existing standalone and singleton Kubernetes paths retain their behavior. + +## Capture + +`captureNodeExecution({db, vmId, accountId, nodeId?, trustedNodes, +readLaunchReceipt?, timeoutMs?, signal?})` returns a deeply frozen internal object: + +```js +{ + vmId, accountId, + slotgate: { + trusted: true, pgBindingReady: true, nodeId, vmId, fence, + manifest: {revision, digest}, + link: {address, port, token, certSha256}, + }, + nodeManifest: {revision, digest, presetId, policyDigest, epoch}, + workerStatus, +} +``` + +The link token is an internal transport credential. This object is not a public +panel response or a serializable request-log payload. + +`trustedNodes` is a bootstrap-installed, fixed Map or object keyed by node ID. +Each entry contains `{address, port, token, certSha256, +presetId: 'claude-docker-v1', policyDigest}`. The reader snapshots this configuration +before its first asynchronous lookup. No endpoint, pin, token or fence comes from +VM JSON. `nodeId`, when supplied, only constrains the independently read receipt. + +By default the reader calls `NodeManagementGrants.readLaunchReceipt({vmId,nodeId?})`. +That method must verify the private allocation/publish/launch chain against the +current maintenance epoch and return the original Go launch receipt. A completed +management grant need not remain authorized. The caller already owns a database +request/bootstrap/background activity. Synthetic tests inject only this reader; +they do not stand in for real PG grant-chain verification. + +Only the ordinary Claude Docker preset and the configured policy digest are +accepted. The ACTIVE fence is copied from `launchReceipt.fence`, including the +immutable container, Docker daemon, node boot and StartedAt identity. The original +VACANT `intent.fence` is never used for inference. The reader does not obtain a +fresh fence from node `state` or retry an old operation with a newer generation. + +Capture awaits authenticated node health for the original fence and manifest. +Only `health.credential.account_uuid` authorizes `accountId`; email or a separate +identity field cannot replace it. If `health.identity.account_uuid` is present it +must agree. After health, the reader reads the PG receipt again and rejects any +epoch/receipt change. No PG transaction or grant lock spans the TLS operation. +The returned historical receipt alone does not prove that an executor is running. + +The server and scheduler must use this capture before candidate admission, reuse +the returned `workerStatus`, and durably verify/bind the same receipt and node +identity before submitting inference. Those integrations are outside this change. + +## Typed health RPC + +The existing pinned TLS link carries one `KSG2 ` preface and JSON request: + +```js +{op: 'health', vm_id, fence, manifest_revision, manifest_digest} +``` + +The reply is: + +```js +{ok: true, fence, manifest_revision, manifest_digest, health: { + ok, healthy, recovering, ready_slots, closed_slots, cli_restarts, cli_pid, + engine, provider, worker_version, version, unhealthy_reason, + classifier_request_context, + credential: {has_access, has_refresh, needs_refresh, expires_at, generation, + account_uuid, org_uuid, email}, + identity: {account_uuid, org_uuid, email}, +}} +``` + +Optional public fields retain their validated types; unknown fields and nested +provider extensions are omitted. The client requires the exact captured fence +and manifest acknowledgement. Abort, timeout, invalid JSON, missing proof or an +unknown RPC never fall back to KSG1, UDS, SSH, Docker or an injected legacy ensure. + +The Go implementation must verify the captured ACTIVE fence and manifest, read +only the fixed kernel health route and node credential summary, and perform no +implicit refresh or external provider request. Health does not create an inference +command and must not use `prepare`/`forward` for a GET. The existing normal Claude +terminal observer only finishes the supported Messages protocol. + +## Transport boundary and remaining work + +HA transport requires a capability before mocks or legacy execution. Fenced +preparation skips host kernel binary checks, Pod credential links and local +recycling. Health caches separate fences and manifests. For published manifests, +the node owns kernel-token injection; the Pod does not read an internal-token file. + +Until separately implemented, fenced credential ensure/import, auxiliary worker +GET and count-tokens return an explicit unsupported result before local IO or a +legacy call. Credential ensure identifies this as not started. A future typed +node refresh must preserve the node-owned generation and terminal receipt; it +must not reintroduce Pod credential publication or SFTP mirroring. + +The HA startup path, PG-only VM catalog, management capability matrix, background +producer gates, real Go health RPC and two active Pod verification are separate +required integrations. This change's TLS tests prove only the capability/client +contract and transport isolation. No production node or provider is used. diff --git a/src/lib/cluster/node-execution-capabilities.mjs b/src/lib/cluster/node-execution-capabilities.mjs new file mode 100644 index 000000000..ff504a718 --- /dev/null +++ b/src/lib/cluster/node-execution-capabilities.mjs @@ -0,0 +1,130 @@ +import { isDeepStrictEqual } from 'node:util' +import { getDb } from '../db/database.mjs' +import { NodeManagementGrants } from './node-management-grants.mjs' +import { SlotgateClient } from '../transport/slotgate-client.mjs' +import { normalizeNodeFence } from '../core/node-execution-proof.mjs' + +function failure(code = 'node_capability_unavailable') { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code, status: 503 }) +} + +function freeze(value) { + for (const item of Object.values(value)) if (item && typeof item === 'object') freeze(item) + return Object.freeze(value) +} + +function decimal(value) { + return typeof value === 'string' && /^[1-9][0-9]{0,19}$/.test(value) && BigInt(value) <= 18446744073709551615n +} + +/** + * Capture an independently verified PG launch receipt, never a VM JSON fence. + * trustedNodes is the fixed, bootstrap-installed TLS/policy configuration. + * The caller already owns a request/bootstrap/background database activity. + */ +export async function captureNodeExecution({ + db = getDb(), + vmId, + accountId, + nodeId, + trustedNodes, + readLaunchReceipt, + timeoutMs = 3000, + signal, +}) { + if ( + typeof vmId !== 'string' || + !/^[A-Za-z0-9_-]{1,128}$/.test(vmId) || + typeof accountId !== 'string' || + !accountId || + accountId.trim() !== accountId + ) + throw failure() + // The fixed trust configuration is captured before the first asynchronous + // lookup, so an in-flight operation cannot inherit a later endpoint or pin. + const nodes = structuredClone(trustedNodes) + const read = readLaunchReceipt || ((input) => new NodeManagementGrants(db).readLaunchReceipt(input)) + // This method validates the private allocation -> publish -> launch chain + // against its current maintenance epoch. It does not read live node state. + const original = await read({ vmId, ...(nodeId === undefined ? {} : { nodeId }) }) + if (!original) throw failure() + const receipt = structuredClone(original) + const intent = receipt.intent + const fixed = + nodes instanceof Map + ? nodes.get(intent?.node_id) + : nodes && Object.hasOwn(nodes, intent?.node_id) + ? nodes[intent.node_id] + : null + if (!fixed) throw failure() + const configured = structuredClone(fixed) + if ( + intent?.action !== 'launch' || + intent.vm_id !== vmId || + (nodeId !== undefined && intent.node_id !== nodeId) || + intent.preset_id !== 'claude-docker-v1' || + configured.presetId !== intent.preset_id || + !/^[a-f0-9]{64}$/.test(intent.policy_digest) || + configured.policyDigest !== intent.policy_digest || + !/^[a-f0-9]{64}$/.test(receipt.intent_digest) || + !decimal(intent.epoch) || + !decimal(receipt.manifest_revision) || + receipt.manifest_revision !== intent.manifest_revision || + receipt.manifest_revision !== intent.expected_manifest_revision || + !/^[a-f0-9]{64}$/.test(receipt.manifest_digest) || + receipt.manifest_digest !== intent.manifest_digest + ) + throw failure() + let fence + try { + fence = normalizeNodeFence(receipt.fence) + } catch { + throw failure() + } + const slotgate = { + trusted: true, + pgBindingReady: true, + nodeId: intent.node_id, + vmId, + fence, + manifest: { revision: receipt.manifest_revision, digest: receipt.manifest_digest }, + link: { + address: configured.address, + port: configured.port, + token: configured.token, + certSha256: configured.certSha256, + }, + } + let client + try { + client = new SlotgateClient(slotgate) + } catch { + throw failure() + } + const workerStatus = await client.health({ timeoutMs, signal }) + if (workerStatus.ok !== true) throw failure('node_health_unavailable') + // Email is display metadata. Only the node-owned credential's account UUID + // may authorize the scheduler's account, and conflicting identity is denied. + if ( + workerStatus.credential?.account_uuid !== accountId || + (workerStatus.identity?.account_uuid !== undefined && workerStatus.identity.account_uuid !== accountId) + ) + throw failure('node_account_mismatch') + // A response to the old fence must not be relabeled after a concurrent PG + // epoch/receipt change. No transaction or grant lock spans the TLS request. + const after = await read({ vmId, ...(nodeId === undefined ? {} : { nodeId }) }) + if (!isDeepStrictEqual(receipt, after)) throw failure('node_capability_changed') + return freeze({ + vmId, + accountId, + slotgate, + nodeManifest: { + revision: receipt.manifest_revision, + digest: receipt.manifest_digest, + presetId: intent.preset_id, + policyDigest: intent.policy_digest, + epoch: intent.epoch, + }, + workerStatus, + }) +} diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index 6c7c32952..f6ff9aab2 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -35,6 +35,12 @@ import { import { extraHeadersFromLimitError } from '../pool/quota-window.mjs' import { SlotgateClient } from './slotgate-client.mjs' import { credentialGeneration } from '../oauth/shared-credential-refresh.mjs' +import { isKubernetesHaMode } from '../core/control-plane-mode.mjs' + +function requireNodeCapability(exec) { + if (isKubernetesHaMode() && exec?.slotgate == null) + throw Object.assign(new Error('trusted node capability required'), { code: 'slotgate_capability_required' }) +} const MAX_BODY = 64 * 1024 * 1024 const credentialRefreshTail = new Map() @@ -89,6 +95,7 @@ function workerRequest( return Promise.reject( Object.assign(new Error('fenced worker route unavailable'), { code: 'slotgate_route_unavailable' }), ) + requireNodeCapability(exec) return new Promise((resolve, reject) => { const { socketPath } = workerPaths(exec) if (!socketPath) { @@ -376,7 +383,9 @@ async function completeFencedExecution(client, handle, execution) { } function forwardFencedWorker(exec, client, handle, request) { - const token = readInternalToken(exec) + // Published manifests keep kernel authentication on the trusted node. Older + // explicitly injected test capabilities retain their legacy token contract. + const token = client.manifest ? '' : readInternalToken(exec) return client.forward(handle, { ...request, headers: { @@ -594,8 +603,9 @@ export async function callGoWorker({ requestId = randomUUID(), onExecutionStart, } = {}) { + requireNodeCapability(exec) const execution = { requestId, started: false, ended: true, reason: isCrsMock() ? 'mock' : 'not_started' } - if (isCrsMock()) { + if (isCrsMock() && exec?.slotgate == null) { const { body: outboundBody, headers } = finalizeWorkerPayload({ body, reqHeaders, @@ -738,8 +748,9 @@ export async function streamGoWorker({ requestId = randomUUID(), onExecutionStart, } = {}) { + requireNodeCapability(exec) const execution = { requestId, started: false, ended: true, reason: isCrsMock() ? 'mock' : 'not_started' } - if (isCrsMock()) { + if (isCrsMock() && exec?.slotgate == null) { const { body: outboundBody, headers } = finalizeWorkerPayload({ body, reqHeaders, @@ -1085,6 +1096,14 @@ export async function streamGoWorker({ } export async function workerHealth(exec, { timeoutMs = 3000, signal } = {}) { + requireNodeCapability(exec) + if (exec?.slotgate != null) { + try { + return await new SlotgateClient(exec.slotgate).health({ timeoutMs, signal }) + } catch (error) { + return { ok: false, status: 0, code: error.code || 'slotgate_health_unconfirmed' } + } + } if (isCrsMock()) { const claude = exec?.vm?.claude || {} const hasCredential = hasCredentialPresence(claude) @@ -1121,6 +1140,15 @@ export async function workerHealth(exec, { timeoutMs = 3000, signal } = {}) { } export async function ensureWorkerCredential(exec, { force = false, ...refreshOptions } = {}) { + requireNodeCapability(exec) + if (exec?.slotgate != null) + return { + ok: false, + status: 503, + refreshed: false, + credentialRefreshNotStarted: true, + error: { code: 'slotgate_route_unavailable', message: 'Node credential refresh is not available' }, + } if (isCrsMock()) { return { ok: true, @@ -1254,6 +1282,13 @@ function credentialSummary(credential, now = Date.now()) { } export async function importWorkerCredential(exec, credential, { timeoutMs = 60000, signal } = {}) { + requireNodeCapability(exec) + if (exec?.slotgate != null) + return { + ok: false, + status: 503, + error: { code: 'slotgate_route_unavailable', message: 'Node credential import is not available' }, + } try { writeWorkerCredentialFile(exec.homeDir, credential) // Import is the one case where the control plane's credential replaces the slot's copy. @@ -1278,6 +1313,17 @@ export async function importWorkerCredential(exec, credential, { timeoutMs = 600 } export async function callWorkerGet(exec, requestPath, { timeoutMs = 30000, signal } = {}) { + requireNodeCapability(exec) + if (exec?.slotgate != null) + return { + ok: false, + status: 503, + body: { error: { code: 'slotgate_route_unavailable', message: 'Node auxiliary route is not available' } }, + headers: {}, + via: 'slotgate', + attemptFinished: true, + taskNotStarted: true, + } if (isCrsMock()) { if (requestPath === '/internal/v1/models') { return { @@ -1370,6 +1416,17 @@ export async function callWorkerGet(exec, requestPath, { timeoutMs = 30000, sign } export async function countTokensViaWorker(exec, { body, headers = {}, timeoutMs = 45000 } = {}) { + requireNodeCapability(exec) + if (exec?.slotgate != null) + return { + ok: false, + status: 503, + body: { error: { code: 'slotgate_route_unavailable', message: 'Node count tokens is not available' } }, + headers: {}, + via: 'slotgate', + attemptFinished: true, + taskNotStarted: true, + } if (isCrsMock()) { return { ok: true, status: 200, body: { input_tokens: 8 }, headers: {}, via: 'go-worker-mock' } } diff --git a/src/lib/transport/kernel-router.mjs b/src/lib/transport/kernel-router.mjs index 1b964e17c..a77274f8d 100644 --- a/src/lib/transport/kernel-router.mjs +++ b/src/lib/transport/kernel-router.mjs @@ -8,6 +8,7 @@ import { randomUUID } from 'node:crypto' import { ensureWorkerCredential } from './go-worker-client.mjs' import { ensureOfficialCredentialLink, slotUidGidFromHomeDir } from '../oauth/oauth-credentials.mjs' import { slotHost } from '../vm/slot-host.mjs' +import { isKubernetesHaMode } from '../core/control-plane-mode.mjs' import { streamRustKernel, callRustKernel, @@ -81,12 +82,15 @@ export async function waitForReadySlot(exec, timeoutMs = DEFAULT_SLOT_WAIT_MS, p } export function clearRustHealthCache(vmId = null) { - if (vmId) rustHealthCache.delete(String(vmId)) - else rustHealthCache.clear() + if (vmId) { + for (const key of rustHealthCache.keys()) + if (key === String(vmId) || key.startsWith(`${vmId}:`)) rustHealthCache.delete(key) + } else rustHealthCache.clear() } function cacheKey(exec) { - return String(exec?.vmId || exec?.vm?.id || '') + const id = String(exec?.vmId || exec?.vm?.id || '') + return exec?.slotgate ? `${id}:${JSON.stringify(exec.slotgate.fence)}:${JSON.stringify(exec.slotgate.manifest)}` : id } export function rememberRustHealth(exec, ready) { @@ -188,6 +192,16 @@ async function prepareSlotCredentials(exec) { } async function prepareRust(exec, { ensure, routing, slotWaitMs } = {}) { + if (exec?.slotgate != null) { + const health = await rustKernelHealth(exec, { timeoutMs: 800 }) + // The node validates the captured fence and manifest. Never use the local + // credential, host binary/recycler or an injected legacy ensure callback. + return rustKernelReachable(health) + ? { ok: true, reason: 'node_health', health } + : { ok: false, reason: 'kernel_unavailable', health, error: health?.code || 'node health unavailable' } + } + if (isKubernetesHaMode()) + throw Object.assign(new Error('trusted node capability required'), { code: 'slotgate_capability_required' }) await awaitWrapRecycle(exec) const fault = kernelFaults.get(cacheKey(exec)) if (fault) return { ok: false, reason: 'kernel_unavailable', error: fault } @@ -226,8 +240,13 @@ async function prepareRust(exec, { ensure, routing, slotWaitMs } = {}) { } async function runHop({ mode, opts }) { + if (isKubernetesHaMode() && opts.exec?.slotgate == null) + throw Object.assign(new Error('trusted node capability required'), { code: 'slotgate_capability_required' }) const routing = opts.routing || {} - const decision = resolveHopEngine(opts.exec?.vm, routing) + const decision = + opts.exec?.slotgate != null + ? { engine: 'rust', wanted: 'rust', reason: 'node_capability', fallback: false } + : resolveHopEngine(opts.exec?.vm, routing) let engine = 'rust' let reason = decision.reason if (!decision.blocked) { diff --git a/src/lib/transport/slotgate-client.mjs b/src/lib/transport/slotgate-client.mjs index b08df56ea..028a94731 100644 --- a/src/lib/transport/slotgate-client.mjs +++ b/src/lib/transport/slotgate-client.mjs @@ -26,6 +26,67 @@ function parse(value) { ) } +function publicHealth(value) { + if (!value || typeof value !== 'object' || typeof value.ok !== 'boolean') throw failure('slotgate_health_unconfirmed') + const out = {} + const copy = (source, target, booleans, numbers, strings) => { + for (const key of booleans) + if (Object.hasOwn(source, key)) { + if (typeof source[key] !== 'boolean') throw failure('slotgate_health_unconfirmed') + target[key] = source[key] + } + for (const key of numbers) + if (Object.hasOwn(source, key)) { + if (!Number.isSafeInteger(source[key]) || source[key] < 0) throw failure('slotgate_health_unconfirmed') + target[key] = source[key] + } + for (const key of strings) + if (Object.hasOwn(source, key)) { + if ( + source[key] !== null && + (typeof source[key] !== 'string' || source[key].length > 512 || /[\x00-\x1f\x7f]/.test(source[key])) + ) + throw failure('slotgate_health_unconfirmed') + target[key] = source[key] + } + } + copy( + value, + out, + ['ok', 'healthy', 'recovering', 'classifier_request_context'], + ['ready_slots', 'closed_slots', 'cli_restarts', 'cli_pid'], + ['engine', 'provider', 'worker_version', 'version', 'unhealthy_reason'], + ) + if (value.credential !== undefined) { + if (!value.credential || typeof value.credential !== 'object') throw failure('slotgate_health_unconfirmed') + out.credential = {} + copy( + value.credential, + out.credential, + ['has_access', 'has_refresh', 'needs_refresh'], + [], + ['account_uuid', 'org_uuid', 'email'], + ) + for (const key of ['generation', 'expires_at']) + if (Object.hasOwn(value.credential, key)) { + const item = value.credential[key] + if ( + item !== null && + !(typeof item === 'string' && item.length > 0 && item.length <= 128 && !/[\x00-\x1f\x7f]/.test(item)) && + !(Number.isSafeInteger(item) && item >= 0) + ) + throw failure('slotgate_health_unconfirmed') + out.credential[key] = item + } + } + if (value.identity !== undefined) { + if (!value.identity || typeof value.identity !== 'object') throw failure('slotgate_health_unconfirmed') + out.identity = {} + copy(value.identity, out.identity, [], [], ['account_uuid', 'org_uuid', 'email']) + } + return out +} + // The gate reads the full request before replying OK. This Duplex sends HTTP // immediately, while consuming only the bounded gate prefix before HTTP parses. class ForwardSocket extends Duplex { @@ -126,6 +187,17 @@ export class SlotgateClient { ) throw failure('slotgate_capability_invalid') this.link = freeze({ address: link.address, port: link.port, token: link.token, certSha256: link.certSha256 }) + if (config.manifest !== undefined) { + if ( + !config.manifest || + !/^[1-9][0-9]{0,19}$/.test(config.manifest.revision) || + typeof config.manifest.revision !== 'string' || + BigInt(config.manifest.revision) > 18446744073709551615n || + !/^[a-f0-9]{64}$/.test(config.manifest.digest) + ) + throw failure('slotgate_capability_invalid') + this.manifest = freeze({ revision: config.manifest.revision, digest: config.manifest.digest }) + } } catch { throw failure('slotgate_capability_invalid') } @@ -157,8 +229,9 @@ export class SlotgateClient { return value } - _connect(timeoutMs) { + _connect(timeoutMs, signal) { return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(failure('ABORT_ERR')) let settled = false const socket = tls.connect({ host: this.link.address, @@ -171,10 +244,13 @@ export class SlotgateClient { if (settled) return settled = true clearTimeout(timer) + signal?.removeEventListener('abort', abort) socket.destroy() reject(error) } const timer = setTimeout(() => fail(failure('slotgate_timeout')), timeoutMs) + const abort = () => fail(failure('ABORT_ERR')) + signal?.addEventListener('abort', abort, { once: true }) socket.once('error', fail) socket.once('close', () => fail(failure('slotgate_disconnected'))) socket.once('secureConnect', () => { @@ -182,6 +258,7 @@ export class SlotgateClient { if (pin !== this.link.certSha256) return fail(failure('slotgate_cert_mismatch')) settled = true clearTimeout(timer) + signal?.removeEventListener('abort', abort) socket.off('error', fail) // Install an error observer until the protocol consumer attaches. socket.on('error', () => {}) @@ -190,8 +267,8 @@ export class SlotgateClient { }) } - async _rpc(command, timeoutMs = 3000) { - const socket = await this._connect(timeoutMs) + async _rpc(command, timeoutMs = 3000, signal) { + const socket = await this._connect(timeoutMs, signal) return new Promise((resolve, reject) => { let settled = false let buffer = Buffer.alloc(0) @@ -199,12 +276,16 @@ export class SlotgateClient { if (settled) return settled = true clearTimeout(timer) + signal?.removeEventListener('abort', abort) socket.removeAllListeners('data') socket.destroy() if (error) reject(error) else resolve(value) } const timer = setTimeout(() => done(failure('slotgate_timeout')), timeoutMs) + const abort = () => done(failure('ABORT_ERR')) + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted) return abort() socket.once('error', (error) => done(error)) socket.once('end', () => done(failure('slotgate_bad_reply'))) socket.once('close', () => done(failure('slotgate_disconnected'))) @@ -225,6 +306,32 @@ export class SlotgateClient { }) } + async health({ timeoutMs = 3000, signal } = {}) { + if (!this.manifest) throw failure('slotgate_route_unavailable') + const reply = await this._rpc( + { + op: 'health', + vm_id: this.nodeBinding.vmId, + fence: this.nodeBinding.fence, + manifest_revision: this.manifest.revision, + manifest_digest: this.manifest.digest, + }, + timeoutMs, + signal, + ) + try { + if ( + !sameNodeBinding({ ...this.nodeBinding, fence: reply.fence }, this.nodeBinding) || + reply.manifest_revision !== this.manifest.revision || + reply.manifest_digest !== this.manifest.digest + ) + throw failure('slotgate_health_unconfirmed') + return { ...publicHealth(reply.health), status: 200, source: 'slotgate' } + } catch { + throw failure('slotgate_health_unconfirmed') + } + } + async prepare(requestId) { if ( typeof requestId !== 'string' || diff --git a/test/fixtures/fake-slotgate.mjs b/test/fixtures/fake-slotgate.mjs index 71eec6671..61741d3c1 100644 --- a/test/fixtures/fake-slotgate.mjs +++ b/test/fixtures/fake-slotgate.mjs @@ -23,7 +23,7 @@ export const assistant = { id: 'synthetic-message', type: 'message', role: 'assistant', content: [{ type: 'text', text: 'synthetic answer' }], stop_reason: 'end_turn', } -const wire = (value) => JSON.stringify(value, (key, item) => key === 'generation' ? JSON.rawJSON(String(item)) : item) +const wire = (value) => JSON.stringify(value, (key, item) => key === 'generation' && /^[1-9][0-9]*$/.test(String(item)) ? JSON.rawJSON(String(item)) : item) // A local protocol double, not a node/Docker or provider acceptance test. export async function fakeSlotgate(t, options = {}) { diff --git a/test/unit/node-execution-capabilities.test.mjs b/test/unit/node-execution-capabilities.test.mjs index dcdf6f2c0..aa1350e3f 100644 --- a/test/unit/node-execution-capabilities.test.mjs +++ b/test/unit/node-execution-capabilities.test.mjs @@ -4,8 +4,20 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { SlotgateClient } from '../../src/lib/transport/slotgate-client.mjs' -import { workerHealth, ensureWorkerCredential, callGoWorker, streamGoWorker, callWorkerGet, countTokensViaWorker } from '../../src/lib/transport/go-worker-client.mjs' -import { clearRustHealthCache, dispatchCallInference, rememberRustHealth, peekRustHealth } from '../../src/lib/transport/kernel-router.mjs' +import { + workerHealth, + ensureWorkerCredential, + callGoWorker, + streamGoWorker, + callWorkerGet, + countTokensViaWorker, +} from '../../src/lib/transport/go-worker-client.mjs' +import { + clearRustHealthCache, + dispatchCallInference, + rememberRustHealth, + peekRustHealth, +} from '../../src/lib/transport/kernel-router.mjs' import { fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' function manifest(config) { @@ -15,13 +27,26 @@ function manifest(config) { function receipt(config) { return { - intent: { action: 'launch', node_id: config.nodeId, vm_id: config.vmId, - allocation_id: 'synthetic-allocation', operation_id: 'synthetic-operation', - owner_id: 'synthetic-owner', command_id: 'synthetic-launch', epoch: '1', + intent: { + action: 'launch', + node_id: config.nodeId, + vm_id: config.vmId, + allocation_id: 'synthetic-allocation', + operation_id: 'synthetic-operation', + owner_id: 'synthetic-owner', + command_id: 'synthetic-launch', + epoch: '1', fence: { generation: 1, incarnation: config.fence.incarnation, executor: { kind: '', node_boot_id: '' } }, - expected_manifest_revision: '1', manifest_revision: '1', payload_digest: '', - preset_id: 'claude-docker-v1', policy_digest: 'c'.repeat(64), manifest_digest: config.manifest.digest }, - intent_digest: 'd'.repeat(64), manifest_revision: '1', manifest_digest: config.manifest.digest, + expected_manifest_revision: '1', + manifest_revision: '1', + payload_digest: '', + preset_id: 'claude-docker-v1', + policy_digest: 'c'.repeat(64), + manifest_digest: config.manifest.digest, + }, + intent_digest: 'd'.repeat(64), + manifest_revision: '1', + manifest_digest: config.manifest.digest, fence: structuredClone(config.fence), } } @@ -29,9 +54,12 @@ function receipt(config) { async function capture(config, patch = {}) { const { captureNodeExecution } = await import('../../src/lib/cluster/node-execution-capabilities.mjs') return captureNodeExecution({ - db: { schema: 'synthetic_capability' }, vmId: config.vmId, accountId: 'synthetic-account', + db: { schema: 'synthetic_capability' }, + vmId: config.vmId, + accountId: 'synthetic-account', trustedNodes: { [config.nodeId]: { ...config.link, presetId: 'claude-docker-v1', policyDigest: 'c'.repeat(64) } }, - readLaunchReceipt: async () => receipt(config), ...patch, + readLaunchReceipt: async () => receipt(config), + ...patch, }) } @@ -45,7 +73,10 @@ test('capture obtains the original ACTIVE fence from the PG receipt and confirms assert.equal(result.nodeManifest.epoch, '1') assert.equal(Object.isFrozen(result.slotgate.link), true) assert.equal(Object.isFrozen(result.slotgate.fence.executor), true) - assert.deepEqual(fx.requests.map((value) => value.op), ['health']) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['health'], + ) assert.equal(fx.requests[0].manifest_revision, '1') assert.equal(fx.requests[0].manifest_digest, 'b'.repeat(64)) }) @@ -53,10 +84,17 @@ test('capture obtains the original ACTIVE fence from the PG receipt and confirms test('capture rejects missing PG proof and untrusted links before a node request', async (t) => { const fx = await fakeSlotgate(t) manifest(fx.config) - await assert.rejects(capture(fx.config, { readLaunchReceipt: async () => null }), { code: 'node_capability_unavailable' }) + await assert.rejects(capture(fx.config, { readLaunchReceipt: async () => null }), { + code: 'node_capability_unavailable', + }) await assert.rejects(capture(fx.config, { trustedNodes: {} }), { code: 'node_capability_unavailable' }) await assert.rejects(capture(fx.config, { nodeId: 'other-node' }), { code: 'node_capability_unavailable' }) - await assert.rejects(capture(fx.config, { trustedNodes: { [fx.config.nodeId]: { ...fx.config.link, presetId: 'codex', policyDigest: 'c'.repeat(64) } } }), { code: 'node_capability_unavailable' }) + await assert.rejects( + capture(fx.config, { + trustedNodes: { [fx.config.nodeId]: { ...fx.config.link, presetId: 'codex', policyDigest: 'c'.repeat(64) } }, + }), + { code: 'node_capability_unavailable' }, + ) assert.equal(fx.requests.length, 0) }) @@ -64,33 +102,51 @@ test('capture does not adopt a fresh PG receipt after its original node health c const fx = await fakeSlotgate(t) manifest(fx.config) let reads = 0 - await assert.rejects(capture(fx.config, { readLaunchReceipt: async () => { - const value = receipt(fx.config) - if (++reads > 1) value.intent.epoch = '2' - return value - } }), { code: 'node_capability_changed' }) + await assert.rejects( + capture(fx.config, { + readLaunchReceipt: async () => { + const value = receipt(fx.config) + if (++reads > 1) value.intent.epoch = '2' + return value + }, + }), + { code: 'node_capability_changed' }, + ) assert.equal(reads, 2) - assert.deepEqual(fx.requests.map((value) => value.op), ['health']) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['health'], + ) }) test('capture refuses a different account without preparing inference', async (t) => { const fx = await fakeSlotgate(t) manifest(fx.config) await assert.rejects(capture(fx.config, { accountId: 'other-account' }), { code: 'node_account_mismatch' }) - assert.deepEqual(fx.requests.map((value) => value.op), ['health']) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['health'], + ) }) test('capture snapshots fixed TLS configuration before awaiting the PG receipt', async (t) => { const fx = await fakeSlotgate(t) manifest(fx.config) - const trustedNodes = new Map([[fx.config.nodeId, { ...fx.config.link, presetId: 'claude-docker-v1', policyDigest: 'c'.repeat(64) }]]) + const trustedNodes = new Map([ + [fx.config.nodeId, { ...fx.config.link, presetId: 'claude-docker-v1', policyDigest: 'c'.repeat(64) }], + ]) let resume - const blocked = new Promise((resolve) => { resume = resolve }) + const blocked = new Promise((resolve) => { + resume = resolve + }) let reads = 0 - const pending = capture(fx.config, { trustedNodes, readLaunchReceipt: async () => { - if (++reads === 1) await blocked - return receipt(fx.config) - } }) + const pending = capture(fx.config, { + trustedNodes, + readLaunchReceipt: async () => { + if (++reads === 1) await blocked + return receipt(fx.config) + }, + }) // Wait until capture has entered its original lookup, not until node IO. await new Promise((resolve) => setImmediate(resolve)) trustedNodes.get(fx.config.nodeId).certSha256 = '0'.repeat(64) @@ -101,65 +157,122 @@ test('capture snapshots fixed TLS configuration before awaiting the PG receipt', for (const account of [undefined, null, '', 'other-account']) { test(`capture does not authorize account by display email or conflicting identity (${account})`, async (t) => { - const fx = await fakeSlotgate(t, { health: (command) => ({ - ok: true, fence: command.fence, manifest_revision: command.manifest_revision, manifest_digest: command.manifest_digest, - health: { ok: true, credential: { has_access: true, account_uuid: account, email: 'synthetic-account' }, - identity: { account_uuid: 'synthetic-account' } }, - }) }) + const fx = await fakeSlotgate(t, { + health: (command) => ({ + ok: true, + fence: command.fence, + manifest_revision: command.manifest_revision, + manifest_digest: command.manifest_digest, + health: { + ok: true, + credential: { has_access: true, account_uuid: account, email: 'synthetic-account' }, + identity: { account_uuid: 'synthetic-account' }, + }, + }), + }) manifest(fx.config) await assert.rejects(capture(fx.config), { code: 'node_account_mismatch' }) - assert.deepEqual(fx.requests.map((value) => value.op), ['health']) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['health'], + ) }) } test('a PG receipt failure has no TLS or legacy fallback', async (t) => { const fx = await fakeSlotgate(t) manifest(fx.config) - await assert.rejects(capture(fx.config, { readLaunchReceipt: async () => { throw Object.assign(new Error('synthetic PG failure'), { code: 'synthetic_pg_failure' }) } }), { code: 'synthetic_pg_failure' }) + await assert.rejects( + capture(fx.config, { + readLaunchReceipt: async () => { + throw Object.assign(new Error('synthetic PG failure'), { code: 'synthetic_pg_failure' }) + }, + }), + { code: 'synthetic_pg_failure' }, + ) assert.equal(fx.requests.length, 0) }) test('health cache separates captured fences and manifests and VM invalidation clears each', async () => { clearRustHealthCache() - const first = { vmId: 'cache-vm', slotgate: { fence: syntheticFence(), manifest: { revision: '1', digest: 'a'.repeat(64) } } } + const first = { + vmId: 'cache-vm', + slotgate: { fence: syntheticFence(), manifest: { revision: '1', digest: 'a'.repeat(64) } }, + } rememberRustHealth(first, { ok: true, health: { ok: true } }) assert.equal(peekRustHealth(first, 1000).health.ok, true) const next = { ...first, slotgate: { ...first.slotgate, fence: syntheticFence('2') } } assert.equal(peekRustHealth(next, 1000), null) - const changedManifest = { ...first, slotgate: { ...first.slotgate, manifest: { revision: '2', digest: 'b'.repeat(64) } } } + const changedManifest = { + ...first, + slotgate: { ...first.slotgate, manifest: { revision: '2', digest: 'b'.repeat(64) } }, + } assert.equal(peekRustHealth(changedManifest, 1000), null) clearRustHealthCache('cache-vm') assert.equal(peekRustHealth(first, 1000), null) }) test('typed health returns only public fields and never needs a Pod kernel token', async (t) => { - const fx = await fakeSlotgate(t, { health: (command) => ({ - ok: true, fence: command.fence, manifest_revision: command.manifest_revision, manifest_digest: command.manifest_digest, - health: { ok: true, engine: 'rust', ready_slots: 2, healthy: true, internal_token: 'must-not-escape', - credential: { has_access: true, has_refresh: true, generation: 'g1', expires_at: 1981929600000, - account_uuid: 'synthetic-account', org_uuid: 'synthetic-org', email: 'synthetic@example.invalid', - access_token: 'must-not-escape', refresh_token: 'must-not-escape', provider_extension: { secret: 'must-not-escape' } } }, - }) }) + const fx = await fakeSlotgate(t, { + health: (command) => ({ + ok: true, + fence: command.fence, + manifest_revision: command.manifest_revision, + manifest_digest: command.manifest_digest, + health: { + ok: true, + engine: 'rust', + ready_slots: 2, + healthy: true, + internal_token: 'must-not-escape', + credential: { + has_access: true, + has_refresh: true, + generation: 'g1', + expires_at: 1981929600000, + account_uuid: 'synthetic-account', + org_uuid: 'synthetic-org', + email: 'synthetic@example.invalid', + access_token: 'must-not-escape', + refresh_token: 'must-not-escape', + provider_extension: { secret: 'must-not-escape' }, + }, + }, + }), + }) const config = manifest(fx.config) const value = await workerHealth({ vmId: config.vmId, slotgate: config }) assert.equal(value.ok, true) assert.equal(value.credential.account_uuid, 'synthetic-account') assert.equal(JSON.stringify(value).includes('must-not-escape'), false) - assert.deepEqual(fx.requests.map((entry) => entry.op), ['health']) + assert.deepEqual( + fx.requests.map((entry) => entry.op), + ['health'], + ) }) for (const kind of ['fence', 'manifest', 'missing']) { test(`typed health rejects ${kind} proof and never prepares inference`, async (t) => { - const fx = await fakeSlotgate(t, { health: (command) => kind === 'missing' ? { ok: true, health: { ok: true } } : { - ok: true, fence: kind === 'fence' ? syntheticFence('2') : command.fence, - manifest_revision: kind === 'manifest' ? '2' : command.manifest_revision, - manifest_digest: command.manifest_digest, health: { ok: true, healthy: true, ready_slots: 1 }, - } }) + const fx = await fakeSlotgate(t, { + health: (command) => + kind === 'missing' + ? { ok: true, health: { ok: true } } + : { + ok: true, + fence: kind === 'fence' ? syntheticFence('2') : command.fence, + manifest_revision: kind === 'manifest' ? '2' : command.manifest_revision, + manifest_digest: command.manifest_digest, + health: { ok: true, healthy: true, ready_slots: 1 }, + }, + }) manifest(fx.config) const value = await workerHealth({ vmId: fx.config.vmId, slotgate: fx.config }) assert.equal(value.ok, false) assert.equal(value.code, 'slotgate_health_unconfirmed') - assert.deepEqual(fx.requests.map((entry) => entry.op), ['health']) + assert.deepEqual( + fx.requests.map((entry) => entry.op), + ['health'], + ) }) } @@ -170,7 +283,10 @@ test('typed health observes cancellation without a legacy cancel request', async const pending = client.health({ signal: controller.signal, timeoutMs: 1000 }) setTimeout(() => controller.abort(), 40) await assert.rejects(pending, { code: 'ABORT_ERR' }) - assert.equal(fx.requests.every((value) => value.op === 'health'), true) + assert.equal( + fx.requests.every((value) => value.op === 'health'), + true, + ) }) test('fenced router never creates Pod credential links or reads a host kernel binary', async (t) => { @@ -188,7 +304,10 @@ test('fenced router never creates Pod credential links or reads a host kernel bi assert.equal(result.ok, true) assert.equal(result.execution.ended, true) assert.equal(fs.existsSync(homeDir), false) - assert.deepEqual(fx.requests.map((value) => value.op), ['health', 'prepare', 'forward', 'command']) + assert.deepEqual( + fx.requests.map((value) => value.op), + ['health', 'prepare', 'forward', 'command'], + ) }) test('fenced credential ensure fails before touching a Pod credential file', async (t) => { @@ -210,10 +329,18 @@ test('HA transport refuses a missing trusted capability before mock or legacy ex process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' process.env.KIN_CRS_MOCK = '1' try { - await assert.rejects(callGoWorker({ exec: { vmId: 'legacy-vm' }, envelope: { body: {} } }), { code: 'slotgate_capability_required' }) - await assert.rejects(streamGoWorker({ exec: { vmId: 'legacy-vm' }, envelope: { body: {} } }), { code: 'slotgate_capability_required' }) - await assert.rejects(callWorkerGet({ vmId: 'legacy-vm' }, '/internal/v1/models'), { code: 'slotgate_capability_required' }) - await assert.rejects(countTokensViaWorker({ vmId: 'legacy-vm' }, { body: {} }), { code: 'slotgate_capability_required' }) + await assert.rejects(callGoWorker({ exec: { vmId: 'legacy-vm' }, envelope: { body: {} } }), { + code: 'slotgate_capability_required', + }) + await assert.rejects(streamGoWorker({ exec: { vmId: 'legacy-vm' }, envelope: { body: {} } }), { + code: 'slotgate_capability_required', + }) + await assert.rejects(callWorkerGet({ vmId: 'legacy-vm' }, '/internal/v1/models'), { + code: 'slotgate_capability_required', + }) + await assert.rejects(countTokensViaWorker({ vmId: 'legacy-vm' }, { body: {} }), { + code: 'slotgate_capability_required', + }) } finally { if (before == null) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = before diff --git a/test/unit/slotgate-client.test.mjs b/test/unit/slotgate-client.test.mjs index 53dfc2647..00359563f 100644 --- a/test/unit/slotgate-client.test.mjs +++ b/test/unit/slotgate-client.test.mjs @@ -211,11 +211,11 @@ test('router preserves durable PG bind acknowledgement for the fenced worker cli else process.env.KIN_KERNEL_BIN = previous }) const fx = await fakeSlotgate(t) + fx.config.manifest = { revision: '1', digest: 'b'.repeat(64) } let completed const result = await dispatchCallInference({ exec: execFor(fx.config), envelope: { body: {} }, - ensureRust: async () => ({ ok: true }), onExecutionStart: async (value) => acknowledged(value), onExecutionEnd: async (value) => { completed = value From 5291b8082fecf4c32d771d10c18cc6df8e45a328 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:26:43 +0800 Subject: [PATCH 362/473] test: require durable initial publication before node Docker launch --- .../internal/slotgate/publish_initial_test.go | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 worker/internal/slotgate/publish_initial_test.go diff --git a/worker/internal/slotgate/publish_initial_test.go b/worker/internal/slotgate/publish_initial_test.go new file mode 100644 index 000000000..37ea76f3f --- /dev/null +++ b/worker/internal/slotgate/publish_initial_test.go @@ -0,0 +1,29 @@ +package slotgate + +import ( + "context" + "testing" +) + +// An existing directory is not proof that node-owned publication completed. +// In particular, a leftover tree after rename/journal failure must not launch. +func TestInitialPublishUnrecordedTreeCannotReachDocker(t *testing.T) { + _, launcher, engine, state, _ := launchFixture(t) + if _, err := launcher.Launch(context.Background(), "v", state.Fence, "unrecorded-publication"); err == nil { + t.Fatal("launch accepted files without a durable completed manifest") + } + engine.mu.Lock() + defer engine.mu.Unlock() + if len(engine.calls) != 0 { + t.Fatalf("unrecorded publication reached Docker: %v", engine.calls) + } +} + +// Allocation and inference authorization cannot stand in for a publish grant. +func TestInitialPublishRequiresIndependentManagementAuthorization(t *testing.T) { + addr, _, _, state, _ := fencedGate(t) + reply := gate2Control(t, addr, gate2Request{Op: "publish-initial", VMID: "v", Fence: state.Fence, CommandID: "unauthorized-publication"}) + if reply.OK || reply.Error != ErrManagementDenied.Error() { + t.Fatalf("publish without an independent action authorizer: %+v", reply) + } +} From ed80951316d2d83c317d22ffac175b73634bc877 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:38:54 +0800 Subject: [PATCH 363/473] feat: publish node-owned initial files and bind Docker launch to a manifest --- .../2026-10-22-node-initial-publication.md | 54 ++ worker/internal/slotgate/authority.go | 51 +- worker/internal/slotgate/docker_launch.go | 39 +- .../internal/slotgate/docker_launch_test.go | 69 ++- worker/internal/slotgate/ksg2.go | 92 ++- worker/internal/slotgate/management_intent.go | 103 ++++ worker/internal/slotgate/pg_authorizer.go | 69 +++ .../internal/slotgate/publish_files_linux.go | 344 +++++++++++ .../internal/slotgate/publish_files_other.go | 16 + worker/internal/slotgate/publish_initial.go | 340 +++++++++++ .../internal/slotgate/publish_initial_test.go | 533 +++++++++++++++++- worker/internal/slotgate/slotgate.go | 22 +- worker/internal/slotgate/slotgate_test.go | 6 +- 13 files changed, 1679 insertions(+), 59 deletions(-) create mode 100644 docs/plans/2026-10-22-node-initial-publication.md create mode 100644 worker/internal/slotgate/management_intent.go create mode 100644 worker/internal/slotgate/publish_files_linux.go create mode 100644 worker/internal/slotgate/publish_files_other.go create mode 100644 worker/internal/slotgate/publish_initial.go diff --git a/docs/plans/2026-10-22-node-initial-publication.md b/docs/plans/2026-10-22-node-initial-publication.md new file mode 100644 index 000000000..4e73eb182 --- /dev/null +++ b/docs/plans/2026-10-22-node-initial-publication.md @@ -0,0 +1,54 @@ +# 节点启动前发布协议 + +本批基于 `9d06160d`，仅覆盖 Linux 上普通 Claude Docker 的初次发布和启动。复用 slot image 内现成 `/opt/kin/kin-kernel`、`/opt/kin/cli-node`，不修改闭源 payload。测试只使用合成凭据、临时目录、Unix/TLS 监听和 fake Engine。没有实际 Docker、SSH、生产节点或 provider 验收。 + +## 授权与身份 + +沿用 allocation 的私有 authority schema、maintenance epoch 和 node/VM advisory resource，但采用独立 `command_protocol_version=1`、`publish-initial`/`launch` action。缺新 view、feature marker、权限或 PG 可用性均拒绝；不得退回 allocation grant、静态 forwarding token、KSG1 或 SSH。 + +KSG2 metadata 为 `{op,vm_id,management:,content_length?}`。除 `op`、`vm_id` 外，旧 flat fence/command/allocation/operation/owner/kind/ticket 必须为空。publish metadata 后紧跟 `content_length` 个 payload bytes，连接仅执行一次动作。metadata 仍受 8 KiB 上限约束；payload 上限 64 KiB。 + +ManagementIntent 固定字段：`action,node_id,vm_id,allocation_id,operation_id,owner_id,command_id,epoch,fence,expected_manifest_revision,manifest_revision,payload_digest,preset_id,policy_digest,manifest_digest`。epoch 和 revision 使用规范十进制字符串。fence 是 PG allocation receipt 的原始 generation 1、incarnation 和空 executor，不读取最新 state 给旧动作重新授权。 + +`intent_digest` 是以下字符串以 NUL 连接后的 SHA256 小写 hex： + +``` +slotgate-command-v1, action, node_id, vm_id, allocation_id, +operation_id, owner_id, command_id, epoch, decimal(fence.generation), +fence.incarnation, expected_manifest_revision, manifest_revision, +payload_digest, preset_id, policy_digest, manifest_digest +``` + +publish 固定 `0→1`、`payload_digest=SHA256(payload bytes)`、空 `manifest_digest`。launch 固定 `1→1`、空 `payload_digest`，绑定 PG 保存的 published manifest digest。preset 固定 `claude-docker-v1`。policy digest 为 NUL 连接 `slotgate-policy-v1,claude-docker-v1,image_id,network_id,decimal(uid),decimal(gid)` 的 SHA256。 + +节点受限 PG role 通过 `authorized_commands` 精确匹配原 command/allocation/node/VM/operation/owner/action，再比较 epoch 和完整 intent digest。该 view 与 private feature marker、maintenance epoch、live VM operation 的验证由配套 PG service 提供。授权共享锁只覆盖本地 journal 和最终 rename；不跨上传或 Docker I/O。 + +## 固定内容和私有文件 + +payload 只接受 `{credential:{type,access_token?,refresh_token?,api_key?,expires_at?,scopes?}}`。type 为 `oauth`、`setup-token`、`official-setup-token`、`apikey`。未知字段、重复字段、额外 JSON、文件名、路径、mode、UID、URL、env、argv 和原始 settings/config 均拒绝。单 token 上限 16 KiB；scopes 最多 32 项，每项 128 字符。setup-token 不接受 refresh token；API key 不与 OAuth 字段混用。 + +节点生成固定 kernel config、256-bit 随机 internal token 和固定 `.credentials.json → credentials.json` link。内部 token 不返回 controller。批准的 image ID、network ID、UID/GID 来自 node-owned DockerLaunchPolicy；RPC 不能改变这些值。 + +受信任初始化先提供 node-owned 0700 Root。publisher 经 nofollow 目录描述符创建 `vms//incarnations`，再创建私有 `.stage-`。输入不能选择 staging 名称。stage 内只生成 `config/home/claude/run`，config 属于节点；guest 只拥有后面三个叶目录。journal、staging 父目录和 Docker socket 不挂载给 guest。已有未知 incarnation 目录不得覆盖或采用。 + +Linux 文件操作使用 descriptor-relative `openat/mkdirat`、`O_NOFOLLOW`、`O_EXCL`，检查 owner/mode、普通文件类型和 link count=1。所有输入 link 均拒绝；唯一固定 credential link 由节点自己创建并验证。文件和目录 fsync 后，最终 publish 再检查原授权、原 VACANT Fence、command、expected revision 和文件摘要，使用 `renameat2(RENAME_NOREPLACE)` 发布，再 fsync 父目录并持久化 completed manifest。非 Linux 平台明确拒绝本接口。 + +## 完成与未知状态 + +BeginPublish 先持久化 pending，再接收或写入内容。上传、stage、rename、fsync、journal 或响应不明时，不换 ID重发，不把存在的目录当完成证明。pending/unknown publication 阻止 launch 和 generation 推进；节点重启将 pending 标记 unknown。已完成的同意图重读返回原 receipt，不重写文件。 + +publication receipt 固定为 `{intent,intent_digest,manifest_revision,manifest_digest}`。journal 仅额外保存私有 config/token/credential 摘要，不保存 secret bytes。公开 receipt 不包含各 secret 文件摘要。 + +launch 必须先验证 completed manifest 和实际私有文件。节点在短 PG 授权内持久接受 manifest-bound launch，再复用现有 create/start/inspect/bind 状态机；Docker I/O 不持 Authority 全局锁。已接受的 launch 是 pending 动作，不能因 HTTP 断开、grant 撤销或 timeout 被视为未开始。接管仍要求原 immutable container/daemon/boot 身份的终止证明。 + +launch receipt 为 `{intent,intent_digest,manifest_revision,manifest_digest,fence}`，其中 intent.fence 保持原 VACANT Fence，输出 fence 含实际 ACTIVE executor，kind 为 `container`。controller 必须保存确认 receipt 后再释放 operation；未知结果继续保留。receipt 是历史启动证明，不证明当前仍在运行。 + +managed forward 只允许 kernel `POST /internal/v1/messages`，无 query，body.request_id 必须等原 command ID。Dispatch 内按捕获的 incarnation 检查 node-owned config/token 摘要，覆盖 `x-kin-internal-token` 和 `x-internal-token`。不能用此转发访问 credential/import 等管理 HTTP。正常响应仍须查询原 ticket 的持久 command completion 才能释放 PG execution；不会因 token 注入放宽 terminal parser。 + +## 验证与未覆盖范围 + +失败规格 `9c6fad09` 先复现未记录目录可以 launch，以及缺独立 publish 授权入口。后续专用测试覆盖完成 receipt、重放、错误 Fence/revision/大小/摘要、截断上传、PG authorizer hook 拒绝、rename 前 revoke、symbolic/hard link、目录冲突、journal fault/restart、不同 VM 并行、manifest-bound launch、node token 覆盖和管理 HTTP 拒绝。 + +这些测试不代替配套 PG service 与 Go reader 的双连接组合验证，也不代替现成 payload 的实际启动与推理验收。未提供 health/credential-summary RPC、运行期 credential refresh/config 热更新、新 generation 再发布、native Mac/KVM/Codex executor 或自动未知状态恢复。 + +runtime 的旧 SSH/SFTP/Docker 权限及已建立会话尚未撤销。节点服务、批准 image/network、受限 PG role 和文件权限仍需另行授权的可信初始化。因此本批不能作为生产开启多副本 HA 的条件。 diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index e53bcd51e..cb4c287bc 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -87,6 +87,8 @@ type CommandRecord struct { Dispatched bool `json:"dispatched"` TerminationID string `json:"termination_id,omitempty"` LaunchTarget *DockerLaunchTarget `json:"launch_target,omitempty"` + Management *ManagementIntent `json:"management,omitempty"` + LaunchReceipt *LaunchReceipt `json:"launch_receipt,omitempty"` } // DockerLaunchTarget is recorded after create returns and before start. A @@ -101,6 +103,7 @@ type DockerLaunchTarget struct { type AuthorityState struct { Allocation *AllocationReceipt `json:"allocation,omitempty"` + InitialManifest *initialManifest `json:"initial_manifest,omitempty"` Fence Fence `json:"fence"` Phase AuthorityPhase `json:"phase"` Commands map[string]CommandRecord `json:"commands"` @@ -464,9 +467,15 @@ func (a *Authority) BindExecutor(handle CommandHandle, executor ExecutorIdentity } } command.Status = CommandCompleted - state.Commands[handle.ID] = command state.Fence.Executor = executor state.Phase = PhaseActive + if intent := command.Management; intent != nil { + if state.InitialManifest == nil || intent.Action != "launch" || state.InitialManifest.Receipt.ManifestDigest != intent.ManifestDigest { + return AuthorityState{}, ErrInvalidAuthority + } + command.LaunchReceipt = &LaunchReceipt{Intent: *intent, IntentDigest: intent.digest(), ManifestRevision: "1", ManifestDigest: intent.ManifestDigest, Fence: state.Fence} + } + state.Commands[handle.ID] = command return a.commit(handle.VMID, state) } @@ -485,7 +494,7 @@ func (a *Authority) CompleteCommand(handle CommandHandle) error { if command.Status != CommandPending { return ErrCommandOwner } - if command.Kind == commandLaunch && command.Dispatched { + if command.Kind == commandPublish || (command.Kind == commandLaunch && command.Dispatched) { return ErrAuthorityPhase } command.Status = CommandCompleted @@ -666,12 +675,24 @@ func (a *Authority) persist(next authorityJournal) error { } func cloneState(state AuthorityState) AuthorityState { + if state.InitialManifest != nil { + manifest := *state.InitialManifest + state.InitialManifest = &manifest + } if state.Allocation != nil { receipt := *state.Allocation state.Allocation = &receipt } commands := make(map[string]CommandRecord, len(state.Commands)) for id, command := range state.Commands { + if command.Management != nil { + intent := *command.Management + command.Management = &intent + } + if command.LaunchReceipt != nil { + receipt := *command.LaunchReceipt + command.LaunchReceipt = &receipt + } if command.LaunchTarget != nil { target := *command.LaunchTarget command.LaunchTarget = &target @@ -759,6 +780,9 @@ func validateJournal(journal authorityJournal) error { return ErrInvalidAuthority } for vm, state := range journal.Slots { + if err := validateInitialManifest(vm, state); err != nil { + return err + } if receipt := state.Allocation; receipt != nil { if journal.Management != nil && receipt.NodeID != journal.Management.NodeID { return ErrInvalidAuthority @@ -787,9 +811,30 @@ func validateJournal(journal authorityJournal) error { if id != command.ID || !validAuthorityID(id) || !validAuthorityID(command.Runtime) || !validAuthorityID(command.Token) || command.Fence.Generation == 0 || command.Fence.Generation > state.Fence.Generation || !validAuthorityID(command.Fence.Incarnation) { return ErrInvalidAuthority } - if command.Kind != CommandForward && command.Kind != CommandMutation && command.Kind != commandLaunch { + if command.Kind != CommandForward && command.Kind != CommandMutation && command.Kind != commandLaunch && command.Kind != commandPublish { + return ErrInvalidAuthority + } + if command.Kind == commandPublish && command.Management == nil { + return ErrInvalidAuthority + } + if command.Kind == commandPublish && command.Status == CommandCompleted && state.InitialManifest == nil { return ErrInvalidAuthority } + if intent := command.Management; intent != nil { + if !validManagementIntent(*intent) || intent.VMID != vm || intent.CommandID != id || intent.Fence != command.Fence || + (command.Kind == commandPublish && intent.Action != "publish-initial") || + (command.Kind == commandLaunch && intent.Action != "launch") || + (command.Kind != commandPublish && command.Kind != commandLaunch) { + return ErrInvalidAuthority + } + } + if receipt := command.LaunchReceipt; receipt != nil { + if command.Management == nil || receipt.Intent != *command.Management || receipt.IntentDigest != receipt.Intent.digest() || + receipt.ManifestRevision != "1" || receipt.ManifestDigest != receipt.Intent.ManifestDigest || command.Kind != commandLaunch || + command.Status != CommandCompleted || receipt.Fence.Generation != command.Fence.Generation || receipt.Fence.Incarnation != command.Fence.Incarnation || validateExecutor(receipt.Fence.Executor) != nil { + return ErrInvalidAuthority + } + } if command.Status != CommandPending && command.Status != CommandUnknown && command.Status != CommandCompleted && command.Status != CommandTerminated { return ErrInvalidAuthority } diff --git a/worker/internal/slotgate/docker_launch.go b/worker/internal/slotgate/docker_launch.go index a9c24edf4..dd992d26f 100644 --- a/worker/internal/slotgate/docker_launch.go +++ b/worker/internal/slotgate/docker_launch.go @@ -61,6 +61,32 @@ func newDockerLauncher(socket, root, boot string, a *Authority, policy DockerLau } func (l *dockerLauncher) Launch(ctx context.Context, vm string, expected Fence, id string) (AuthorityState, error) { + return l.launch(ctx, vm, expected, id, nil) +} + +func (l *dockerLauncher) LaunchManifest(ctx context.Context, intent ManagementIntent, authorize CommandAuthorizer) (LaunchReceipt, error) { + if authorize == nil { + return LaunchReceipt{}, ErrManagementDenied + } + if !validManagementIntent(intent) || intent.Action != "launch" || intent.PolicyDigest != l.policy.digest() { + return LaunchReceipt{}, ErrInvalidAuthority + } + state, err := l.launch(ctx, intent.VMID, intent.Fence, intent.CommandID, func() (CommandHandle, error) { + var handle CommandHandle + err := authorize(ctx, intent, func() error { var e error; handle, e = l.authority.beginManifestLaunch(intent); return e }) + return handle, err + }) + if err != nil { + return LaunchReceipt{}, err + } + command := state.Commands[intent.CommandID] + if command.LaunchReceipt == nil { + return LaunchReceipt{}, ErrAuthorityUnavailable + } + return *command.LaunchReceipt, nil +} + +func (l *dockerLauncher) launch(ctx context.Context, vm string, expected Fence, id string, accept func() (CommandHandle, error)) (AuthorityState, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() state, err := l.authority.State(vm) @@ -79,6 +105,12 @@ func (l *dockerLauncher) Launch(ctx context.Context, vm string, expected Fence, if !validAuthorityID(id) { return AuthorityState{}, ErrInvalidAuthority } + if err = checkInitialManifest(state, l.policy); err != nil { + return AuthorityState{}, err + } + if _, err = readPublishedInitial(l.root, vm, expected, l.policy, *state.InitialManifest, true); err != nil { + return AuthorityState{}, err + } if err = l.privateFiles(vm, expected); err != nil { return AuthorityState{}, err } @@ -116,7 +148,12 @@ func (l *dockerLauncher) Launch(ctx context.Context, vm string, expected Fence, return AuthorityState{}, ErrInvalidAuthority } } - handle, err := l.authority.BeginLaunch(vm, expected, id) + var handle CommandHandle + if accept != nil { + handle, err = accept() + } else { + handle, err = l.authority.BeginLaunch(vm, expected, id) + } if err != nil { return AuthorityState{}, err } diff --git a/worker/internal/slotgate/docker_launch_test.go b/worker/internal/slotgate/docker_launch_test.go index 4ff4126c9..dfa19e790 100644 --- a/worker/internal/slotgate/docker_launch_test.go +++ b/worker/internal/slotgate/docker_launch_test.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "strings" "sync" "testing" @@ -95,45 +96,63 @@ func (f *fakeLaunchEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func launchFixture(t *testing.T) (*Authority, *dockerLauncher, *fakeLaunchEngine, AuthorityState, string) { + return launchFixtureConfigured(t, true, "none") +} + +func launchFixtureConfigured(t *testing.T, published bool, network string) (*Authority, *dockerLauncher, *fakeLaunchEngine, AuthorityState, string) { t.Helper() + if runtime.GOOS != "linux" { + t.Skip("initial publication requires Linux descriptor-relative file operations") + } root := shortRoot(t) boot, err := nodeBootID() if err != nil { t.Fatal(err) } a := authorityForTest(t, filepath.Join(root, "authority"), terminatedExecutor) - state, err := a.CreateSlot("v") + allocation, err := a.Allocate(AllocationIntent{NodeID: "node-test", VMID: "v", AllocationID: "allocation-test", OperationID: "allocation-operation", OwnerID: "allocation-owner"}, "1") if err != nil { t.Fatal(err) } - policy := DockerLaunchPolicy{ImageID: "sha256:" + strings.Repeat("b", 64), NetworkID: "none", UID: os.Getuid(), GID: os.Getgid()} + state, _ := a.State("v") + policy := DockerLaunchPolicy{ImageID: "sha256:" + strings.Repeat("b", 64), NetworkID: network, UID: os.Getuid(), GID: os.Getgid()} if policy.UID == 0 { policy.UID = 10000 policy.GID = 10000 } private := filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation) - for _, dir := range []string{"config", "run", "home", "claude"} { - if err = os.MkdirAll(filepath.Join(private, dir), 0o700); err != nil { - t.Fatal(err) - } - if dir != "config" && policy.UID != os.Geteuid() { - if err = os.Chown(filepath.Join(private, dir), policy.UID, policy.GID); err != nil { + if !published { + for _, dir := range []string{"config", "run", "home", "claude"} { + if err = os.MkdirAll(filepath.Join(private, dir), 0o700); err != nil { t.Fatal(err) } + if dir != "config" && policy.UID != os.Geteuid() { + if err = os.Chown(filepath.Join(private, dir), policy.UID, policy.GID); err != nil { + t.Fatal(err) + } + } } - } - for _, file := range []string{"kernel.json", "internal.token"} { - if err = os.WriteFile(filepath.Join(private, "config", file), []byte("synthetic-only"), 0o600); err != nil { - t.Fatal(err) - } - if policy.UID != os.Geteuid() { - if err = os.Chown(filepath.Join(private, "config", file), os.Geteuid(), policy.GID); err != nil { + for _, file := range []string{"kernel.json", "internal.token"} { + if err = os.WriteFile(filepath.Join(private, "config", file), []byte("synthetic-only"), 0o600); err != nil { t.Fatal(err) } - if err = os.Chmod(filepath.Join(private, "config", file), 0o640); err != nil { - t.Fatal(err) + if policy.UID != os.Geteuid() { + if err = os.Chown(filepath.Join(private, "config", file), os.Geteuid(), policy.GID); err != nil { + t.Fatal(err) + } + if err = os.Chmod(filepath.Join(private, "config", file), 0o640); err != nil { + t.Fatal(err) + } } } + } else { + payload := []byte(`{"credential":{"type":"oauth","access_token":"synthetic-access","refresh_token":"synthetic-refresh"}}`) + intent := publicationIntentForTest(allocation, policy, payload) + publisher := initialPublisher{authority: a, root: root, nodeID: allocation.NodeID, policy: policy, authorize: allowNodeCommandForTest} + if _, err = publisher.Publish(context.Background(), intent, len(payload), strings.NewReader(string(payload))); err != nil { + t.Fatal(err) + } + state, _ = a.State("v") } fake := &fakeLaunchEngine{image: policy.ImageID, container: strings.Repeat("c", 64), startedAt: "2026-10-21T01:00:00Z"} socket := filepath.Join(shortRoot(t), "docker.sock") @@ -393,7 +412,7 @@ func TestKSG2LaunchCreatesAndBindsWithNodePolicyAndRejectsRawSpec(t *testing.T) if err := a.Close(); err != nil { t.Fatal(err) } - s, err := New(Config{Listen: "127.0.0.1:0", Root: root, Token: testToken, TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: filepath.Join(root, "authority"), DockerSocket: engine.socket, Launch: &launcher.policy}) + s, err := New(Config{Listen: "127.0.0.1:0", Root: root, Token: testToken, TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: filepath.Join(root, "authority"), DockerSocket: engine.socket, Launch: &launcher.policy, Management: managementConfigForTest()}) if err != nil { t.Fatal(err) } @@ -428,18 +447,22 @@ func TestKSG2LaunchCreatesAndBindsWithNodePolicyAndRejectsRawSpec(t *testing.T) if count != 0 { t.Fatal("raw spec reached Engine") } - s.authorizeManagement = func(_ context.Context, _ gate2Request, dispatch func() error) error { return dispatch() } - reply := gate2Control(t, ln.Addr().String(), gate2Request{Op: "launch", VMID: "v", Fence: state.Fence, CommandID: "trusted-node-launch"}) - if !reply.OK || reply.State == nil || reply.State.Phase != PhaseActive || reply.State.Fence.Executor.ContainerID != engine.container { + s.authorizeCommand = allowNodeCommandForTest + intent := launchIntentForTest(state, "trusted-node-launch") + reply := gate2Control(t, ln.Addr().String(), gate2Request{Op: "launch", VMID: "v", Management: &intent}) + if !reply.OK { t.Fatalf("KSG2 launch: %+v", reply) } + active, _ := s.authority.State("v") + if active.Phase != PhaseActive || active.Fence.Executor.ContainerID != engine.container { + t.Fatalf("KSG2 active: %+v", active) + } } func TestDockerLaunchChecksFixedBridgeNetworkBeforeCreate(t *testing.T) { for _, driver := range []string{"bridge", "host"} { t.Run(driver, func(t *testing.T) { - _, launcher, engine, state, _ := launchFixture(t) - launcher.policy.NetworkID = strings.Repeat("e", 64) + _, launcher, engine, state, _ := launchFixtureConfigured(t, true, strings.Repeat("e", 64)) engine.networkDriver = driver _, err := launcher.Launch(context.Background(), "v", state.Fence, "fixed-network") if driver == "bridge" { diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index 0e46a3043..c1d2c7a7f 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -28,15 +28,17 @@ var errGateTicketInvalid = errors.New("ticket_invalid") // commands continue with one parsed HTTP/1 request and an HTTP response after // OK\n. KSG1 raw splicing is never accepted when fenced mode is enabled. type gate2Request struct { - Op string `json:"op"` - VMID string `json:"vm_id"` - Fence Fence `json:"fence"` - CommandID string `json:"command_id,omitempty"` - Kind string `json:"kind,omitempty"` - Ticket string `json:"ticket,omitempty"` - AllocationID string `json:"allocation_id,omitempty"` - OperationID string `json:"operation_id,omitempty"` - OwnerID string `json:"owner_id,omitempty"` + Op string `json:"op"` + VMID string `json:"vm_id"` + Fence Fence `json:"fence"` + CommandID string `json:"command_id,omitempty"` + Kind string `json:"kind,omitempty"` + Ticket string `json:"ticket,omitempty"` + AllocationID string `json:"allocation_id,omitempty"` + OperationID string `json:"operation_id,omitempty"` + OwnerID string `json:"owner_id,omitempty"` + Management *ManagementIntent `json:"management,omitempty"` + ContentLength int `json:"content_length,omitempty"` } type gate2State struct { @@ -98,17 +100,15 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. return } var request gate2Request - decoder := json.NewDecoder(bytes.NewReader(lineBytes)) - decoder.DisallowUnknownFields() - if err = decoder.Decode(&request); err != nil { + if err = strictJSON(lineBytes, &request); err != nil { io.WriteString(conn, "ERR bad_request\n") return } - if err = decoder.Decode(new(any)); err != io.EOF { + if !vmIDRe.MatchString(request.VMID) { io.WriteString(conn, "ERR bad_request\n") return } - if !vmIDRe.MatchString(request.VMID) { + if request.Op != "publish-initial" && request.Op != "launch" && (request.Management != nil || request.ContentLength != 0) { io.WriteString(conn, "ERR bad_request\n") return } @@ -135,6 +135,10 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. case "state": state, err = s.authority.State(request.VMID) case "prepare": + if managed, e := s.authority.State(request.VMID); e == nil && managed.Allocation != nil && (managed.InitialManifest == nil || request.Kind != "kernel") { + err = ErrManagementDenied + break + } if _, ok := kinds[request.Kind]; !ok { err = ErrInvalidAuthority break @@ -157,19 +161,36 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. } err = s.authorizeManagement(ctx, request, func() error { var e error; state, e = s.authority.Advance(ctx, request.VMID, request.Fence); return e }) case "launch": - if s.authorizeManagement == nil { + if s.authorizeCommand == nil || s.managementNodeID == "" { + err = ErrManagementDenied + break + } + if s.launcher == nil || !s.validManagementRequest(request) || request.ContentLength != 0 { + err = ErrInvalidAuthority + break + } + var receipt LaunchReceipt + receipt, err = s.launcher.LaunchManifest(ctx, *request.Management, s.authorizeCommand) + if err == nil { + reply["launch"] = receipt + } + case "publish-initial": + if s.authorizeCommand == nil || s.managementNodeID == "" { err = ErrManagementDenied break } - if s.launcher == nil || request.Kind != "" || request.Ticket != "" { + if s.publisher == nil || !s.validManagementRequest(request) { err = ErrInvalidAuthority break } - err = s.authorizeManagement(ctx, request, func() error { - var e error - state, e = s.launcher.Launch(ctx, request.VMID, request.Fence, request.CommandID) - return e - }) + publisher := *s.publisher + publisher.nodeID = s.managementNodeID + publisher.authorize = s.authorizeCommand + var receipt PublicationReceipt + receipt, err = publisher.Publish(ctx, *request.Management, request.ContentLength, reader) + if err == nil { + reply["publication"] = receipt + } case "command": var handle CommandHandle handle, err = s.forwardHandle(request) @@ -197,13 +218,18 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["error"] = gate2Code(err) } else { reply["ok"] = true - if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" { + if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" { reply["state"] = publicGateState(state) } } json.NewEncoder(conn).Encode(reply) } +func (s *Server) validManagementRequest(request gate2Request) bool { + return request.Management != nil && request.Management.Action == request.Op && request.Management.VMID == request.VMID && request.Management.NodeID == s.managementNodeID && + request.Fence == (Fence{}) && request.CommandID == "" && request.Kind == "" && request.Ticket == "" && request.AllocationID == "" && request.OperationID == "" && request.OwnerID == "" && validManagementIntent(*request.Management) +} + func gate2Ticket(nonce string, request gate2Request) string { mac := hmac.New(sha256.New, []byte(nonce)) // The opcode/ticket are excluded so prepare and forward share one grant. @@ -267,6 +293,20 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio io.WriteString(conn, "ERR http_request\n") return } + state, err := s.authority.State(request.VMID) + if err != nil || state.Fence != request.Fence { + fmt.Fprint(conn, "ERR fence_mismatch\n") + return + } + if state.Allocation != nil { + var envelope map[string]json.RawMessage + var id string + if state.InitialManifest == nil || s.launcher == nil || request.Kind != "kernel" || req.Method != http.MethodPost || req.URL.Path != "/internal/v1/messages" || req.URL.RawQuery != "" || + strictJSON(body, &envelope) != nil || json.Unmarshal(envelope["request_id"], &id) != nil || id != request.CommandID { + fmt.Fprint(conn, "ERR management_denied\n") + return + } + } req.Body = io.NopCloser(bytes.NewReader(body)) req.ContentLength = int64(len(body)) req.TransferEncoding = nil @@ -278,6 +318,14 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio conn.SetDeadline(time.Time{}) var target net.Conn err = s.authority.DispatchCommand(handle, func(ExecutorIdentity) error { + if state.Allocation != nil { + token, e := readPublishedInitial(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy, *state.InitialManifest, false) + if e != nil { + return e + } + req.Header.Set("x-kin-internal-token", token) + req.Header.Set("x-internal-token", token) + } var dialErr error // The incarnation has a private socket namespace. No fallback to run/. socket := filepath.Join(s.cfg.Root, "vms", request.VMID, "incarnations", request.Fence.Incarnation, "run", kinds[request.Kind]) diff --git a/worker/internal/slotgate/management_intent.go b/worker/internal/slotgate/management_intent.go new file mode 100644 index 000000000..930f279a5 --- /dev/null +++ b/worker/internal/slotgate/management_intent.go @@ -0,0 +1,103 @@ +package slotgate + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strconv" + "strings" +) + +const initialPresetID = "claude-docker-v1" +const initialPublishMax = 64 << 10 +const commandPublish CommandKind = "publish-initial" + +// ManagementIntent is an exact issued grant, not a reusable node capability. +// All decimal fields remain strings across the PG/JavaScript boundary. +type ManagementIntent struct { + Action string `json:"action"` + NodeID string `json:"node_id"` + VMID string `json:"vm_id"` + AllocationID string `json:"allocation_id"` + OperationID string `json:"operation_id"` + OwnerID string `json:"owner_id"` + CommandID string `json:"command_id"` + Epoch string `json:"epoch"` + Fence Fence `json:"fence"` + ExpectedManifestRevision string `json:"expected_manifest_revision"` + ManifestRevision string `json:"manifest_revision"` + PayloadDigest string `json:"payload_digest"` + PresetID string `json:"preset_id"` + PolicyDigest string `json:"policy_digest"` + ManifestDigest string `json:"manifest_digest"` +} + +// CommandAuthorizer holds the node/VM grant's shared authorization lock only +// across dispatch's bounded local journal/rename work, never upload or Engine I/O. +type CommandAuthorizer func(context.Context, ManagementIntent, func() error) error + +func (i ManagementIntent) digest() string { + return hashText(strings.Join([]string{"slotgate-command-v1", i.Action, i.NodeID, i.VMID, i.AllocationID, + i.OperationID, i.OwnerID, i.CommandID, i.Epoch, strconv.FormatUint(i.Fence.Generation, 10), i.Fence.Incarnation, + i.ExpectedManifestRevision, i.ManifestRevision, i.PayloadDigest, i.PresetID, i.PolicyDigest, i.ManifestDigest}, "\x00")) +} + +func hashText(value string) string { return hashBytes([]byte(value)) } +func hashBytes(value []byte) string { sum := sha256.Sum256(value); return hex.EncodeToString(sum[:]) } +func validDigest(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 32 && strings.ToLower(value) == value +} + +func validManagementIntent(i ManagementIntent) bool { + if !validAllocation(AllocationIntent{i.NodeID, i.VMID, i.AllocationID, i.OperationID, i.OwnerID}, i.Epoch) || + !vmIDRe.MatchString(i.CommandID) || i.Fence.Generation != 1 || i.Fence.Executor != (ExecutorIdentity{}) || + i.PresetID != initialPresetID || !validDigest(i.PolicyDigest) || i.ManifestRevision != "1" { + return false + } + incarnation, err := hex.DecodeString(i.Fence.Incarnation) + if err != nil || len(incarnation) != 16 || strings.ToLower(i.Fence.Incarnation) != i.Fence.Incarnation { + return false + } + switch i.Action { + case "publish-initial": + return i.ExpectedManifestRevision == "0" && validDigest(i.PayloadDigest) && i.ManifestDigest == "" + case "launch": + return i.ExpectedManifestRevision == "1" && i.PayloadDigest == "" && validDigest(i.ManifestDigest) + } + return false +} + +func (p DockerLaunchPolicy) digest() string { + return hashText(strings.Join([]string{"slotgate-policy-v1", initialPresetID, p.ImageID, p.NetworkID, + strconv.Itoa(p.UID), strconv.Itoa(p.GID)}, "\x00")) +} + +type PublicationReceipt struct { + Intent ManagementIntent `json:"intent"` + IntentDigest string `json:"intent_digest"` + ManifestRevision string `json:"manifest_revision"` + ManifestDigest string `json:"manifest_digest"` +} + +type LaunchReceipt struct { + Intent ManagementIntent `json:"intent"` + IntentDigest string `json:"intent_digest"` + ManifestRevision string `json:"manifest_revision"` + ManifestDigest string `json:"manifest_digest"` + Fence Fence `json:"fence"` +} + +// Private file hashes stay in the journal; RPC receipts expose only the +// aggregate manifest and never credentials, internal tokens or their hashes. +type initialManifest struct { + Receipt PublicationReceipt `json:"receipt"` + KernelDigest string `json:"kernel_digest"` + TokenDigest string `json:"token_digest"` + CredentialDigest string `json:"credential_digest"` +} + +func (m initialManifest) digest() string { + return hashText(strings.Join([]string{"slotgate-manifest-v1", m.Receipt.Intent.digest(), "1", + m.KernelDigest, m.TokenDigest, m.CredentialDigest}, "\x00")) +} diff --git a/worker/internal/slotgate/pg_authorizer.go b/worker/internal/slotgate/pg_authorizer.go index d0d2448a0..0c248aa23 100644 --- a/worker/internal/slotgate/pg_authorizer.go +++ b/worker/internal/slotgate/pg_authorizer.go @@ -158,3 +158,72 @@ func (a *pgAllocationAuthorizer) Authorize(ctx context.Context, intent Allocatio } return nil } + +// Missing command protocol/view/role privilege always denies; allocation authorization is never a fallback. +func (a *pgAllocationAuthorizer) AuthorizeCommand(ctx context.Context, intent ManagementIntent, dispatch func() error) error { + if intent.NodeID != a.nodeID || !validManagementIntent(intent) { + return ErrManagementDenied + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + connection, err := pgx.ConnectConfig(ctx, a.config.Copy()) + if err != nil { + return ErrManagementUnavailable + } + defer func() { + closed, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + connection.Close(closed) + }() + var directRole bool + if err = connection.QueryRow(ctx, "SELECT current_user=session_user").Scan(&directRole); err != nil || !directRole { + return ErrManagementDenied + } + q := pgx.Identifier{a.schema}.Sanitize() + var nodeID, applicationSchema string + var version int + if err = connection.QueryRow(ctx, "SELECT node_id,application_schema,protocol_version FROM "+q+".reader_identity()").Scan(&nodeID, &applicationSchema, &version); err != nil { + return ErrManagementUnavailable + } + expectedSchema := sha256.Sum256([]byte(applicationSchema)) + if version != 1 || nodeID != a.nodeID || a.schema != "vm2api_node_"+hex.EncodeToString(expectedSchema[:16]) { + return ErrManagementDenied + } + tx, err := connection.BeginTx(ctx, pgx.TxOptions{AccessMode: pgx.ReadOnly}) + if err != nil { + return ErrManagementUnavailable + } + defer func() { + rollback, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + tx.Rollback(rollback) + }() + if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))", + "vm2api:node-management:"+a.schema, "node:"+a.nodeID+":vm:"+intent.VMID); err != nil { + return ErrManagementUnavailable + } + var epoch, digest string + err = tx.QueryRow(ctx, "SELECT epoch::text,intent_digest FROM "+q+".authorized_commands WHERE allocation_id=$1 AND node_id=$2 AND vm_id=$3 AND operation_id=$4 AND owner_id=$5 AND command_id=$6 AND action=$7", + intent.AllocationID, intent.NodeID, intent.VMID, intent.OperationID, intent.OwnerID, intent.CommandID, intent.Action).Scan(&epoch, &digest) + if err == pgx.ErrNoRows { + return ErrManagementDenied + } + if err != nil { + return ErrManagementUnavailable + } + if epoch != intent.Epoch || digest != intent.digest() { + return ErrManagementDenied + } + if ctx.Err() != nil { + return ErrManagementUnavailable + } + // Only bounded local journal I/O belongs in dispatch. Never Docker or an + // outbound RPC. A failed COMMIT cannot acknowledge a local receipt. + if err = dispatch(); err != nil { + return err + } + if err = tx.Commit(ctx); err != nil { + return ErrManagementUnavailable + } + return nil +} diff --git a/worker/internal/slotgate/publish_files_linux.go b/worker/internal/slotgate/publish_files_linux.go new file mode 100644 index 000000000..1210caf9b --- /dev/null +++ b/worker/internal/slotgate/publish_files_linux.go @@ -0,0 +1,344 @@ +//go:build linux + +package slotgate + +import ( + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +type initialFileTree struct { + parent *os.File + stage *os.File + name string + incarnation string + policy DockerLaunchPolicy +} + +func secureRoot(root string) (*os.File, error) { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return nil, ErrInvalidAuthority + } + fd, err := unix.Open("/", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + for _, part := range strings.Split(strings.TrimPrefix(root, "/"), "/") { + if part == "" { + continue + } + next, e := unix.Openat(fd, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + unix.Close(fd) + if e != nil { + return nil, e + } + fd = next + } + file := os.NewFile(uintptr(fd), root) + if err = secureDirectory(file, os.Geteuid()); err != nil { + file.Close() + return nil, err + } + return file, nil +} + +func secureDirectory(dir *os.File, owner int) error { + var stat unix.Stat_t + if err := unix.Fstat(int(dir.Fd()), &stat); err != nil { + return err + } + if stat.Mode&unix.S_IFMT != unix.S_IFDIR || int(stat.Uid) != owner || stat.Mode&0o077 != 0 { + return ErrInvalidAuthority + } + return nil +} + +func openDirectoryAt(parent *os.File, name string, create bool, owner int) (*os.File, error) { + if name == "" || name == "." || name == ".." || strings.Contains(name, "/") { + return nil, ErrInvalidAuthority + } + if create { + if err := unix.Mkdirat(int(parent.Fd()), name, 0o700); err != nil && !errors.Is(err, unix.EEXIST) { + return nil, err + } + } + fd, err := unix.Openat(int(parent.Fd()), name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), name) + if err = secureDirectory(file, owner); err != nil { + file.Close() + return nil, err + } + return file, nil +} + +func incarnationParent(root, vm string, create bool) (*os.File, error) { + if !vmIDRe.MatchString(vm) { + return nil, ErrInvalidAuthority + } + current, err := secureRoot(root) + if err != nil { + return nil, err + } + for _, name := range []string{"vms", vm, "incarnations"} { + next, e := openDirectoryAt(current, name, create, os.Geteuid()) + if e == nil && create { + e = current.Sync() + } + current.Close() + if e != nil { + if next != nil { + next.Close() + } + return nil, e + } + current = next + } + return current, nil +} + +func writeInitialFile(dir *os.File, name string, data []byte, uid, gid int, mode uint32) error { + fd, err := unix.Openat(int(dir.Fd()), name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, mode) + if err != nil { + return err + } + file := os.NewFile(uintptr(fd), name) + defer file.Close() + var stat unix.Stat_t + if err = unix.Fstat(fd, &stat); err != nil { + return err + } + if stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Nlink != 1 { + return ErrInvalidAuthority + } + if _, err = file.Write(data); err != nil { + return err + } + if int(stat.Uid) != uid || int(stat.Gid) != gid { + if err = unix.Fchown(fd, uid, gid); err != nil { + return err + } + } + if err = unix.Fchmod(fd, mode); err != nil { + return err + } + return file.Sync() +} + +func stageInitialFiles(root string, intent ManagementIntent, policy DockerLaunchPolicy, kernel, token, credential []byte, fault func(string) error) (*initialFileTree, error) { + parent, err := incarnationParent(root, intent.VMID, true) + if err != nil { + return nil, err + } + id, err := randomAuthorityID() + if err != nil { + parent.Close() + return nil, err + } + tree := &initialFileTree{parent: parent, name: ".stage-" + id, incarnation: intent.Fence.Incarnation, policy: policy} + fail := func(err error) (*initialFileTree, error) { tree.Close(); return nil, err } + if err = unix.Mkdirat(int(parent.Fd()), tree.name, 0o700); err != nil { + return fail(err) + } + tree.stage, err = openDirectoryAt(parent, tree.name, false, os.Geteuid()) + if err != nil { + return fail(err) + } + for _, name := range []string{"config", "home", "claude", "run"} { + dir, e := openDirectoryAt(tree.stage, name, true, os.Geteuid()) + if e != nil { + return fail(e) + } + if name == "config" { + mode := uint32(0o600) + if policy.UID != os.Geteuid() { + mode = 0o640 + } + e = writeInitialFile(dir, "kernel.json", kernel, os.Geteuid(), policy.GID, mode) + if e == nil { + e = writeInitialFile(dir, "internal.token", token, os.Geteuid(), policy.GID, mode) + } + } else if name == "claude" { + e = writeInitialFile(dir, "credentials.json", credential, policy.UID, policy.GID, 0o400) + if e == nil { + e = unix.Symlinkat("credentials.json", int(dir.Fd()), ".credentials.json") + } + } + if e == nil && name != "config" { + e = unix.Fchown(int(dir.Fd()), policy.UID, policy.GID) + } + if e == nil { + e = dir.Sync() + } + dir.Close() + if e != nil { + return fail(e) + } + } + if err = tree.stage.Sync(); err != nil { + return fail(err) + } + if err = parent.Sync(); err != nil { + return fail(err) + } + if err = publishFault(fault, "staged"); err != nil { + return fail(err) + } + return tree, nil +} + +func readInitialFile(dir *os.File, name string, owner int, max int64) ([]byte, error) { + fd, err := unix.Openat(int(dir.Fd()), name, unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), name) + defer file.Close() + var stat unix.Stat_t + if err = unix.Fstat(fd, &stat); err != nil { + return nil, err + } + if stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Nlink != 1 || int(stat.Uid) != owner || stat.Mode&0o022 != 0 || stat.Size <= 0 || stat.Size > max { + return nil, ErrInvalidAuthority + } + data, err := io.ReadAll(io.LimitReader(file, max+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > max { + return nil, ErrInvalidAuthority + } + return data, nil +} + +func checkInitialTree(base *os.File, policy DockerLaunchPolicy, manifest initialManifest, credentials bool) (string, error) { + if err := secureDirectory(base, os.Geteuid()); err != nil { + return "", err + } + // Fixed enumeration also refuses unexpected directories/files in staging. + entries, err := base.ReadDir(-1) + if err != nil { + return "", err + } + if _, err = base.Seek(0, 0); err != nil { + return "", err + } + if len(entries) != 4 { + return "", ErrInvalidAuthority + } + for _, entry := range entries { + if entry.Name() != "config" && entry.Name() != "home" && entry.Name() != "claude" && entry.Name() != "run" { + return "", ErrInvalidAuthority + } + } + config, err := openDirectoryAt(base, "config", false, os.Geteuid()) + if err != nil { + return "", err + } + defer config.Close() + entries, err = config.ReadDir(-1) + if err != nil || len(entries) != 2 { + return "", ErrInvalidAuthority + } + kernel, err := readInitialFile(config, "kernel.json", os.Geteuid(), gate2LineMax) + if err != nil { + return "", err + } + token, err := readInitialFile(config, "internal.token", os.Geteuid(), 256) + if err != nil { + return "", err + } + if hashBytes(kernel) != manifest.KernelDigest || hashBytes(token) != manifest.TokenDigest { + return "", ErrInvalidAuthority + } + value := strings.TrimSuffix(string(token), "\n") + decoded, e := hex.DecodeString(value) + if e != nil || len(decoded) != 32 || strings.ToLower(value) != value || string(token) != value+"\n" { + return "", ErrInvalidAuthority + } + for _, name := range []string{"home", "claude", "run"} { + dir, e := openDirectoryAt(base, name, false, policy.UID) + if e != nil { + return "", e + } + if credentials && name == "claude" { + entries, e = dir.ReadDir(-1) + if e == nil && len(entries) != 2 { + e = ErrInvalidAuthority + } + var data []byte + if e == nil { + data, e = readInitialFile(dir, "credentials.json", policy.UID, initialPublishMax) + } + if e == nil && hashBytes(data) != manifest.CredentialDigest { + e = ErrInvalidAuthority + } + link := make([]byte, 64) + var n int + if e == nil { + n, e = unix.Readlinkat(int(dir.Fd()), ".credentials.json", link) + } + if e == nil && string(link[:n]) != "credentials.json" { + e = ErrInvalidAuthority + } + } + dir.Close() + if e != nil { + return "", e + } + } + return value, nil +} + +func (t *initialFileTree) Publish(manifest initialManifest, fault func(string) error) error { + if _, err := checkInitialTree(t.stage, t.policy, manifest, true); err != nil { + return err + } + if err := publishFault(fault, "before-rename"); err != nil { + return err + } + if err := unix.Renameat2(int(t.parent.Fd()), t.name, int(t.parent.Fd()), t.incarnation, unix.RENAME_NOREPLACE); err != nil { + return err + } + if err := publishFault(fault, "after-rename"); err != nil { + return err + } + if err := t.parent.Sync(); err != nil { + return err + } + return publishFault(fault, "before-manifest-journal") +} + +func (t *initialFileTree) Close() { + if t.stage != nil { + t.stage.Close() + } + if t.parent != nil { + t.parent.Close() + } +} + +func readPublishedInitial(root, vm string, fence Fence, policy DockerLaunchPolicy, manifest initialManifest, credentials bool) (string, error) { + if fence.Generation != manifest.Receipt.Intent.Fence.Generation || fence.Incarnation != manifest.Receipt.Intent.Fence.Incarnation { + return "", ErrFenceMismatch + } + parent, err := incarnationParent(root, vm, false) + if err != nil { + return "", err + } + defer parent.Close() + base, err := openDirectoryAt(parent, fence.Incarnation, false, os.Geteuid()) + if err != nil { + return "", err + } + defer base.Close() + return checkInitialTree(base, policy, manifest, credentials) +} diff --git a/worker/internal/slotgate/publish_files_other.go b/worker/internal/slotgate/publish_files_other.go new file mode 100644 index 000000000..cee190f83 --- /dev/null +++ b/worker/internal/slotgate/publish_files_other.go @@ -0,0 +1,16 @@ +//go:build !linux + +package slotgate + +type initialFileTree struct{} + +func stageInitialFiles(string, ManagementIntent, DockerLaunchPolicy, []byte, []byte, []byte, func(string) error) (*initialFileTree, error) { + return nil, errPublishUnsupported +} +func (*initialFileTree) Publish(initialManifest, func(string) error) error { + return errPublishUnsupported +} +func (*initialFileTree) Close() {} +func readPublishedInitial(string, string, Fence, DockerLaunchPolicy, initialManifest, bool) (string, error) { + return "", errPublishUnsupported +} diff --git a/worker/internal/slotgate/publish_initial.go b/worker/internal/slotgate/publish_initial.go new file mode 100644 index 000000000..fb6bd3775 --- /dev/null +++ b/worker/internal/slotgate/publish_initial.go @@ -0,0 +1,340 @@ +package slotgate + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "io" + "strings" +) + +type initialCredential struct { + Type string `json:"type"` + AccessToken string `json:"access_token,omitempty"` + RefreshToken string `json:"refresh_token,omitempty"` + APIKey string `json:"api_key,omitempty"` + ExpiresAt int64 `json:"expires_at,omitempty"` + Scopes []string `json:"scopes,omitempty"` +} +type initialPayload struct { + Credential initialCredential `json:"credential"` +} + +// No filename, path, settings, environment, executable, URL, uid or mode is +// accepted from the runtime controller. The only uploaded file is credentials. +func initialCredentialBytes(data []byte) ([]byte, error) { + var payload initialPayload + if len(data) == 0 || len(data) > initialPublishMax || strictJSON(data, &payload) != nil { + return nil, ErrInvalidAuthority + } + c := payload.Credential + if len(c.AccessToken) > 16<<10 || len(c.RefreshToken) > 16<<10 || len(c.APIKey) > 16<<10 || c.ExpiresAt < 0 || len(c.Scopes) > 32 { + return nil, ErrInvalidAuthority + } + for _, scope := range c.Scopes { + if len(scope) == 0 || len(scope) > 128 || strings.ContainsAny(scope, "\x00\r\n") { + return nil, ErrInvalidAuthority + } + } + var document map[string]any + switch c.Type { + case "apikey": + if strings.TrimSpace(c.APIKey) == "" || c.AccessToken != "" || c.RefreshToken != "" || c.ExpiresAt != 0 || len(c.Scopes) != 0 { + return nil, ErrInvalidAuthority + } + document = map[string]any{"type": "apikey", "authScheme": "x_api_key", "kinGeneration": 1, "anthropicApiKey": map[string]any{"apiKey": c.APIKey, "baseUrl": "https://api.anthropic.com"}} + case "oauth", "setup-token", "official-setup-token": + if strings.TrimSpace(c.AccessToken) == "" || c.APIKey != "" || (c.Type != "oauth" && c.RefreshToken != "") { + return nil, ErrInvalidAuthority + } + oauth := map[string]any{"accessToken": c.AccessToken, "kinGeneration": 1} + if c.RefreshToken != "" { + oauth["refreshToken"] = c.RefreshToken + } + if c.ExpiresAt > 0 { + oauth["expiresAt"] = c.ExpiresAt + } + if c.Type == "oauth" { + if len(c.Scopes) > 0 { + oauth["scopes"] = c.Scopes + } + } else { + oauth["type"] = c.Type + oauth["scopes"] = []string{"user:inference"} + } + document = map[string]any{"type": c.Type, "authScheme": "authorization_bearer", "kinGeneration": 1, "claudeAiOauth": oauth} + default: + return nil, ErrInvalidAuthority + } + return json.Marshal(document) +} + +// DisallowUnknownFields alone accepts duplicate keys. Detect them before +// decoding so the controller and node cannot assign different input meanings. +func strictJSON(data []byte, value any) error { + keys := json.NewDecoder(bytes.NewReader(data)) + var scan func() error + scan = func() error { + token, err := keys.Token() + if err != nil { + return err + } + if delim, ok := token.(json.Delim); ok { + switch delim { + case '{': + seen := map[string]bool{} + for keys.More() { + key, err := keys.Token() + if err != nil { + return err + } + name, ok := key.(string) + if !ok || seen[name] { + return ErrInvalidAuthority + } + seen[name] = true + if err = scan(); err != nil { + return err + } + } + case '[': + for keys.More() { + if err = scan(); err != nil { + return err + } + } + default: + return ErrInvalidAuthority + } + _, err = keys.Token() + if err != nil { + return err + } + } + return nil + } + if err := scan(); err != nil { + return err + } + if _, err := keys.Token(); err != io.EOF { + return ErrInvalidAuthority + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(value); err != nil { + return err + } + if decoder.Decode(new(any)) != io.EOF { + return ErrInvalidAuthority + } + return nil +} + +func initialKernelBytes(vm, token string) ([]byte, error) { + return json.Marshal(map[string]any{ + "vm_id": vm, "socket_path": "/run/kin/kernel.sock", "credential_path": "/home/kincli/.claude/credentials.json", + "internal_token": token, "provider": "local_cli", "dataplane": "wrap", "claude_bin": "/opt/kin/cli-node", + "proxy_url": "", "proxy_required": false, "runtime_kind": "docker", "delivery_mode": "realtime", + "refresh_skew_seconds": 300, "request_timeout_seconds": 0, "first_byte_timeout_seconds": 600, "idle_timeout_seconds": 180, + "max_request_bytes": 16 << 20, "max_response_bytes": 64 << 20, "max_event_bytes": 32 << 20, "slots_per_worker": 1, + "timezone": "UTC", "test_endpoints": false, + }) +} + +type initialPublisher struct { + authority *Authority + root string + nodeID string + policy DockerLaunchPolicy + authorize CommandAuthorizer + // Fault injection stays node-local; it is never part of Config or RPC. + fault func(string) error +} + +func (p *initialPublisher) Publish(ctx context.Context, intent ManagementIntent, length int, source io.Reader) (PublicationReceipt, error) { + if p.authorize == nil || intent.NodeID != p.nodeID { + return PublicationReceipt{}, ErrManagementDenied + } + if !validManagementIntent(intent) || intent.Action != "publish-initial" || intent.PolicyDigest != p.policy.digest() || length <= 0 || length > initialPublishMax { + return PublicationReceipt{}, ErrInvalidAuthority + } + var handle CommandHandle + var receipt *PublicationReceipt + err := p.authorize(ctx, intent, func() error { + var err error + handle, receipt, err = p.authority.beginInitialPublish(intent) + return err + }) + if err != nil { + return PublicationReceipt{}, err + } + if receipt != nil { + return *receipt, nil + } + if ctx.Err() != nil { + return PublicationReceipt{}, ctx.Err() + } + data := make([]byte, length) + if _, err = io.ReadFull(source, data); err != nil { + return PublicationReceipt{}, err + } + if ctx.Err() != nil { + return PublicationReceipt{}, ctx.Err() + } + if hashBytes(data) != intent.PayloadDigest { + return PublicationReceipt{}, ErrInvalidAuthority + } + credential, err := initialCredentialBytes(data) + if err != nil { + return PublicationReceipt{}, err + } + var secret [32]byte + if _, err = rand.Read(secret[:]); err != nil { + return PublicationReceipt{}, err + } + token := hex.EncodeToString(secret[:]) + kernel, err := initialKernelBytes(intent.VMID, token) + if err != nil { + return PublicationReceipt{}, err + } + manifest := initialManifest{Receipt: PublicationReceipt{Intent: intent, IntentDigest: intent.digest(), ManifestRevision: "1"}, KernelDigest: hashBytes(kernel), TokenDigest: hashText(token + "\n"), CredentialDigest: hashBytes(credential)} + manifest.Receipt.ManifestDigest = manifest.digest() + tree, err := stageInitialFiles(p.root, intent, p.policy, kernel, []byte(token+"\n"), credential, p.fault) + if err != nil { + return PublicationReceipt{}, err + } + defer tree.Close() + if ctx.Err() != nil { + return PublicationReceipt{}, ctx.Err() + } + err = p.authorize(ctx, intent, func() error { + return p.authority.completeInitialPublish(handle, intent, manifest, func() error { return tree.Publish(manifest, p.fault) }) + }) + if err != nil { + return PublicationReceipt{}, err + } + return manifest.Receipt, nil +} + +func (a *Authority) beginInitialPublish(intent ManagementIntent) (CommandHandle, *PublicationReceipt, error) { + a.mu.Lock() + defer a.mu.Unlock() + if !validManagementIntent(intent) || intent.Action != "publish-initial" { + return CommandHandle{}, nil, ErrInvalidAuthority + } + state, err := a.match(intent.VMID, intent.Fence) + if err != nil { + return CommandHandle{}, nil, err + } + if state.Phase != PhaseVacant || state.Allocation == nil || state.Allocation.AllocationID != intent.AllocationID || state.Allocation.NodeID != intent.NodeID || state.Allocation.Fence != intent.Fence { + return CommandHandle{}, nil, ErrAuthorityPhase + } + if old, ok := state.Commands[intent.CommandID]; ok { + if old.Kind == commandPublish && old.Management != nil && *old.Management == intent && old.Status == CommandCompleted && state.InitialManifest != nil { + receipt := state.InitialManifest.Receipt + return CommandHandle{}, &receipt, nil + } + return CommandHandle{}, nil, ErrCommandExists + } + if state.InitialManifest != nil || hasPendingMutation(state) { + return CommandHandle{}, nil, ErrPendingMutation + } + token, err := randomAuthorityID() + if err != nil { + return CommandHandle{}, nil, err + } + copy := intent + state.Commands[intent.CommandID] = CommandRecord{ID: intent.CommandID, Kind: commandPublish, Status: CommandPending, Fence: intent.Fence, Runtime: a.runtime, Token: token, Dispatched: true, Management: ©} + if _, err = a.commit(intent.VMID, state); err != nil { + return CommandHandle{}, nil, err + } + return CommandHandle{VMID: intent.VMID, ID: intent.CommandID, Fence: intent.Fence, runtime: a.runtime, token: token}, nil, nil +} + +func (a *Authority) completeInitialPublish(handle CommandHandle, intent ManagementIntent, manifest initialManifest, publish func() error) error { + a.mu.Lock() + defer a.mu.Unlock() + state, command, err := a.command(handle) + if err != nil { + return err + } + if state.Phase != PhaseVacant || state.InitialManifest != nil || command.Kind != commandPublish || command.Status != CommandPending || command.Management == nil || *command.Management != intent || manifest.Receipt.Intent != intent { + return ErrAuthorityPhase + } + if err = publish(); err != nil { + return err + } + command.Status = CommandCompleted + state.Commands[handle.ID] = command + state.InitialManifest = &manifest + _, err = a.commit(handle.VMID, state) + return err +} + +func validateInitialManifest(vm string, state AuthorityState) error { + m := state.InitialManifest + if m == nil { + return nil + } + i := m.Receipt.Intent + if state.Allocation == nil || !validManagementIntent(i) || i.Action != "publish-initial" || i.VMID != vm || i.AllocationID != state.Allocation.AllocationID || i.NodeID != state.Allocation.NodeID || i.Fence != state.Allocation.Fence || + m.Receipt.IntentDigest != i.digest() || m.Receipt.ManifestRevision != "1" || !validDigest(m.KernelDigest) || !validDigest(m.TokenDigest) || !validDigest(m.CredentialDigest) || m.Receipt.ManifestDigest != m.digest() { + return ErrInvalidAuthority + } + command, ok := state.Commands[i.CommandID] + if !ok || command.Kind != commandPublish || command.Status != CommandCompleted || command.Management == nil || *command.Management != i { + return ErrInvalidAuthority + } + return nil +} + +func checkInitialManifest(state AuthorityState, policy DockerLaunchPolicy) error { + if state.Allocation == nil || state.InitialManifest == nil || validateInitialManifest(state.Allocation.VMID, state) != nil || state.InitialManifest.Receipt.Intent.PolicyDigest != policy.digest() { + return ErrInvalidAuthority + } + if state.InitialManifest.Receipt.Intent.Fence != state.Fence { + return ErrFenceMismatch + } + return nil +} + +func publishFault(fault func(string) error, point string) error { + if fault != nil { + return fault(point) + } + return nil +} + +var errPublishUnsupported = errors.New("initial_publish_unsupported") + +func (a *Authority) beginManifestLaunch(intent ManagementIntent) (CommandHandle, error) { + a.mu.Lock() + defer a.mu.Unlock() + if !validManagementIntent(intent) || intent.Action != "launch" { + return CommandHandle{}, ErrInvalidAuthority + } + state, err := a.match(intent.VMID, intent.Fence) + if err != nil { + return CommandHandle{}, err + } + if state.Phase != PhaseVacant || state.InitialManifest == nil || state.Allocation == nil || state.Allocation.AllocationID != intent.AllocationID || state.Allocation.NodeID != intent.NodeID || + state.InitialManifest.Receipt.ManifestDigest != intent.ManifestDigest || state.InitialManifest.Receipt.Intent.PolicyDigest != intent.PolicyDigest || hasPendingMutation(state) { + return CommandHandle{}, ErrAuthorityPhase + } + if _, ok := state.Commands[intent.CommandID]; ok { + return CommandHandle{}, ErrCommandExists + } + token, err := randomAuthorityID() + if err != nil { + return CommandHandle{}, err + } + copy := intent + state.Commands[intent.CommandID] = CommandRecord{ID: intent.CommandID, Kind: commandLaunch, Status: CommandPending, Fence: intent.Fence, Runtime: a.runtime, Token: token, Management: ©} + if _, err = a.commit(intent.VMID, state); err != nil { + return CommandHandle{}, err + } + return CommandHandle{VMID: intent.VMID, ID: intent.CommandID, Fence: intent.Fence, runtime: a.runtime, token: token}, nil +} diff --git a/worker/internal/slotgate/publish_initial_test.go b/worker/internal/slotgate/publish_initial_test.go index 37ea76f3f..f5e354e41 100644 --- a/worker/internal/slotgate/publish_initial_test.go +++ b/worker/internal/slotgate/publish_initial_test.go @@ -2,13 +2,79 @@ package slotgate import ( "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" "testing" + "time" ) +func allowNodeCommandForTest(_ context.Context, _ ManagementIntent, dispatch func() error) error { + return dispatch() +} +func managementConfigForTest() *PGManagementConfig { + return &PGManagementConfig{ConnectionString: "postgres://synthetic_reader@127.0.0.1:1/synthetic_test?sslmode=disable", AuthoritySchema: "vm2api_node_" + strings.Repeat("a", 32), NodeID: "node-test"} +} +func publicationIntentForTest(allocation AllocationReceipt, policy DockerLaunchPolicy, payload []byte) ManagementIntent { + return ManagementIntent{Action: "publish-initial", NodeID: allocation.NodeID, VMID: allocation.VMID, AllocationID: allocation.AllocationID, OperationID: "publish-operation", OwnerID: "publish-owner", CommandID: "publish-command", Epoch: "1", Fence: allocation.Fence, ExpectedManifestRevision: "0", ManifestRevision: "1", PayloadDigest: hashBytes(payload), PresetID: initialPresetID, PolicyDigest: policy.digest()} +} +func launchIntentForTest(state AuthorityState, id string) ManagementIntent { + i := state.InitialManifest.Receipt.Intent + i.Action = "launch" + i.OperationID = "launch-operation" + i.OwnerID = "launch-owner" + i.CommandID = id + i.ExpectedManifestRevision = "1" + i.PayloadDigest = "" + i.ManifestDigest = state.InitialManifest.Receipt.ManifestDigest + return i +} + +type publicationFixture struct { + a *Authority + p *initialPublisher + l *dockerLauncher + engine *fakeLaunchEngine + state AuthorityState + root string + intent ManagementIntent + payload []byte +} + +func publishingFixture(t *testing.T) *publicationFixture { + t.Helper() + a, l, engine, state, root := launchFixtureConfigured(t, false, "none") + if err := os.RemoveAll(filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation)); err != nil { + t.Fatal(err) + } + payload := []byte(`{"credential":{"type":"oauth","access_token":"synthetic-access","refresh_token":"synthetic-refresh","scopes":["user:inference"]}}`) + intent := publicationIntentForTest(*state.Allocation, l.policy, payload) + return &publicationFixture{a: a, p: &initialPublisher{authority: a, root: root, nodeID: intent.NodeID, policy: l.policy, authorize: allowNodeCommandForTest}, l: l, engine: engine, state: state, root: root, intent: intent, payload: payload} +} +func (f *publicationFixture) publish(t *testing.T) PublicationReceipt { + t.Helper() + receipt, err := f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(string(f.payload))) + if err != nil { + t.Fatal(err) + } + return receipt +} +func (f *publicationFixture) private() string { + return filepath.Join(f.root, "vms", "v", "incarnations", f.intent.Fence.Incarnation) +} + // An existing directory is not proof that node-owned publication completed. // In particular, a leftover tree after rename/journal failure must not launch. func TestInitialPublishUnrecordedTreeCannotReachDocker(t *testing.T) { - _, launcher, engine, state, _ := launchFixture(t) + _, launcher, engine, state, _ := launchFixtureConfigured(t, false, "none") if _, err := launcher.Launch(context.Background(), "v", state.Fence, "unrecorded-publication"); err == nil { t.Fatal("launch accepted files without a durable completed manifest") } @@ -27,3 +93,468 @@ func TestInitialPublishRequiresIndependentManagementAuthorization(t *testing.T) t.Fatalf("publish without an independent action authorizer: %+v", reply) } } + +func TestInitialPublishGeneratesPrivateFilesAndPersistsOnlyDigests(t *testing.T) { + f := publishingFixture(t) + receipt := f.publish(t) + if receipt.Intent != f.intent || receipt.IntentDigest != f.intent.digest() || receipt.ManifestRevision != "1" || !validDigest(receipt.ManifestDigest) { + t.Fatalf("receipt identity: %+v", receipt) + } + state, err := f.a.State("v") + if err != nil { + t.Fatal(err) + } + if state.InitialManifest == nil || state.Commands[f.intent.CommandID].Status != CommandCompleted { + t.Fatalf("missing durable manifest: %+v", state) + } + token, err := readPublishedInitial(f.root, "v", f.intent.Fence, f.l.policy, *state.InitialManifest, true) + if err != nil || len(token) != 64 { + t.Fatalf("private files: %v", err) + } + kernel, err := os.ReadFile(filepath.Join(f.private(), "config", "kernel.json")) + if err != nil { + t.Fatal(err) + } + var config map[string]any + if json.Unmarshal(kernel, &config) != nil || config["internal_token"] != token || config["claude_bin"] != "/opt/kin/cli-node" || config["test_endpoints"] != false { + t.Fatal("node did not generate the approved config") + } + journal, err := os.ReadFile(filepath.Join(f.a.root, "authority.json")) + if err != nil { + t.Fatal(err) + } + encoded, _ := json.Marshal(receipt) + for _, body := range [][]byte{journal, encoded} { + if strings.Contains(string(body), token) || strings.Contains(string(body), "synthetic-access") || strings.Contains(string(body), "synthetic-refresh") { + t.Fatal("secret bytes escaped into journal or receipt") + } + } + second := f.publish(t) + if second != receipt { + t.Fatal("completed replay changed its receipt") + } + state.InitialManifest.Receipt.ManifestDigest = "mutated-local-copy" + stored, _ := f.a.State("v") + if stored.InitialManifest.Receipt != receipt { + t.Fatal("state exposed mutable manifest") + } +} + +func TestInitialPublishFaultsRemainUnknownAndNeverReissue(t *testing.T) { + for _, point := range []string{"staged", "before-rename", "after-rename", "before-manifest-journal"} { + t.Run(point, func(t *testing.T) { + f := publishingFixture(t) + failure := errors.New("synthetic " + point) + f.p.fault = func(got string) error { + if got == point { + return failure + } + return nil + } + _, err := f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(string(f.payload))) + if err != failure { + t.Fatalf("fault identity: %v", err) + } + state, _ := f.a.State("v") + if state.InitialManifest != nil || state.Commands[f.intent.CommandID].Status != CommandPending { + t.Fatalf("unknown falsely completed: %+v", state) + } + f.p.fault = nil + if _, err = f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(string(f.payload))); !errors.Is(err, ErrCommandExists) { + t.Fatalf("unknown was reissued: %v", err) + } + if _, err = f.l.Launch(context.Background(), "v", f.intent.Fence, "launch-after-unknown"); err == nil { + t.Fatal("unknown publication launched") + } + if _, err = f.a.Revoke("v", f.intent.Fence); err != nil { + t.Fatal(err) + } + if _, err = f.a.Advance(context.Background(), "v", f.intent.Fence); !errors.Is(err, ErrPendingMutation) { + t.Fatalf("unknown publication advanced: %v", err) + } + if err = f.a.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, f.a.root, terminatedExecutor) + state, _ = reopened.State("v") + if state.Phase != PhaseRevoking || state.Commands[f.intent.CommandID].Status != CommandUnknown { + t.Fatalf("restart adopted unknown publication: %+v", state) + } + }) + } +} + +func TestInitialPublishRevalidatesGrantAndOriginalFenceBeforeRename(t *testing.T) { + for _, revoke := range []bool{false, true} { + t.Run(fmt.Sprint(revoke), func(t *testing.T) { + f := publishingFixture(t) + calls := 0 + f.p.authorize = func(ctx context.Context, intent ManagementIntent, dispatch func() error) error { + calls++ + if calls == 2 { + if revoke { + if _, err := f.a.Revoke("v", f.intent.Fence); err != nil { + t.Fatal(err) + } + } else { + return ErrManagementDenied + } + } + return dispatch() + } + if _, err := f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(string(f.payload))); err == nil { + t.Fatal("revoked/fence-changed publication succeeded") + } + if _, err := os.Lstat(f.private()); !os.IsNotExist(err) { + t.Fatalf("final path changed without fresh original authorization: %v", err) + } + }) + } +} + +func TestInitialPublishRejectsSymlinksHardlinksUnexpectedFilesAndChangedBytes(t *testing.T) { + for _, kind := range []string{"parent-symlink", "kernel-hardlink", "credential-symlink", "extra-file", "changed-token", "existing-final"} { + t.Run(kind, func(t *testing.T) { + f := publishingFixture(t) + if kind == "parent-symlink" { + parent := filepath.Dir(f.private()) + if err := os.Remove(parent); err != nil { + t.Fatal(err) + } + if err := os.Symlink(t.TempDir(), parent); err != nil { + t.Fatal(err) + } + } + f.p.fault = func(point string) error { + if point != "staged" { + return nil + } + parent := filepath.Dir(f.private()) + entries, err := os.ReadDir(parent) + if err != nil { + return err + } + var stage string + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".stage-") { + stage = filepath.Join(parent, entry.Name()) + } + } + switch kind { + case "kernel-hardlink": + return os.Link(filepath.Join(stage, "config", "kernel.json"), filepath.Join(f.root, "extra-hardlink")) + case "credential-symlink": + file := filepath.Join(stage, "claude", "credentials.json") + if err := os.Remove(file); err != nil { + return err + } + return os.Symlink(filepath.Join(f.a.root, "authority.json"), file) + case "extra-file": + return os.WriteFile(filepath.Join(stage, "extra"), []byte("denied"), 0o600) + case "changed-token": + return os.WriteFile(filepath.Join(stage, "config", "internal.token"), []byte(strings.Repeat("c", 64)+"\n"), 0o600) + case "existing-final": + return os.Mkdir(f.private(), 0o700) + } + return nil + } + if _, err := f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(string(f.payload))); err == nil { + t.Fatal("unsafe tree was published") + } + state, _ := f.a.State("v") + if state.InitialManifest != nil { + t.Fatal("unsafe tree received completed manifest") + } + }) + } +} + +func TestInitialPublishRejectsArbitraryConfigAndMalformedCredentialPayload(t *testing.T) { + for _, payload := range []string{ + `{"credential":{"type":"oauth","access_token":"synthetic"},"path":"../../authority.json"}`, + `{"credential":{"type":"oauth","access_token":"synthetic","base_url":"http://unapproved"}}`, + `{"credential":{"type":"oauth","access_token":"synthetic","access_token":"changed"}}`, + `{"credential":{"type":"oauth","access_token":"synthetic"}} {}`, + `{"credential":{"type":"arbitrary","access_token":"synthetic"}}`, + `{"credential":{"type":"official-setup-token","access_token":"synthetic","refresh_token":"unexpected"}}`, + } { + t.Run(hashText(payload)[:8], func(t *testing.T) { + f := publishingFixture(t) + f.payload = []byte(payload) + f.intent.PayloadDigest = hashBytes(f.payload) + if _, err := f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(payload)); err == nil { + t.Fatal("unapproved payload accepted") + } + if _, err := os.Lstat(f.private()); !os.IsNotExist(err) { + t.Fatal("invalid payload touched destination") + } + }) + } +} + +func TestInitialPublishJournalFailureCannotReturnCompletionOrLaunch(t *testing.T) { + f := publishingFixture(t) + f.p.fault = func(point string) error { + if point != "before-manifest-journal" { + return nil + } + journal := filepath.Join(f.a.root, "authority.json") + if err := os.Rename(journal, journal+".saved"); err != nil { + return err + } + return os.Mkdir(journal, 0o700) + } + if _, err := f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(string(f.payload))); err == nil { + t.Fatal("journal failure returned completed") + } + if _, err := f.a.State("v"); !errors.Is(err, ErrAuthorityUnavailable) { + t.Fatalf("journal failure did not poison authority: %v", err) + } + journal := filepath.Join(f.a.root, "authority.json") + if err := os.Remove(journal); err != nil { + t.Fatal(err) + } + if err := os.Rename(journal+".saved", journal); err != nil { + t.Fatal(err) + } + if err := f.a.Close(); err != nil { + t.Fatal(err) + } + a := authorityForTest(t, f.a.root, terminatedExecutor) + state, _ := a.State("v") + if state.InitialManifest != nil || state.Phase != PhaseRevoking { + t.Fatal("restart adopted orphaned final directory") + } +} + +func publishingGate(t *testing.T, f *publicationFixture) (string, *Server) { + t.Helper() + if err := f.a.Close(); err != nil { + t.Fatal(err) + } + s, err := New(Config{Listen: "127.0.0.1:0", Root: f.root, Token: testToken, TLSDir: filepath.Join(f.root, "tls"), Fenced: true, AuthorityDir: f.a.root, DockerSocket: f.engine.socket, Launch: &f.l.policy, Management: managementConfigForTest()}) + if err != nil { + t.Fatal(err) + } + s.authorizeCommand = allowNodeCommandForTest + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.ServeListener(ctx, tls.NewListener(ln, s.TLSConfig())) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + return ln.Addr().String(), s +} + +func TestInitialPublishKSG2LaunchAndForwardUseNodeTokenAndDenyManagementHTTP(t *testing.T) { + f := publishingFixture(t) + addr, s := publishingGate(t, f) + c, reader := gate2Connect(t, addr, testToken, gate2Request{Op: "publish-initial", VMID: "v", Management: &f.intent, ContentLength: len(f.payload)}, string(f.payload)) + var publication struct { + OK bool `json:"ok"` + Publication PublicationReceipt `json:"publication"` + Error string `json:"error"` + } + if err := json.NewDecoder(reader).Decode(&publication); err != nil { + t.Fatal(err) + } + c.Close() + if !publication.OK || publication.Publication.Intent != f.intent { + t.Fatalf("publish wire: %+v", publication) + } + state, _ := s.authority.State("v") + launch := launchIntentForTest(state, "launch-wire") + if reply := gate2Control(t, addr, gate2Request{Op: "launch", VMID: "v", Management: &launch}); !reply.OK { + t.Fatalf("launch wire: %+v", reply) + } + state, _ = s.authority.State("v") + expectedToken, err := readPublishedInitial(f.root, "v", state.Fence, f.l.policy, *state.InitialManifest, false) + if err != nil { + t.Fatal(err) + } + var mu sync.Mutex + calls := 0 + socket := filepath.Join(f.private(), "run", "kernel.sock") + unixHTTP(t, socket, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + calls++ + mu.Unlock() + if r.Header.Get("x-kin-internal-token") != expectedToken || r.Header.Get("x-internal-token") != expectedToken { + t.Error("controller overwrote node authentication") + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, normalClaudeJSON) + })) + request := forwardRequest(t, addr, state, "inference-wire") + inferenceRequest := request + body := `{"request_id":"inference-wire","body":{}}` + c, reader = gate2Connect(t, addr, testToken, request, fmt.Sprintf("POST /internal/v1/messages HTTP/1.1\r\nHost: kernel\r\nx-kin-internal-token: attacker\r\nx-internal-token: attacker\r\nContent-Length: %d\r\n\r\n%s", len(body), body)) + if line, _ := reader.ReadString('\n'); line != "OK\n" { + t.Fatalf("forward: %q", line) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, response.Body) + response.Body.Close() + c.Close() + request = forwardRequest(t, addr, state, "forbidden-import") + c, reader = gate2Connect(t, addr, testToken, request, "POST /internal/import HTTP/1.1\r\nHost: kernel\r\nContent-Length: 2\r\n\r\n{}") + if line, _ := reader.ReadString('\n'); line != "ERR management_denied\n" { + t.Fatalf("management HTTP forwarded: %q", line) + } + c.Close() + mu.Lock() + defer mu.Unlock() + if calls != 1 { + t.Fatalf("unexpected kernel calls: %d", calls) + } + stored, _ := s.authority.State("v") + deadline := time.Now().Add(time.Second) + for stored.Commands["inference-wire"].Status != CommandCompleted && time.Now().Before(deadline) { + inferenceRequest.Op = "command" + connection, reply := gate2Connect(t, addr, testToken, inferenceRequest, "") + var status struct { + OK bool `json:"ok"` + Command struct { + Status CommandStatus `json:"status"` + } `json:"command"` + } + if err := json.NewDecoder(reply).Decode(&status); err != nil { + t.Fatal(err) + } + connection.Close() + if !status.OK { + t.Fatal("original command status rejected") + } + stored, _ = s.authority.State("v") + } + if stored.Commands["inference-wire"].Status != CommandCompleted { + t.Fatalf("normal inference did not persist completion: %+v", stored.Commands["inference-wire"]) + } +} + +func TestInitialPublishLaunchPinsManifestAndIndependentAction(t *testing.T) { + f := publishingFixture(t) + f.publish(t) + state, _ := f.a.State("v") + intent := launchIntentForTest(state, "launch-grant") + bad := intent + bad.ManifestDigest = strings.Repeat("a", 64) + if _, err := f.l.LaunchManifest(context.Background(), bad, allowNodeCommandForTest); err == nil { + t.Fatal("wrong manifest launched") + } + if _, err := f.l.LaunchManifest(context.Background(), intent, nil); !errors.Is(err, ErrManagementDenied) { + t.Fatalf("missing independent launch authorizer: %v", err) + } + receipt, err := f.l.LaunchManifest(context.Background(), intent, allowNodeCommandForTest) + if err != nil { + t.Fatal(err) + } + if receipt.Intent != intent || receipt.IntentDigest != intent.digest() || receipt.Fence.Executor.ContainerID != f.engine.container || receipt.Fence.Generation != intent.Fence.Generation || receipt.ManifestDigest != intent.ManifestDigest { + t.Fatalf("launch receipt: %+v", receipt) + } + if _, err = f.l.LaunchManifest(context.Background(), intent, allowNodeCommandForTest); err == nil { + t.Fatal("completed launch was reissued") + } +} + +func TestInitialPublishUploadDoesNotLockOtherVMsAndDuplicateCannotUpload(t *testing.T) { + f := publishingFixture(t) + other := launchForTest(t, f.a, "other", testExecutor("d")) + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + started := make(chan struct{}) + var once sync.Once + f.p.authorize = func(_ context.Context, _ ManagementIntent, dispatch func() error) error { + err := dispatch() + once.Do(func() { close(started) }) + return err + } + done := make(chan error, 1) + go func() { _, err := f.p.Publish(context.Background(), f.intent, len(f.payload), reader); done <- err }() + <-started + admitted := make(chan error, 1) + go func() { + _, err := f.a.BeginCommand("other", other.Fence, "parallel-during-upload", CommandForward) + admitted <- err + }() + select { + case err := <-admitted: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("upload held Authority lock") + } + if _, err := f.p.Publish(context.Background(), f.intent, len(f.payload), strings.NewReader(string(f.payload))); !errors.Is(err, ErrCommandExists) { + t.Fatalf("duplicate upload accepted: %v", err) + } + writer.CloseWithError(errors.New("synthetic upload interruption")) + if err := <-done; err == nil { + t.Fatal("interrupted upload completed") + } + if _, err := f.l.Launch(context.Background(), "v", f.intent.Fence, "launch-interrupted-upload"); err == nil { + t.Fatal("interrupted upload launched") + } +} + +func TestInitialPublishRejectsStaleFenceRevisionAndOversizedInputBeforeStaging(t *testing.T) { + for _, kind := range []string{"fence", "revision", "size", "truncated", "digest"} { + t.Run(kind, func(t *testing.T) { + f := publishingFixture(t) + intent := f.intent + length := len(f.payload) + source := string(f.payload) + switch kind { + case "fence": + intent.Fence.Incarnation = strings.Repeat("d", 32) + case "revision": + intent.ExpectedManifestRevision = "1" + case "size": + length = initialPublishMax + 1 + case "truncated": + source = source[:len(source)-1] + case "digest": + intent.PayloadDigest = strings.Repeat("a", 64) + } + if _, err := f.p.Publish(context.Background(), intent, length, strings.NewReader(source)); err == nil { + t.Fatal("invalid publication accepted") + } + if _, err := os.Lstat(f.private()); !os.IsNotExist(err) { + t.Fatal("invalid publication changed final path") + } + }) + } +} + +func TestInitialPublishCompletedManifestAndLaunchReceiptSurviveNodeRestart(t *testing.T) { + f := publishingFixture(t) + receipt := f.publish(t) + state, _ := f.a.State("v") + intent := launchIntentForTest(state, "persisted-launch") + launched, err := f.l.LaunchManifest(context.Background(), intent, allowNodeCommandForTest) + if err != nil { + t.Fatal(err) + } + if err = f.a.Close(); err != nil { + t.Fatal(err) + } + a := authorityForTest(t, f.a.root, terminatedExecutor) + state, err = a.State("v") + if err != nil { + t.Fatal(err) + } + if state.InitialManifest == nil || state.InitialManifest.Receipt != receipt || state.Phase != PhaseActive || state.Commands[intent.CommandID].LaunchReceipt == nil || *state.Commands[intent.CommandID].LaunchReceipt != launched { + t.Fatalf("durable publication/launch lost: %+v", state) + } +} diff --git a/worker/internal/slotgate/slotgate.go b/worker/internal/slotgate/slotgate.go index c7c510cc2..f686e867b 100644 --- a/worker/internal/slotgate/slotgate.go +++ b/worker/internal/slotgate/slotgate.go @@ -54,14 +54,17 @@ var ( ) type Server struct { - cfg Config - token [sha256.Size]byte - tls *tls.Config - slots chan struct{} - wg sync.WaitGroup - authority *Authority - launcher *dockerLauncher - allocator *allocationManager + cfg Config + token [sha256.Size]byte + tls *tls.Config + slots chan struct{} + wg sync.WaitGroup + authority *Authority + launcher *dockerLauncher + allocator *allocationManager + publisher *initialPublisher + authorizeCommand CommandAuthorizer + managementNodeID string // Internal injection for trusted adapters/tests. Config alone never grants // unimplemented network mutations to a holder of the forwarding token. authorizeManagement func(context.Context, gate2Request, func() error) error @@ -124,6 +127,7 @@ func New(cfg Config) (*Server, error) { srv.authority.Close() return nil, err } + srv.publisher = &initialPublisher{authority: srv.authority, root: cfg.Root, policy: *cfg.Launch} } if cfg.Management != nil { authorizer, authErr := newPGAllocationAuthorizer(*cfg.Management) @@ -132,6 +136,8 @@ func New(cfg Config) (*Server, error) { return nil, authErr } srv.allocator = &allocationManager{authority: srv.authority, nodeID: cfg.Management.NodeID, authorize: authorizer.Authorize} + srv.managementNodeID = cfg.Management.NodeID + srv.authorizeCommand = authorizer.AuthorizeCommand } } return srv, nil diff --git a/worker/internal/slotgate/slotgate_test.go b/worker/internal/slotgate/slotgate_test.go index 54a6ad924..800ae150c 100644 --- a/worker/internal/slotgate/slotgate_test.go +++ b/worker/internal/slotgate/slotgate_test.go @@ -21,7 +21,11 @@ const testToken = "0123456789abcdef0123456789abcdef" // Unix socket paths are capped near 104 bytes, so the tree lives in a short /tmp dir. func shortRoot(t *testing.T) string { t.Helper() - dir, err := os.MkdirTemp("/tmp", "ksg") + base := os.Getenv("VM2API_SLOTGATE_TEST_ROOT") + if base == "" { + base = "/tmp" + } + dir, err := os.MkdirTemp(base, "ksg") if err != nil { t.Fatal(err) } From 31aeaa0e65527dc64ace32129c44116194d93071 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:42:31 +0800 Subject: [PATCH 364/473] test: require fixed HA node trust and deny legacy management routes --- test/unit/ha-control-plane-policy.test.mjs | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 test/unit/ha-control-plane-policy.test.mjs diff --git a/test/unit/ha-control-plane-policy.test.mjs b/test/unit/ha-control-plane-policy.test.mjs new file mode 100644 index 000000000..bd4ee6722 --- /dev/null +++ b/test/unit/ha-control-plane-policy.test.mjs @@ -0,0 +1,50 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +test('HA accepts fixed bootstrap TLS policy and refuses missing or mutable VM-derived trust', async (t) => { + const { loadHaNodeTrust } = await import('../../src/lib/core/ha-control-plane-policy.mjs') + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ha-trust-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const file = path.join(root, 'nodes.json') + const nodes = { 'ha-node': { + address: '127.0.0.1', port: 19999, token: 'synthetic-transport-token', + certSha256: 'a'.repeat(64), presetId: 'claude-docker-v1', policyDigest: 'b'.repeat(64), + } } + fs.writeFileSync(file, JSON.stringify({ nodes }), { mode: 0o600 }) + const fixed = loadHaNodeTrust(file) + assert.equal(fixed['ha-node'].certSha256, 'a'.repeat(64)) + assert.ok(Object.isFrozen(fixed['ha-node'])) + nodes['ha-node'].token = 'changed' + fs.writeFileSync(file, JSON.stringify({ nodes })) + assert.equal(fixed['ha-node'].token, 'synthetic-transport-token') + assert.throws(() => loadHaNodeTrust(), { code: 'ha_node_trust_unavailable' }) + fs.writeFileSync(file, JSON.stringify({ nodes: { 'ha-node': { ...nodes['ha-node'], presetId: 'native' } } })) + assert.throws(() => loadHaNodeTrust(file), { code: 'ha_node_trust_unavailable' }) +}) + +test('HA permits normal inference and PG metadata while refusing legacy management before dispatch', async () => { + const { haRouteAllowed, assertHaRoutingPatch } = await import('../../src/lib/core/ha-control-plane-policy.mjs') + for (const p of ['/v1/messages', '/messages', '/v1/chat/completions', '/v1/responses']) { + assert.equal(haRouteAllowed('POST', p), true, p) + } + assert.equal(haRouteAllowed('GET', '/v1/responses', { upgrade: true }), true) + assert.equal(haRouteAllowed('GET', '/api/panel/pool/stream'), true) + assert.equal(haRouteAllowed('PUT', '/api/panel/routing'), true) + assert.equal(haRouteAllowed('POST', '/api/panel/login'), true) + for (const [method, p] of [ + ['POST', '/api/panel/vms/create'], ['POST', '/api/panel/vms/import'], + ['POST', '/admin/vms/probe-all'], ['POST', '/api/panel/proxies'], + ['GET', '/api/panel/vms/create-options'], ['GET', '/api/panel/oauth'], + ['POST', '/api/panel/update'], ['POST', '/api/panel/backups'], + ['POST', '/v1/alpha/search'], ['POST', '/v1/messages/count_tokens'], + ['GET', '/api/panel/cluster/nodes/ha-node/shell'], + ]) assert.equal(haRouteAllowed(method, p), false, p) + assert.equal(haRouteAllowed('GET', '/api/panel/vms/x/shell', { upgrade: true }), false) + assert.doesNotThrow(() => assertHaRoutingPatch({ pool: { queue_max: 4 }, concurrency: { default_max_per_account: 1 } })) + for (const patch of [{ telemetry: {} }, { vm: {} }, { inference: {} }, { official_cc: {} }, { compatibility: {} }]) { + assert.throws(() => assertHaRoutingPatch(patch), { code: 'ha_operation_unsupported' }) + } +}) From f4b730d985f76967fa46894af3eaebc0abd3e47a Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:43:28 +0800 Subject: [PATCH 365/473] feat: pin HA node trust and restrict runtime routes to supported operations --- src/lib/core/ha-control-plane-policy.mjs | 120 +++++++++++++++++++++ test/unit/ha-control-plane-policy.test.mjs | 36 +++++-- 2 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 src/lib/core/ha-control-plane-policy.mjs diff --git a/src/lib/core/ha-control-plane-policy.mjs b/src/lib/core/ha-control-plane-policy.mjs new file mode 100644 index 000000000..98276c259 --- /dev/null +++ b/src/lib/core/ha-control-plane-policy.mjs @@ -0,0 +1,120 @@ +import fs from 'node:fs' +import path from 'node:path' +import net from 'node:net' + +export function haUnsupported() { + return Object.assign(new Error('该操作暂不支持多副本部署'), { code: 'ha_operation_unsupported', status: 503 }) +} + +/** Read the bootstrap-mounted trust once; VM records cannot replace it. */ +export function loadHaNodeTrust(file) { + try { + if (typeof file !== 'string' || !path.isAbsolute(file)) throw new Error() + const input = JSON.parse(fs.readFileSync(file, 'utf8')) + if (!input || Object.keys(input).length !== 1 || !input.nodes || Array.isArray(input.nodes)) throw new Error() + const nodes = Object.create(null) + for (const [id, value] of Object.entries(input.nodes)) { + if ( + !/^[A-Za-z0-9_-]{1,128}$/.test(id) || + !value || + Array.isArray(value) || + Object.keys(value).sort().join(',') !== 'address,certSha256,policyDigest,port,presetId,token' || + !net.isIP(value.address) || + !Number.isInteger(value.port) || + value.port < 1 || + value.port > 65535 || + typeof value.token !== 'string' || + !/^[\x21-\x7e]{1,256}$/.test(value.token) || + !/^[a-f0-9]{64}$/.test(value.certSha256) || + !/^[a-f0-9]{64}$/.test(value.policyDigest) || + value.presetId !== 'claude-docker-v1' + ) + throw new Error() + nodes[id] = Object.freeze({ ...value }) + } + if (!Object.keys(nodes).length) throw new Error() + return Object.freeze(nodes) + } catch { + // A malformed trust file can contain transport credentials. Never echo it. + throw Object.assign(new Error('HA node trust is unavailable'), { code: 'ha_node_trust_unavailable', status: 503 }) + } +} + +const inference = new Set([ + '/v1/messages', + '/messages', + '/v1/chat/completions', + '/chat/completions', + '/v1/completions', + '/completions', + '/v1/responses', + '/responses', +]) +const panelReads = new Set([ + 'me', + 'users', + 'api-keys', + 'vm-pools', + 'database/metrics', + 'version', + 'changelog', + 'dashboard', + 'vms', + 'pool/stream', + 'models', + 'model-policy', + 'pricing', + 'routing', + 'request-logs', + 'usage-logs', + 'statistics', + 'billing', +]) + +/** Only the supported HA operations enter a legacy HTTP/WS handler. */ +export function haRouteAllowed(method, pathname, { upgrade = false } = {}) { + if (upgrade) return method === 'GET' && ['/v1/responses', '/responses'].includes(pathname) + if (inference.has(pathname)) return ['POST', 'GET'].includes(method) + if (pathname === '/v1/models') return method === 'GET' + if (['/healthz', '/readyz', '/health'].includes(pathname)) return method === 'GET' || method === 'HEAD' + if (pathname.startsWith('/api/panel/')) { + const suffix = pathname.slice('/api/panel/'.length) + if (method === 'GET') { + if (panelReads.has(suffix)) return true + return /^(?:request-logs|usage-logs|statistics|api-keys|users)\/[^/]+$/.test(suffix) + } + if (method === 'POST' && ['login', 'logout', 'api-keys'].includes(suffix)) return true + if (['PUT', 'DELETE'].includes(method) && /^api-keys\/[^/]+$/.test(suffix)) return true + return method === 'PUT' && ['routing', 'model-policy', 'pricing/overrides'].includes(suffix) + } + if (pathname.startsWith('/admin')) { + return ( + method === 'GET' && ['/admin/routing', '/admin/quota', '/admin/vms', '/admin/usage/summary'].includes(pathname) + ) + } + // Static assets and unknown public paths retain normal GET/HEAD handling. + return !pathname.startsWith('/api/') && (method === 'GET' || method === 'HEAD') +} + +/** Kernel, credentials, payload and egress remain node-owned immutable assets. */ +export function assertHaRoutingPatch(patch) { + const allowed = new Set([ + 'sticky', + 'quota', + 'concurrency', + 'pool', + 'failover', + 'websocket', + 'logging', + 'tiers', + 'rate_limit', + ]) + if ( + !patch || + typeof patch !== 'object' || + Array.isArray(patch) || + Object.keys(patch).some((key) => !allowed.has(key)) + ) { + throw haUnsupported() + } +} diff --git a/test/unit/ha-control-plane-policy.test.mjs b/test/unit/ha-control-plane-policy.test.mjs index bd4ee6722..120a53098 100644 --- a/test/unit/ha-control-plane-policy.test.mjs +++ b/test/unit/ha-control-plane-policy.test.mjs @@ -9,10 +9,16 @@ test('HA accepts fixed bootstrap TLS policy and refuses missing or mutable VM-de const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ha-trust-')) t.after(() => fs.rmSync(root, { recursive: true, force: true })) const file = path.join(root, 'nodes.json') - const nodes = { 'ha-node': { - address: '127.0.0.1', port: 19999, token: 'synthetic-transport-token', - certSha256: 'a'.repeat(64), presetId: 'claude-docker-v1', policyDigest: 'b'.repeat(64), - } } + const nodes = { + 'ha-node': { + address: '127.0.0.1', + port: 19999, + token: 'synthetic-transport-token', + certSha256: 'a'.repeat(64), + presetId: 'claude-docker-v1', + policyDigest: 'b'.repeat(64), + }, + } fs.writeFileSync(file, JSON.stringify({ nodes }), { mode: 0o600 }) const fixed = loadHaNodeTrust(file) assert.equal(fixed['ha-node'].certSha256, 'a'.repeat(64)) @@ -35,15 +41,23 @@ test('HA permits normal inference and PG metadata while refusing legacy manageme assert.equal(haRouteAllowed('PUT', '/api/panel/routing'), true) assert.equal(haRouteAllowed('POST', '/api/panel/login'), true) for (const [method, p] of [ - ['POST', '/api/panel/vms/create'], ['POST', '/api/panel/vms/import'], - ['POST', '/admin/vms/probe-all'], ['POST', '/api/panel/proxies'], - ['GET', '/api/panel/vms/create-options'], ['GET', '/api/panel/oauth'], - ['POST', '/api/panel/update'], ['POST', '/api/panel/backups'], - ['POST', '/v1/alpha/search'], ['POST', '/v1/messages/count_tokens'], + ['POST', '/api/panel/vms/create'], + ['POST', '/api/panel/vms/import'], + ['POST', '/admin/vms/probe-all'], + ['POST', '/api/panel/proxies'], + ['GET', '/api/panel/vms/create-options'], + ['GET', '/api/panel/oauth'], + ['POST', '/api/panel/update'], + ['POST', '/api/panel/backups'], + ['POST', '/v1/alpha/search'], + ['POST', '/v1/messages/count_tokens'], ['GET', '/api/panel/cluster/nodes/ha-node/shell'], - ]) assert.equal(haRouteAllowed(method, p), false, p) + ]) + assert.equal(haRouteAllowed(method, p), false, p) assert.equal(haRouteAllowed('GET', '/api/panel/vms/x/shell', { upgrade: true }), false) - assert.doesNotThrow(() => assertHaRoutingPatch({ pool: { queue_max: 4 }, concurrency: { default_max_per_account: 1 } })) + assert.doesNotThrow(() => + assertHaRoutingPatch({ pool: { queue_max: 4 }, concurrency: { default_max_per_account: 1 } }), + ) for (const patch of [{ telemetry: {} }, { vm: {} }, { inference: {} }, { official_cc: {} }, { compatibility: {} }]) { assert.throws(() => assertHaRoutingPatch(patch), { code: 'ha_operation_unsupported' }) } From adc60eaf836d5bdb4217ccee14c743419403404f Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:44:53 +0800 Subject: [PATCH 366/473] test: reject legacy host mutation and Pod VM writes in HA --- test/unit/ha-host-boundary.test.mjs | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/unit/ha-host-boundary.test.mjs diff --git a/test/unit/ha-host-boundary.test.mjs b/test/unit/ha-host-boundary.test.mjs new file mode 100644 index 000000000..7f1d30065 --- /dev/null +++ b/test/unit/ha-host-boundary.test.mjs @@ -0,0 +1,45 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { slotHost } from '../../src/lib/vm/slot-host.mjs' +import { atomicWriteJson, writeJsonIfChanged, withVmLock } from '../../src/lib/vm/vm-file.mjs' + +test('HA legacy slot hosts refuse every mutating operation without invoking Docker or SSH', async (t) => { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + t.after(() => { + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) + for (const vm of [{ id: 'local-ha-vm' }, { id: 'remote-ha-vm', node_id: 'ha-node' }, { id: 'kvm-ha-vm', runtime_type: 'kvm' }]) { + const host = slotHost(vm) + assert.equal(host.supports('codex'), false) + for (const method of ['start', 'reload', 'stop', 'destroy', 'dockerApi', 'dockerEnv', 'syncRun', 'queueSyncRun', 'onImport', 'afterRefresh', 'touch', 'setProxyEgressEnabled']) { + await assert.rejects(async () => host[method](vm, '/unused'), { code: 'ha_operation_unsupported' }, method) + } + } +}) + +test('HA legacy VM writes and locks reject before filesystem or callback effects', async (t) => { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-ha-host-boundary-')) + t.after(() => { + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + fs.rmSync(root, { recursive: true, force: true }) + }) + const dir = path.join(root, 'vms') + fs.mkdirSync(dir) + const file = path.join(dir, 'ha-vm.json') + fs.writeFileSync(file, '{"id":"ha-vm"}') + assert.throws(() => atomicWriteJson(file, { id: 'changed' }), { code: 'ha_operation_unsupported' }) + assert.throws(() => writeJsonIfChanged(file, { id: 'changed' }), { code: 'ha_operation_unsupported' }) + let calls = 0 + await assert.rejects(async () => withVmLock(file, () => { calls++ }), { code: 'ha_operation_unsupported' }) + assert.equal(calls, 0) + assert.equal(fs.readFileSync(file, 'utf8'), '{"id":"ha-vm"}') + assert.deepEqual(fs.readdirSync(dir), ['ha-vm.json']) +}) From ec90dfd193ecbb40b32218665f7304b073dca12c Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:45:16 +0800 Subject: [PATCH 367/473] feat: prevent HA controllers from invoking legacy host mutations --- src/lib/vm/slot-host.mjs | 29 ++++++++++++++++++++++++++++- src/lib/vm/vm-file.mjs | 10 +++++++++- test/unit/ha-host-boundary.test.mjs | 29 ++++++++++++++++++++++++++--- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/lib/vm/slot-host.mjs b/src/lib/vm/slot-host.mjs index a27b631e9..2e4756dbc 100644 --- a/src/lib/vm/slot-host.mjs +++ b/src/lib/vm/slot-host.mjs @@ -92,7 +92,8 @@ import { isCodexVm } from './vm-kind.mjs' import { nodeEgressProxyUrl } from '../cluster/node-egress-socks.mjs' import { isKvmRuntime } from './runtime-kind.mjs' import { stopCodexKernel, finishCodexRecycle } from '../transport/codex-kernel-supervisor.mjs' -import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { isKubernetesMode, isKubernetesHaMode } from '../core/control-plane-mode.mjs' +import { haUnsupported } from '../core/ha-control-plane-policy.mjs' const noop = async () => ({ skipped: true }) @@ -217,6 +218,32 @@ function nativeHost(nodeId, record) { } export function slotHost(vm) { + if (isKubernetesHaMode()) { + const deny = () => { + throw haUnsupported() + } + return { + kind: 'managed', + nodeId: vmNodeId(vm), + bakedKernel: true, + ownsSocketFiles: false, + supports: () => false, + dockerEnv: deny, + dockerApi: deny, + execUser: deny, + start: deny, + reload: deny, + stop: deny, + destroy: deny, + setProxyEgressEnabled: deny, + localExitProxyUrl: deny, + syncRun: deny, + queueSyncRun: deny, + onImport: deny, + afterRefresh: deny, + touch: deny, + } + } const nodeId = vmNodeId(vm) if (nodeId && isNativeNodeVm(vm)) return nativeHost(nodeId, vm) const base = nodeId ? nodeHost(nodeId) : LOCAL_HOST diff --git a/src/lib/vm/vm-file.mjs b/src/lib/vm/vm-file.mjs index 09e28b443..99a4d0134 100644 --- a/src/lib/vm/vm-file.mjs +++ b/src/lib/vm/vm-file.mjs @@ -12,7 +12,8 @@ import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' import { getDb } from '../db/database.mjs' -import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { isKubernetesMode, isKubernetesHaMode } from '../core/control-plane-mode.mjs' +import { haUnsupported } from '../core/ha-control-plane-policy.mjs' import { SharedVmOperations } from './shared-vm-operations.mjs' /** @@ -178,6 +179,7 @@ function notifyVmWrite(filePath) { } export function withVmLock(key, fn, { operations, vmId, kind = 'mutation' } = {}) { + if (isKubernetesHaMode()) throw haUnsupported() if (operations || isKubernetesMode()) { if (vmId == null) { const name = path.basename(String(key || '')) @@ -231,6 +233,7 @@ export function withVmLock(key, fn, { operations, vmId, kind = 'mutation' } = {} } export function atomicWriteJson(filePath, obj, { mode } = {}) { + assertVmWriteAllowed(filePath) const dir = path.dirname(filePath) const tmp = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${crypto.randomBytes(4).toString('hex')}.tmp`) const data = typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2) @@ -255,6 +258,7 @@ export function atomicWriteJson(filePath, obj, { mode } = {}) { } export function writeJsonIfChanged(filePath, obj, { mode } = {}) { + assertVmWriteAllowed(filePath) const data = typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2) try { if (fs.existsSync(filePath) && readFileCached(filePath) === data) { @@ -264,3 +268,7 @@ export function writeJsonIfChanged(filePath, obj, { mode } = {}) { atomicWriteJson(filePath, data, { mode }) return true } + +function assertVmWriteAllowed(filePath) { + if (isKubernetesHaMode() && path.resolve(filePath).split(path.sep).includes('vms')) throw haUnsupported() +} diff --git a/test/unit/ha-host-boundary.test.mjs b/test/unit/ha-host-boundary.test.mjs index 7f1d30065..f664b5bd5 100644 --- a/test/unit/ha-host-boundary.test.mjs +++ b/test/unit/ha-host-boundary.test.mjs @@ -13,10 +13,27 @@ test('HA legacy slot hosts refuse every mutating operation without invoking Dock if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = previous }) - for (const vm of [{ id: 'local-ha-vm' }, { id: 'remote-ha-vm', node_id: 'ha-node' }, { id: 'kvm-ha-vm', runtime_type: 'kvm' }]) { + for (const vm of [ + { id: 'local-ha-vm' }, + { id: 'remote-ha-vm', node_id: 'ha-node' }, + { id: 'kvm-ha-vm', runtime_type: 'kvm' }, + ]) { const host = slotHost(vm) assert.equal(host.supports('codex'), false) - for (const method of ['start', 'reload', 'stop', 'destroy', 'dockerApi', 'dockerEnv', 'syncRun', 'queueSyncRun', 'onImport', 'afterRefresh', 'touch', 'setProxyEgressEnabled']) { + for (const method of [ + 'start', + 'reload', + 'stop', + 'destroy', + 'dockerApi', + 'dockerEnv', + 'syncRun', + 'queueSyncRun', + 'onImport', + 'afterRefresh', + 'touch', + 'setProxyEgressEnabled', + ]) { await assert.rejects(async () => host[method](vm, '/unused'), { code: 'ha_operation_unsupported' }, method) } } @@ -38,7 +55,13 @@ test('HA legacy VM writes and locks reject before filesystem or callback effects assert.throws(() => atomicWriteJson(file, { id: 'changed' }), { code: 'ha_operation_unsupported' }) assert.throws(() => writeJsonIfChanged(file, { id: 'changed' }), { code: 'ha_operation_unsupported' }) let calls = 0 - await assert.rejects(async () => withVmLock(file, () => { calls++ }), { code: 'ha_operation_unsupported' }) + await assert.rejects( + async () => + withVmLock(file, () => { + calls++ + }), + { code: 'ha_operation_unsupported' }, + ) assert.equal(calls, 0) assert.equal(fs.readFileSync(file, 'utf8'), '{"id":"ha-vm"}') assert.deepEqual(fs.readdirSync(dir), ['ha-vm.json']) From 6de3c0f4b51d6329b81c47056e02a843d36fbb99 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:32:36 +0800 Subject: [PATCH 368/473] test: specify independent node publication and launch grants --- test/unit/node-command-grants.test.mjs | 330 +++++++++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 test/unit/node-command-grants.test.mjs diff --git a/test/unit/node-command-grants.test.mjs b/test/unit/node-command-grants.test.mjs new file mode 100644 index 000000000..601ec051c --- /dev/null +++ b/test/unit/node-command-grants.test.mjs @@ -0,0 +1,330 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import pg from 'pg' +import { setTimeout as delay } from 'node:timers/promises' +import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { nodeGrantNamespace, nodeGrantResource } from '../../src/lib/db/node-management-authority.mjs' + +const nodeId = 'synthetic-node' +const vmId = 'synthetic-vm' +const allocationId = 'synthetic-allocation' +const capturedFence = { generation: '1', incarnation: 'a'.repeat(32), executor: { kind: '', node_boot_id: '' } } +const payloadDigest = 'b'.repeat(64) +const policyDigest = 'c'.repeat(64) +const manifestDigest = 'd'.repeat(64) + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const maintenance = new GlobalMaintenanceCoordinator(db) + const service = new NodeManagementGrants(db) + const other = new NodeManagementGrants(peer) + const roles = [] + const readers = [] + t.after(async () => { + await maintenance.stop() + await peer.close() + for (const client of readers) await client.end() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + for (const role of roles) + await cleanup.query(`DROP OWNED BY ${quoteIdentifier(role)}; DROP ROLE ${quoteIdentifier(role)}`) + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE`) + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(maintenance.schema)} CASCADE`) + } finally { + await cleanup.end() + } + }) + await maintenance.start() + await maintenance.acknowledgeEpoch(1) + await service.initialize() + const run = (kind, callback) => new SharedVmOperations(db).run(vmId, callback, { kind }) + await run('node_allocate', async (operation) => { + const grant = await service.issueAllocation({ operation, nodeId, vmId, allocationId }) + const receipt = { + node_id: nodeId, + vm_id: vmId, + allocation_id: allocationId, + operation_id: grant.operationId, + owner_id: grant.ownerId, + epoch: grant.epoch, + intent_digest: grant.intentDigest, + fence: capturedFence, + } + // Synthetic node receipt: allocation transport/receipt authentication has its own real-node tests. + await db.query( + `UPDATE ${quoteIdentifier(service.schema)}.allocations SET receipt=$2::jsonb WHERE allocation_id=$1`, + [allocationId, JSON.stringify(receipt)], + ) + }) + const input = (operation, patch = {}) => ({ + operation, + nodeId, + vmId, + allocationId, + commandId: 'publish-one', + capturedFence, + expectedManifestRevision: '0', + revision: '1', + payloadDigest, + presetId: 'claude-docker-v1', + policyDigest, + ...patch, + }) + const publication = (grant, patch = {}) => ({ + intent: structuredClone(grant.intent), + intent_digest: grant.intentDigest, + manifest_revision: '1', + manifest_digest: manifestDigest, + ...patch, + }) + async function publish() { + let value + await run('node_publish_initial', async (operation) => { + const grant = await service.issueInitialPublish(input(operation)) + value = await service.recordInitialPublishReceipt(grant, publication(grant)) + }) + return value + } + async function reader() { + const role = 'command_reader_' + crypto.randomBytes(8).toString('hex') + const password = crypto.randomBytes(24).toString('hex') + await db.pool.query( + `CREATE ROLE ${quoteIdentifier(role)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, + ) + roles.push(role) + await db.pool.query(service.readerRoleSql({ role, nodeId })) + await db.pool.query(service.commandReaderRoleSql({ role })) + const url = new URL(db.connectionString) + url.username = role + url.password = password + const client = new pg.Client({ connectionString: url.toString() }) + await client.connect() + readers.push(client) + return client + } + return { + db, + peer, + maintenance, + service, + other, + run, + input, + publication, + publish, + reader, + q: quoteIdentifier(service.schema), + } +} + +test('new actions require explicit trusted feature initialization without changing allocation v1', async (t) => { + const fx = await fixture(t) + await fx.run('node_publish_initial', async (operation) => { + await assert.rejects(fx.service.issueInitialPublish(fx.input(operation)), { + code: 'node_command_authority_unavailable', + }) + }) + assert.equal( + (await fx.peer.query('SELECT to_regclass($1) AS relation', [`${fx.service.schema}.command_grants`])).rows[0] + .relation, + null, + ) + await fx.service.initializeCommands() + const meta = (await fx.peer.query(`SELECT protocol_version,command_protocol_version FROM ${fx.q}.meta`)).rows[0] + assert.deepEqual(meta, { protocol_version: 1, command_protocol_version: 1 }) + const marker = ( + await fx.peer.query( + `SELECT protocol_version,command_protocol_version FROM ${quoteIdentifier(fx.maintenance.schema)}.node_authority_features`, + ) + ).rows[0] + assert.deepEqual(marker, meta) + await fx.service.initialize() + assert.equal((await fx.peer.query(`SELECT count(*) AS n FROM ${fx.q}.allocations`)).rows[0].n, 1) +}) + +test('publish and launch bind the original allocation, independent operations, digests and immutable receipts', async (t) => { + const fx = await fixture(t) + await fx.service.initializeCommands() + const published = await fx.publish() + assert.equal(published.manifest_digest, manifestDigest) + let launched + await fx.run('node_launch', async (operation) => { + const grant = await fx.service.issueLaunch( + fx.input(operation, { + commandId: 'launch-one', + expectedManifestRevision: '1', + payloadDigest: '', + manifestDigest, + }), + ) + assert.equal(grant.intent.action, 'launch') + assert.equal(grant.intent.expected_manifest_revision, '1') + assert.equal(grant.intent.payload_digest, '') + const fence = { + generation: '1', + incarnation: capturedFence.incarnation, + executor: { + kind: 'container', + node_boot_id: 'synthetic-boot', + container_id: 'e'.repeat(64), + docker_daemon_id: 'synthetic-daemon', + started_at: '2026-10-22T01:02:03.123456789Z', + }, + } + launched = await fx.service.recordLaunchReceipt(grant, { ...fx.publication(grant), fence }) + }) + const rows = ( + await fx.peer.query(`SELECT action,operation_id,receipt,revoked_at FROM ${fx.q}.command_grants ORDER BY action`) + ).rows + assert.deepEqual( + rows.map((row) => row.action), + ['launch', 'publish-initial'], + ) + assert.notEqual(rows[0].operation_id, rows[1].operation_id) + assert.ok(rows.every((row) => row.revoked_at && row.receipt)) + assert.deepEqual(await fx.other.readLaunchReceipt({ nodeId, vmId }), launched) + assert.equal(launched.fence.incarnation, capturedFence.incarnation) + assert.equal(launched.fence.executor.container_id, 'e'.repeat(64)) +}) + +test('fresh fences, missing publication receipts, wrong revisions and absent exact operation scope are rejected', async (t) => { + const fx = await fixture(t) + await fx.service.initializeCommands() + await assert.rejects(fx.service.issueInitialPublish(fx.input({ operationId: 'forged', ownerId: 'forged' })), { + code: 'node_operation_scope_required', + }) + await fx.run('node_launch', async (operation) => { + await assert.rejects( + fx.service.issueLaunch( + fx.input(operation, { + commandId: 'launch-one', + expectedManifestRevision: '1', + payloadDigest: '', + manifestDigest, + }), + ), + { code: 'node_publication_required' }, + ) + }) + await fx.run('node_publish_initial', async (operation) => { + for (const patch of [ + { capturedFence: { ...capturedFence, incarnation: 'f'.repeat(32) } }, + { expectedManifestRevision: '1' }, + { revision: '01' }, + { payloadDigest: 'not-a-digest' }, + { presetId: 'arbitrary-preset' }, + ]) { + await assert.rejects(fx.service.issueInitialPublish(fx.input(operation, patch)), { code: 'node_command_invalid' }) + } + }) +}) + +test('a command cannot be reissued, relabeled or replaced after an unknown attempt', async (t) => { + const fx = await fixture(t) + await fx.service.initializeCommands() + await fx.run('node_publish_initial', async (operation) => { + const first = await fx.service.issueInitialPublish(fx.input(operation)) + for (const patch of [{}, { payloadDigest: 'f'.repeat(64) }, { commandId: 'new-command' }]) { + await assert.rejects(fx.service.issueInitialPublish(fx.input(operation, patch)), { + code: 'node_command_conflict', + }) + } + await operation.markUncertain() + await assert.rejects(fx.service.recordInitialPublishReceipt(first, fx.publication(first)), { + code: 'node_operation_invalid', + }) + return { vmOperationUncertain: true } + }) + const row = (await fx.peer.query(`SELECT revoked_at,receipt FROM ${fx.q}.command_grants`)).rows[0] + assert.ok(row.revoked_at) + assert.equal(row.receipt, null) + assert.equal((await new SharedVmOperations(fx.peer).tryAcquire({ vmId })).ok, false) +}) + +test('receipt validation rejects secret fields and mismatched intent without committing a partial receipt', async (t) => { + const fx = await fixture(t) + await fx.service.initializeCommands() + await fx.run('node_publish_initial', async (operation) => { + const grant = await fx.service.issueInitialPublish(fx.input(operation)) + for (const receipt of [ + fx.publication(grant, { access_token: 'synthetic-secret-must-not-persist' }), + fx.publication(grant, { intent_digest: 'f'.repeat(64) }), + fx.publication(grant, { intent: { ...grant.intent, owner_id: 'other-owner' } }), + fx.publication(grant, { manifest_revision: '2' }), + ]) { + await assert.rejects(fx.service.recordInitialPublishReceipt(grant, receipt), { + code: 'node_command_receipt_invalid', + }) + assert.equal((await fx.peer.query(`SELECT receipt FROM ${fx.q}.command_grants`)).rows[0].receipt, null) + } + const saved = await fx.service.recordInitialPublishReceipt(grant, fx.publication(grant)) + assert.deepEqual(await fx.service.recordInitialPublishReceipt(grant, fx.publication(grant)), saved) + await assert.rejects( + fx.service.recordInitialPublishReceipt(grant, fx.publication(grant, { manifest_digest: 'f'.repeat(64) })), + { code: 'node_command_conflict' }, + ) + }) +}) + +test('the node reader sees only live commands and release waits for its bounded authorization lock', async (t) => { + const fx = await fixture(t) + await fx.service.initializeCommands() + const reader = await fx.reader() + await assert.rejects(reader.query(`SELECT * FROM ${fx.q}.command_grants`), { code: '42501' }) + await fx.run('node_publish_initial', async (operation) => { + const grant = await fx.service.issueInitialPublish(fx.input(operation)) + const rows = (await reader.query(`SELECT action,epoch::text,intent_digest FROM ${fx.q}.authorized_commands`)).rows + assert.deepEqual(rows, [{ action: 'publish-initial', epoch: '1', intent_digest: grant.intentDigest }]) + await reader.query('BEGIN READ ONLY') + await reader.query('SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))', [ + nodeGrantNamespace(fx.service.schema), + nodeGrantResource(nodeId, vmId), + ]) + const releasing = operation.release() + try { + assert.equal(await Promise.race([releasing.then(() => 'released'), delay(75).then(() => 'waiting')]), 'waiting') + } finally { + await reader.query('COMMIT') + } + assert.equal((await releasing).released, true) + assert.equal((await reader.query(`SELECT * FROM ${fx.q}.authorized_commands`)).rowCount, 0) + }) +}) + +test('enabled command state corruption fails closed instead of performing runtime DDL or releasing ownership', async (t) => { + const fx = await fixture(t) + await fx.service.initializeCommands() + await fx.peer.query(`DROP TABLE ${fx.q}.command_grants CASCADE`) + await assert.rejects( + fx.run('node_publish_initial', (operation) => fx.service.issueInitialPublish(fx.input(operation))), + { code: 'node_command_authority_unavailable' }, + ) + assert.equal( + (await fx.peer.query('SELECT to_regclass($1) AS relation', [`${fx.service.schema}.command_grants`])).rows[0] + .relation, + null, + ) + const rows = (await fx.peer.query("SELECT released_at FROM vm_operations WHERE kind='node_publish_initial'")).rows + assert.equal(rows[0].released_at, null) +}) + +test('business epoch changes reject persisted old command receipts and deny the node reader', async (t) => { + const fx = await fixture(t) + await fx.service.initializeCommands() + const reader = await fx.reader() + await fx.run('node_publish_initial', async (operation) => { + const grant = await fx.service.issueInitialPublish(fx.input(operation)) + await fx.peer.query(`UPDATE ${quoteIdentifier(fx.maintenance.schema)}.control SET epoch=2 WHERE id=1`) + assert.equal((await reader.query(`SELECT * FROM ${fx.q}.authorized_commands`)).rowCount, 0) + await assert.rejects(fx.service.recordInitialPublishReceipt(grant, fx.publication(grant)), { + code: 'node_command_epoch_stale', + }) + }) + assert.equal((await fx.peer.query(`SELECT receipt FROM ${fx.q}.command_grants`)).rows[0].receipt, null) +}) From cac64070f546f45be3a8850fcfbfb20e475fd0a3 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:39:23 +0800 Subject: [PATCH 369/473] feat: persist independent node publication and launch authority --- .../2026-10-22-node-command-authority.md | 46 +++ src/lib/cluster/node-management-grants.mjs | 381 ++++++++++++++++++ src/lib/db/node-management-authority.mjs | 136 ++++++- test/unit/node-command-grants.test.mjs | 350 ++++++++-------- 4 files changed, 750 insertions(+), 163 deletions(-) create mode 100644 docs/plans/2026-10-22-node-command-authority.md diff --git a/docs/plans/2026-10-22-node-command-authority.md b/docs/plans/2026-10-22-node-command-authority.md new file mode 100644 index 000000000..a430a8e59 --- /dev/null +++ b/docs/plans/2026-10-22-node-command-authority.md @@ -0,0 +1,46 @@ +# 节点首次发布与启动授权 + +本增量仅增加独立 PostgreSQL 授权和回执保存。生产启动流程不会自动启用该特性，不改变现有控制面单活限制。 + +`NodeManagementGrants.initializeCommands()` 是可信初始化入口。它要求已有 allocation v1 私有 authority、维护状态为 OPEN，并在一个事务内新增 `command_grants`、只读视图和授权函数，同时将私有 `meta` 与 coordination feature marker 的 `command_protocol_version` 设为 1。allocation 的 `protocol_version` 保持 1。运行时不执行 DDL；缺少任一已启用对象或两处版本不一致时拒绝操作。 + +新对象仍位于由业务 schema 唯一派生的私有 authority schema。业务备份和恢复不包含它们。授权函数动态读取业务 `vm_operations`，不对可替换的业务表建立静态依赖。旧业务操作被恢复后，也必须匹配 coordination 中的当前 epoch。 + +## 动作与接口 + +- `issueInitialPublish(input)`：要求当前精确 `node_publish_initial` VM 操作。输入绑定 PG 中已保存的原始 VACANT allocation 回执、原始 fence、清单版本 0→1、完整受限凭据 payload 的 SHA-256、固定 preset 和节点 policy 摘要。 +- `issueLaunch(input)`：要求当前精确 `node_launch` VM 操作。输入绑定已保存的首次发布回执、清单版本 1 和 manifest 摘要，使用同一原始 VACANT fence。 +- 两个发行入口均返回 `{intent, intentDigest}`。`intent` 使用 Go 协议约定的 snake_case 字段。`epoch` 和 revision 为规范十进制字符串；首次 VACANT fence 的 generation 固定为可无损编码的数字 1。 +- `recordInitialPublishReceipt(grant, receipt)` / `recordLaunchReceipt(grant, receipt)`：在原精确操作仍有效时重新检查授权、epoch 和完整来源链，原子保存固定字段回执。相同回执可重复确认；不同回执拒绝覆盖。 +- `readLaunchReceipt({vmId, nodeId?})`：从 PG 确定节点并验证完整链，返回原 launch 回执。正常撤销授权不会删除已确认回执。它证明历史启动已被确认，不证明容器当前仍在运行，也不获取新 fence 替换原值。 + +每个 allocation 的每种新动作只允许一条命令。command ID、动作、操作、owner、epoch、fence、payload、preset 和 policy 不能通过重新发行修改。未知操作继续占用 VM；本增量不提供超时释放或自动恢复。 + +## 固定协议 + +intent 字段固定为 `action,node_id,vm_id,allocation_id,operation_id,owner_id,command_id,epoch,fence,expected_manifest_revision,manifest_revision,payload_digest,preset_id,policy_digest,manifest_digest`。 + +intent 摘要为以下字符串按 NUL 连接后的 SHA-256： + +```text +slotgate-command-v1 +action,node_id,vm_id,allocation_id,operation_id,owner_id,command_id,epoch +decimal(fence.generation),fence.incarnation +expected_manifest_revision,manifest_revision,payload_digest,preset_id,policy_digest,manifest_digest +``` + +以上换行仅用于排版；摘要输入的所有 17 项只以 NUL 分隔。VACANT executor 必须精确为 `{kind:'',node_boot_id:''}`，因此不另列入摘要。 + +publish 回执固定为 `{intent,intent_digest,manifest_revision,manifest_digest}`。launch 回执增加 `fence`，其中 executor 必须符合已有 `container` 身份规范。禁止额外字段；授权表和回执不保存 token、凭据字节或任意配置内容。`payload_digest` 只用于绑定上传输入，不可作为凭据描述符或登录状态使用。 + +## 授权与锁 + +发行、回执保存和撤销使用 `VM 操作行 → node/VM advisory 排他锁`。节点 reader 使用相同 namespace/resource 的短事务共享锁，通过 `authorized_commands` 查询精确动作、身份和摘要。`command_current` 还检查当前 epoch、OPEN 阶段、有效原 VM 操作和双 feature marker。 + +共享授权窗口只覆盖节点本地 journal 或最终 rename。上传和 Docker I/O 位于授权事务之外。节点必须在发布前和提交时分别核对同一授权；未知结果不能通过后续命令或健康观察覆盖。 + +现有 reader 安装 SQL保持不变。新增 `commandReaderRoleSql({role})` 仅授权新只读视图和函数，不授予基表写权限；它要求可信配置已安装该 role 的节点关联。 + +## 验证范围 + +定向测试使用两个独立 PG 连接、合成 allocation/publish/launch 回执及专用数据库临时 reader role，覆盖显式启用、精确操作、回执不可变、未知保留、共享授权锁与 epoch 拒绝。真实 Go 发布、启动与 TLS 路径由独立 worker 增量负责集成验证。 diff --git a/src/lib/cluster/node-management-grants.mjs b/src/lib/cluster/node-management-grants.mjs index a3217c5e5..b73b8e272 100644 --- a/src/lib/cluster/node-management-grants.mjs +++ b/src/lib/cluster/node-management-grants.mjs @@ -6,12 +6,17 @@ import { physicalContainerName } from '../vm/vm-physical-target.mjs' import { ClusterNodesRepo } from '../db/repos/cluster-nodes-repo.mjs' import { allocateNodeSlot, normalizeAllocationReceipt } from '../transport/slotgate-management-client.mjs' import { isDeepStrictEqual } from 'node:util' +import { normalizeNodeFence } from '../core/node-execution-proof.mjs' import { initializeNodeAuthority, nodeAuthorityState, nodeAuthorityError, nodeAuthoritySchemaFor, lockNodeGrant, + nodeGrantNamespace, + nodeGrantResource, + nodeCommandAuthorityState, + initializeNodeCommandAuthority, } from '../db/node-management-authority.mjs' export { nodeAuthoritySchemaFor } @@ -25,6 +30,156 @@ function literal(value) { return "'" + value.replaceAll("'", "''") + "'" } +const COMMAND_KEYS = [ + 'action', + 'node_id', + 'vm_id', + 'allocation_id', + 'operation_id', + 'owner_id', + 'command_id', + 'epoch', + 'fence', + 'expected_manifest_revision', + 'manifest_revision', + 'payload_digest', + 'preset_id', + 'policy_digest', + 'manifest_digest', +] +const digestPattern = /^[a-f0-9]{64}$/ +function isDigest(value) { + return typeof value === 'string' && digestPattern.test(value) +} +function commandFailure(code = 'node_command_invalid') { + return nodeAuthorityError(code) +} +function exactShape(value, keys) { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + ![Object.prototype, null].includes(Object.getPrototypeOf(value)) || + Object.keys(value).length !== keys.length || + keys.some((key) => !Object.hasOwn(value, key)) + ) + throw commandFailure() +} +function vacantFence(value) { + exactShape(value, ['generation', 'incarnation', 'executor']) + exactShape(value.executor, ['kind', 'node_boot_id']) + if ( + (value.generation !== '1' && value.generation !== 1) || + typeof value.incarnation !== 'string' || + !/^[a-f0-9]{32}$/.test(value.incarnation) || + value.executor.kind !== '' || + value.executor.node_boot_id !== '' + ) + throw commandFailure() + // Initial allocation generation 1 is exactly representable in the existing Go JSON number wire format. + return { generation: 1, incarnation: value.incarnation, executor: { kind: '', node_boot_id: '' } } +} +function commandIntent(value) { + exactShape(value, COMMAND_KEYS) + for (const key of ['node_id', 'vm_id', 'allocation_id', 'operation_id', 'owner_id', 'command_id']) + identifier(value[key]) + if ( + typeof value.epoch !== 'string' || + !/^[1-9][0-9]{0,18}$/.test(value.epoch) || + BigInt(value.epoch) > 9223372036854775807n || + value.manifest_revision !== '1' || + value.preset_id !== 'claude-docker-v1' || + !isDigest(value.policy_digest) + ) + throw commandFailure() + if (value.action === 'publish-initial') { + if (value.expected_manifest_revision !== '0' || !isDigest(value.payload_digest) || value.manifest_digest !== '') + throw commandFailure() + } else if (value.action === 'launch') { + if (value.expected_manifest_revision !== '1' || value.payload_digest !== '' || !isDigest(value.manifest_digest)) + throw commandFailure() + } else throw commandFailure() + return { ...value, fence: vacantFence(value.fence) } +} +function commandDigest(intent) { + return crypto + .createHash('sha256') + .update( + [ + 'slotgate-command-v1', + intent.action, + intent.node_id, + intent.vm_id, + intent.allocation_id, + intent.operation_id, + intent.owner_id, + intent.command_id, + intent.epoch, + String(intent.fence.generation), + intent.fence.incarnation, + intent.expected_manifest_revision, + intent.manifest_revision, + intent.payload_digest, + intent.preset_id, + intent.policy_digest, + intent.manifest_digest, + ].join('\0'), + ) + .digest('hex') +} +function commandGrant(value) { + exactShape(value, ['intent', 'intentDigest']) + const intent = commandIntent(value.intent) + if (value.intentDigest !== commandDigest(intent)) throw commandFailure() + return { intent, intentDigest: value.intentDigest } +} +function storedGrant(row) { + if (!row) throw commandFailure('node_command_conflict') + const grant = commandGrant({ intent: row.intent, intentDigest: row.intent_digest }) + const fields = { + command_id: row.command_id, + node_id: row.node_id, + vm_id: row.vm_id, + allocation_id: row.allocation_id, + operation_id: row.operation_id, + owner_id: row.owner_id, + action: row.action, + epoch: String(row.epoch), + } + if (Object.entries(fields).some(([key, value]) => grant.intent[key] !== value)) + throw commandFailure('node_command_conflict') + return grant +} +function commandReceipt(grant, value) { + try { + const launch = grant.intent.action === 'launch' + exactShape(value, ['intent', 'intent_digest', 'manifest_revision', 'manifest_digest', ...(launch ? ['fence'] : [])]) + const intent = commandIntent(value.intent) + if ( + !isDeepStrictEqual(intent, grant.intent) || + value.intent_digest !== grant.intentDigest || + value.manifest_revision !== '1' || + !isDigest(value.manifest_digest) || + (launch && value.manifest_digest !== intent.manifest_digest) + ) + throw commandFailure() + const result = { + intent, + intent_digest: value.intent_digest, + manifest_revision: '1', + manifest_digest: value.manifest_digest, + } + if (launch) { + result.fence = normalizeNodeFence(value.fence) + if (result.fence.generation !== '1' || result.fence.incarnation !== intent.fence.incarnation) + throw commandFailure() + } + return result + } catch { + throw commandFailure('node_command_receipt_invalid') + } +} + export class NodeManagementGrants { constructor(db, { resolveNodeLink } = {}) { this.db = db @@ -35,6 +190,232 @@ export class NodeManagementGrants { return initializeNodeAuthority(this.db) } + initializeCommands() { + return initializeNodeCommandAuthority(this.db) + } + + async _commandEpoch() { + const row = ( + await this.db.query( + `SELECT epoch::text,phase FROM ${quoteIdentifier(coordinationSchemaFor(this.db.schema))}.control WHERE id=1 FOR SHARE`, + ) + ).rows[0] + if (row?.phase !== 'open') throw commandFailure('node_command_epoch_stale') + return row.epoch + } + + async _commandContext(intent) { + if (!currentVmOperationIds(this.db).has(intent.operation_id)) throw commandFailure('node_operation_scope_required') + const operation = await this.db + .prepare('SELECT * FROM vm_operations WHERE id=$1 AND owner_id=$2 FOR UPDATE') + .get(intent.operation_id, intent.owner_id) + const kind = intent.action === 'publish-initial' ? 'node_publish_initial' : 'node_launch' + if ( + !operation || + operation.vm_id !== intent.vm_id || + operation.kind !== kind || + operation.released_at || + operation.uncertain_at + ) + throw commandFailure('node_operation_invalid') + const state = await nodeCommandAuthorityState(this.db) + if (!state.enabled) throw commandFailure('node_command_authority_unavailable') + await lockNodeGrant(this.db, this.schema, intent.node_id, intent.vm_id) + const epoch = await this._commandEpoch() + return { state, epoch } + } + + async _commandAllocation(state, intent) { + const row = ( + await this.db.query(`SELECT * FROM ${state.q}.allocations WHERE allocation_id=$1`, [intent.allocation_id]) + ).rows[0] + if (!row?.receipt || row.node_id !== intent.node_id || row.vm_id !== intent.vm_id) + throw commandFailure('node_command_invalid') + if (String(row.epoch) !== intent.epoch) throw commandFailure('node_command_epoch_stale') + let receipt + try { + receipt = normalizeAllocationReceipt(row.receipt, { + nodeId: row.node_id, + vmId: row.vm_id, + allocationId: row.allocation_id, + operationId: row.operation_id, + ownerId: row.owner_id, + epoch: String(row.epoch), + intentDigest: row.intent_digest, + }) + } catch { + throw commandFailure('node_command_invalid') + } + if (!isDeepStrictEqual(vacantFence(receipt.fence), intent.fence)) throw commandFailure() + return receipt + } + + async _commandPublication(state, intent) { + const row = ( + await this.db.query( + `SELECT * FROM ${state.q}.command_grants WHERE allocation_id=$1 AND action='publish-initial'`, + [intent.allocation_id], + ) + ).rows[0] + if (!row?.receipt) throw commandFailure('node_publication_required') + const grant = storedGrant(row) + const receipt = commandReceipt(grant, row.receipt) + if ( + ['node_id', 'vm_id', 'allocation_id', 'epoch', 'preset_id', 'policy_digest'].some( + (key) => grant.intent[key] !== intent[key], + ) || + !isDeepStrictEqual(grant.intent.fence, intent.fence) || + receipt.manifest_digest !== intent.manifest_digest + ) + throw commandFailure('node_command_conflict') + return receipt + } + + async _issueCommand(action, input) { + const operation = input?.operation + if (!operation || !currentVmOperationIds(this.db).has(operation.operationId)) + throw commandFailure('node_operation_scope_required') + if (this.db.isTransaction) throw nodeAuthorityError('node_grant_transaction_active') + try { + for (const value of [ + input.nodeId, + input.vmId, + input.allocationId, + input.commandId, + operation.operationId, + operation.ownerId, + ]) + identifier(value) + } catch { + throw commandFailure() + } + return withTransaction(this.db, async () => { + const identity = { + action, + node_id: input.nodeId, + vm_id: input.vmId, + allocation_id: input.allocationId, + operation_id: operation.operationId, + owner_id: operation.ownerId, + command_id: input.commandId, + } + const { state, epoch } = await this._commandContext(identity) + const intent = commandIntent({ + ...identity, + epoch, + fence: vacantFence(input.capturedFence), + expected_manifest_revision: input.expectedManifestRevision, + manifest_revision: input.revision, + payload_digest: input.payloadDigest, + preset_id: input.presetId, + policy_digest: input.policyDigest, + manifest_digest: input.manifestDigest ?? '', + }) + await this._commandAllocation(state, intent) + if (action === 'launch') await this._commandPublication(state, intent) + const intentDigest = commandDigest(intent) + const result = await this.db.query( + `INSERT INTO ${state.q}.command_grants + (command_id,node_id,vm_id,allocation_id,operation_id,owner_id,action,epoch,intent_digest,intent) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::jsonb) ON CONFLICT DO NOTHING RETURNING command_id`, + [ + intent.command_id, + intent.node_id, + intent.vm_id, + intent.allocation_id, + intent.operation_id, + intent.owner_id, + intent.action, + intent.epoch, + intentDigest, + JSON.stringify(intent), + ], + ) + if (result.rowCount !== 1) throw commandFailure('node_command_conflict') + return { intent, intentDigest } + }) + } + + issueInitialPublish(input) { + return this._issueCommand('publish-initial', input) + } + issueLaunch(input) { + return this._issueCommand('launch', input) + } + + async _recordCommand(grantValue, receiptValue, action) { + const grant = commandGrant(grantValue) + if (grant.intent.action !== action) throw commandFailure('node_command_receipt_invalid') + const receipt = commandReceipt(grant, receiptValue) + return withTransaction(this.db, async () => { + const { state, epoch } = await this._commandContext(grant.intent) + if (grant.intent.epoch !== epoch) throw commandFailure('node_command_epoch_stale') + await this._commandAllocation(state, grant.intent) + if (action === 'launch') await this._commandPublication(state, grant.intent) + const row = ( + await this.db.query(`SELECT * FROM ${state.q}.command_grants WHERE command_id=$1 FOR UPDATE`, [ + grant.intent.command_id, + ]) + ).rows[0] + if (!row || row.revoked_at || !isDeepStrictEqual(storedGrant(row), grant)) + throw commandFailure('node_command_conflict') + if (row.receipt && !isDeepStrictEqual(commandReceipt(grant, row.receipt), receipt)) + throw commandFailure('node_command_conflict') + if (!row.receipt) + await this.db.query(`UPDATE ${state.q}.command_grants SET receipt=$2::jsonb WHERE command_id=$1`, [ + grant.intent.command_id, + JSON.stringify(receipt), + ]) + return receipt + }) + } + + recordInitialPublishReceipt(grant, receipt) { + return this._recordCommand(grant, receipt, 'publish-initial') + } + recordLaunchReceipt(grant, receipt) { + return this._recordCommand(grant, receipt, 'launch') + } + + /** A confirmed historical launch, not a new node state observation or Running proof. */ + async readLaunchReceipt({ nodeId, vmId }) { + identifier(vmId) + if (nodeId != null) identifier(nodeId) + return withTransaction(this.db, async () => { + const state = await nodeCommandAuthorityState(this.db) + if (!state.enabled) throw commandFailure('node_command_authority_unavailable') + const selected = ( + await this.db.query( + `SELECT * FROM ${state.q}.command_grants WHERE vm_id=$1 AND action='launch' AND ($2::text IS NULL OR node_id=$2)`, + [vmId, nodeId ?? null], + ) + ).rows[0] + if (!selected) return null + await this.db.query('SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))', [ + nodeGrantNamespace(this.schema), + nodeGrantResource(selected.node_id, vmId), + ]) + const epoch = await this._commandEpoch() + const row = ( + await this.db.query(`SELECT * FROM ${state.q}.command_grants WHERE command_id=$1`, [selected.command_id]) + ).rows[0] + const grant = storedGrant(row) + if (grant.intent.epoch !== epoch) throw commandFailure('node_command_epoch_stale') + await this._commandAllocation(state, grant.intent) + await this._commandPublication(state, grant.intent) + return row.receipt ? commandReceipt(grant, row.receipt) : null + }) + } + + commandReaderRoleSql({ role }) { + if (typeof role !== 'string' || !/^[a-z_][a-z0-9_]{0,62}$/.test(role)) + throw nodeAuthorityError('node_reader_invalid') + const q = quoteIdentifier(this.schema) + const r = quoteIdentifier(role) + return `GRANT SELECT ON ${q}.authorized_commands TO ${r}; + GRANT EXECUTE ON FUNCTION ${q}.command_current(text) TO ${r};` + } + async issueAllocation({ operation, nodeId, vmId, allocationId }) { identifier(nodeId) identifier(vmId) diff --git a/src/lib/db/node-management-authority.mjs b/src/lib/db/node-management-authority.mjs index 82761e964..574346d48 100644 --- a/src/lib/db/node-management-authority.mjs +++ b/src/lib/db/node-management-authority.mjs @@ -145,16 +145,142 @@ export async function initializeNodeAuthority(db) { }) } +/** New command authority is a separately and explicitly installed feature. */ +export async function nodeCommandAuthorityState(db) { + const base = await nodeAuthorityState(db) + if (!base.enabled) return { enabled: false, schema: base.schema } + const coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + const unavailable = () => nodeAuthorityError('node_command_authority_unavailable') + try { + const objects = ( + await db.query( + `SELECT + EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass($1) AND attname='command_protocol_version' AND NOT attisdropped) AS meta_version, + EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass($2) AND attname='command_protocol_version' AND NOT attisdropped) AS marker_version, + to_regclass($3) IS NOT NULL AS grants, + to_regclass($4) IS NOT NULL AS view, + to_regprocedure($5) IS NOT NULL AS current_function`, + [ + `${base.q}.meta`, + `${coord}.node_authority_features`, + `${base.q}.command_grants`, + `${base.q}.authorized_commands`, + `${base.q}.command_current(text)`, + ], + ) + ).rows[0] + if (Object.values(objects).every((value) => value === false)) return { enabled: false, schema: base.schema } + if (!Object.values(objects).every((value) => value === true)) throw unavailable() + const versions = ( + await db.query( + `SELECT m.command_protocol_version AS local_version, + f.command_protocol_version AS marker_version FROM ${base.q}.meta m + JOIN ${coord}.node_authority_features f ON f.application_schema=m.application_schema + WHERE m.id=1 AND f.authority_schema=$1`, + [base.schema], + ) + ).rows[0] + if (versions?.local_version !== 1 || versions.marker_version !== 1) throw unavailable() + return base + } catch { + throw unavailable() + } +} + +/** Trusted provisioning only. Startup/read/issuance paths never call this DDL. */ +export async function initializeNodeCommandAuthority(db) { + return withTransaction(db, async () => { + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:node-authority-bootstrap:${db.schema}`, + ]) + const base = await nodeAuthorityState(db) + if (!base.enabled) throw nodeAuthorityError('node_command_authority_unavailable') + const existing = await nodeCommandAuthorityState(db) + if (existing.enabled) return existing + const { q, schema } = base + const coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + const state = (await db.query(`SELECT phase FROM ${coord}.control WHERE id=1 FOR SHARE`)).rows[0] + if (state?.phase !== 'open') throw nodeAuthorityError('node_command_authority_unavailable') + await db.query(`ALTER TABLE ${q}.meta ADD COLUMN command_protocol_version integer NOT NULL DEFAULT 1 CHECK(command_protocol_version=1); + ALTER TABLE ${coord}.node_authority_features ADD COLUMN command_protocol_version integer NOT NULL DEFAULT 1 CHECK(command_protocol_version=1); + CREATE TABLE ${q}.command_grants ( + command_id text PRIMARY KEY, node_id text NOT NULL, vm_id text NOT NULL, + allocation_id text NOT NULL REFERENCES ${q}.allocations(allocation_id), + operation_id text NOT NULL, owner_id text NOT NULL, + action text NOT NULL CHECK(action IN ('publish-initial','launch')), + epoch bigint NOT NULL CHECK(epoch>0), intent_digest text NOT NULL CHECK(intent_digest ~ '^[a-f0-9]{64}$'), + intent jsonb NOT NULL, receipt jsonb, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + revoked_at timestamptz, UNIQUE(allocation_id,action)); + CREATE INDEX command_grants_operation ON ${q}.command_grants(operation_id,owner_id) WHERE revoked_at IS NULL; + REVOKE ALL ON ${q}.command_grants FROM PUBLIC; + CREATE FUNCTION ${q}.command_current(p_command text) RETURNS boolean + LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $body$ + DECLARE g record; allowed boolean; + BEGIN + IF NOT EXISTS(SELECT 1 FROM ${q}.reader_identity()) + OR has_table_privilege(session_user,'${schema}.command_grants','INSERT,UPDATE,DELETE,TRUNCATE') THEN RETURN false; END IF; + SELECT a.*,m.application_schema INTO g FROM ${q}.command_grants a + JOIN ${q}.allocations origin ON origin.allocation_id=a.allocation_id + CROSS JOIN ${q}.meta m + JOIN ${q}.reader_roles r ON r.role_name=session_user + JOIN ${coord}.control c ON c.id=1 + JOIN ${coord}.node_authority_features f ON f.application_schema=m.application_schema + WHERE a.command_id=p_command AND a.node_id=r.node_id AND a.revoked_at IS NULL + AND m.id=1 AND m.protocol_version=1 AND m.command_protocol_version=1 + AND f.protocol_version=1 AND f.command_protocol_version=1 AND f.authority_schema='${schema}' + AND c.application_schema=m.application_schema AND c.phase='open' AND c.epoch=a.epoch + AND origin.node_id=a.node_id AND origin.vm_id=a.vm_id AND origin.epoch=a.epoch + AND origin.receipt->>'allocation_id'=a.allocation_id AND origin.receipt->>'epoch'=a.epoch::text + AND origin.receipt->'fence'->>'generation'='1' + AND a.intent->'fence'->>'generation'='1' + AND a.intent->'fence'->>'incarnation'=origin.receipt->'fence'->>'incarnation' + AND a.intent->'fence'->'executor'='{"kind":"","node_boot_id":""}'::jsonb + AND a.intent->>'action'=a.action AND a.intent->>'epoch'=a.epoch::text + AND a.intent->>'node_id'=a.node_id AND a.intent->>'vm_id'=a.vm_id + AND a.intent->>'allocation_id'=a.allocation_id AND a.intent->>'command_id'=a.command_id + AND a.intent->>'operation_id'=a.operation_id AND a.intent->>'owner_id'=a.owner_id + AND a.intent->>'preset_id'='claude-docker-v1' AND a.intent->>'manifest_revision'='1' + AND a.intent->>'policy_digest' ~ '^[a-f0-9]{64}$' + AND ((a.action='publish-initial' AND a.intent->>'expected_manifest_revision'='0' + AND a.intent->>'payload_digest' ~ '^[a-f0-9]{64}$' AND a.intent->>'manifest_digest'='') + OR (a.action='launch' AND a.intent->>'expected_manifest_revision'='1' AND a.intent->>'payload_digest'='' + AND EXISTS(SELECT 1 FROM ${q}.command_grants p WHERE p.allocation_id=a.allocation_id + AND p.action='publish-initial' AND p.epoch=a.epoch AND p.receipt IS NOT NULL + AND p.receipt->>'intent_digest'=p.intent_digest + AND p.receipt->>'manifest_revision'='1' + AND p.receipt->>'manifest_digest'=a.intent->>'manifest_digest' + AND p.intent->>'policy_digest'=a.intent->>'policy_digest' + AND p.intent->'fence'=a.intent->'fence'))); + IF NOT FOUND THEN RETURN false; END IF; + EXECUTE format('SELECT EXISTS(SELECT 1 FROM %I.vm_operations WHERE id=$1 AND owner_id=$2 AND vm_id=$3 + AND kind=$4 AND released_at IS NULL AND uncertain_at IS NULL)',g.application_schema) + INTO allowed USING g.operation_id,g.owner_id,g.vm_id, + CASE g.action WHEN 'publish-initial' THEN 'node_publish_initial' ELSE 'node_launch' END; + RETURN allowed; + EXCEPTION WHEN undefined_table OR invalid_schema_name THEN RETURN false; + END $body$; + REVOKE ALL ON FUNCTION ${q}.command_current(text) FROM PUBLIC; + CREATE VIEW ${q}.authorized_commands WITH(security_barrier=true) AS + SELECT command_id,node_id,vm_id,allocation_id,operation_id,owner_id,action,epoch,intent_digest + FROM ${q}.command_grants WHERE ${q}.command_current(command_id); + REVOKE ALL ON ${q}.authorized_commands FROM PUBLIC`) + return nodeCommandAuthorityState(db) + }) +} + // Caller must already hold the exact VM operation row. This lock order also // governs issuance; the node's shared-lock path uses MVCC reads only. export async function revokeNodeOperationGrants(db, { operationId, ownerId }) { if (!db.isTransaction) throw nodeAuthorityError() const state = await nodeAuthorityState(db) if (!state.enabled) return + const commands = await nodeCommandAuthorityState(db) const rows = ( await db.query( - `SELECT DISTINCT node_id,vm_id FROM ${state.q}.allocations - WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL ORDER BY node_id,vm_id`, + `SELECT DISTINCT node_id,vm_id FROM ( + SELECT node_id,vm_id FROM ${state.q}.allocations WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL + ${commands.enabled ? `UNION ALL SELECT node_id,vm_id FROM ${state.q}.command_grants WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL` : ''} + ) grants ORDER BY node_id,vm_id`, [operationId, ownerId], ) ).rows @@ -164,4 +290,10 @@ export async function revokeNodeOperationGrants(db, { operationId, ownerId }) { WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL`, [operationId, ownerId], ) + if (commands.enabled) + await db.query( + `UPDATE ${state.q}.command_grants SET revoked_at=clock_timestamp() + WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL`, + [operationId, ownerId], + ) } diff --git a/test/unit/node-command-grants.test.mjs b/test/unit/node-command-grants.test.mjs index 601ec051c..656dc56d2 100644 --- a/test/unit/node-command-grants.test.mjs +++ b/test/unit/node-command-grants.test.mjs @@ -25,7 +25,10 @@ async function fixture(t) { const other = new NodeManagementGrants(peer) const roles = [] const readers = [] - t.after(async () => { + let closed = false + const close = async () => { + if (closed) return + closed = true await maintenance.stop() await peer.close() for (const client of readers) await client.end() @@ -39,7 +42,8 @@ async function fixture(t) { } finally { await cleanup.end() } - }) + } + t.after(close) await maintenance.start() await maintenance.acknowledgeEpoch(1) await service.initialize() @@ -119,114 +123,133 @@ async function fixture(t) { publication, publish, reader, + close, q: quoteIdentifier(service.schema), } } -test('new actions require explicit trusted feature initialization without changing allocation v1', async (t) => { - const fx = await fixture(t) - await fx.run('node_publish_initial', async (operation) => { - await assert.rejects(fx.service.issueInitialPublish(fx.input(operation)), { - code: 'node_command_authority_unavailable', - }) +function commandTest(name, callback) { + test(name, async (t) => { + const fx = await fixture(t) + try { + await callback(fx) + } finally { + await fx.close() + } }) - assert.equal( - (await fx.peer.query('SELECT to_regclass($1) AS relation', [`${fx.service.schema}.command_grants`])).rows[0] - .relation, - null, - ) - await fx.service.initializeCommands() - const meta = (await fx.peer.query(`SELECT protocol_version,command_protocol_version FROM ${fx.q}.meta`)).rows[0] - assert.deepEqual(meta, { protocol_version: 1, command_protocol_version: 1 }) - const marker = ( - await fx.peer.query( - `SELECT protocol_version,command_protocol_version FROM ${quoteIdentifier(fx.maintenance.schema)}.node_authority_features`, - ) - ).rows[0] - assert.deepEqual(marker, meta) - await fx.service.initialize() - assert.equal((await fx.peer.query(`SELECT count(*) AS n FROM ${fx.q}.allocations`)).rows[0].n, 1) -}) +} -test('publish and launch bind the original allocation, independent operations, digests and immutable receipts', async (t) => { - const fx = await fixture(t) - await fx.service.initializeCommands() - const published = await fx.publish() - assert.equal(published.manifest_digest, manifestDigest) - let launched - await fx.run('node_launch', async (operation) => { - const grant = await fx.service.issueLaunch( - fx.input(operation, { - commandId: 'launch-one', - expectedManifestRevision: '1', - payloadDigest: '', - manifestDigest, - }), +commandTest( + 'new actions require explicit trusted feature initialization without changing allocation v1', + async (fx) => { + await fx.run('node_publish_initial', async (operation) => { + await assert.rejects(fx.service.issueInitialPublish(fx.input(operation)), { + code: 'node_command_authority_unavailable', + }) + }) + assert.equal( + (await fx.peer.query('SELECT to_regclass($1) AS relation', [`${fx.service.schema}.command_grants`])).rows[0] + .relation, + null, ) - assert.equal(grant.intent.action, 'launch') - assert.equal(grant.intent.expected_manifest_revision, '1') - assert.equal(grant.intent.payload_digest, '') - const fence = { - generation: '1', - incarnation: capturedFence.incarnation, - executor: { - kind: 'container', - node_boot_id: 'synthetic-boot', - container_id: 'e'.repeat(64), - docker_daemon_id: 'synthetic-daemon', - started_at: '2026-10-22T01:02:03.123456789Z', - }, - } - launched = await fx.service.recordLaunchReceipt(grant, { ...fx.publication(grant), fence }) - }) - const rows = ( - await fx.peer.query(`SELECT action,operation_id,receipt,revoked_at FROM ${fx.q}.command_grants ORDER BY action`) - ).rows - assert.deepEqual( - rows.map((row) => row.action), - ['launch', 'publish-initial'], - ) - assert.notEqual(rows[0].operation_id, rows[1].operation_id) - assert.ok(rows.every((row) => row.revoked_at && row.receipt)) - assert.deepEqual(await fx.other.readLaunchReceipt({ nodeId, vmId }), launched) - assert.equal(launched.fence.incarnation, capturedFence.incarnation) - assert.equal(launched.fence.executor.container_id, 'e'.repeat(64)) -}) + await fx.service.initializeCommands() + const meta = (await fx.peer.query(`SELECT protocol_version,command_protocol_version FROM ${fx.q}.meta`)).rows[0] + assert.deepEqual(meta, { protocol_version: 1, command_protocol_version: 1 }) + const marker = ( + await fx.peer.query( + `SELECT protocol_version,command_protocol_version FROM ${quoteIdentifier(fx.maintenance.schema)}.node_authority_features`, + ) + ).rows[0] + assert.deepEqual(marker, meta) + await fx.service.initialize() + assert.equal((await fx.peer.query(`SELECT count(*) AS n FROM ${fx.q}.allocations`)).rows[0].n, 1) + }, +) -test('fresh fences, missing publication receipts, wrong revisions and absent exact operation scope are rejected', async (t) => { - const fx = await fixture(t) - await fx.service.initializeCommands() - await assert.rejects(fx.service.issueInitialPublish(fx.input({ operationId: 'forged', ownerId: 'forged' })), { - code: 'node_operation_scope_required', - }) - await fx.run('node_launch', async (operation) => { - await assert.rejects( - fx.service.issueLaunch( +commandTest( + 'publish and launch bind the original allocation, independent operations, digests and immutable receipts', + async (fx) => { + await fx.service.initializeCommands() + const published = await fx.publish() + assert.equal(published.manifest_digest, manifestDigest) + let launched + await fx.run('node_launch', async (operation) => { + const grant = await fx.service.issueLaunch( fx.input(operation, { commandId: 'launch-one', expectedManifestRevision: '1', payloadDigest: '', manifestDigest, }), - ), - { code: 'node_publication_required' }, + ) + assert.equal(grant.intent.action, 'launch') + assert.equal(grant.intent.expected_manifest_revision, '1') + assert.equal(grant.intent.payload_digest, '') + const fence = { + generation: '1', + incarnation: capturedFence.incarnation, + executor: { + kind: 'container', + node_boot_id: 'synthetic-boot', + container_id: 'e'.repeat(64), + docker_daemon_id: 'synthetic-daemon', + started_at: '2026-10-22T01:02:03.123456789Z', + }, + } + launched = await fx.service.recordLaunchReceipt(grant, { ...fx.publication(grant), fence }) + }) + const rows = ( + await fx.peer.query(`SELECT action,operation_id,receipt,revoked_at FROM ${fx.q}.command_grants ORDER BY action`) + ).rows + assert.deepEqual( + rows.map((row) => row.action), + ['launch', 'publish-initial'], ) - }) - await fx.run('node_publish_initial', async (operation) => { - for (const patch of [ - { capturedFence: { ...capturedFence, incarnation: 'f'.repeat(32) } }, - { expectedManifestRevision: '1' }, - { revision: '01' }, - { payloadDigest: 'not-a-digest' }, - { presetId: 'arbitrary-preset' }, - ]) { - await assert.rejects(fx.service.issueInitialPublish(fx.input(operation, patch)), { code: 'node_command_invalid' }) - } - }) -}) + assert.notEqual(rows[0].operation_id, rows[1].operation_id) + assert.ok(rows.every((row) => row.revoked_at && row.receipt)) + assert.deepEqual(await fx.other.readLaunchReceipt({ nodeId, vmId }), launched) + assert.equal(launched.fence.incarnation, capturedFence.incarnation) + assert.equal(launched.fence.executor.container_id, 'e'.repeat(64)) + }, +) + +commandTest( + 'fresh fences, missing publication receipts, wrong revisions and absent exact operation scope are rejected', + async (fx) => { + await fx.service.initializeCommands() + await assert.rejects(fx.service.issueInitialPublish(fx.input({ operationId: 'forged', ownerId: 'forged' })), { + code: 'node_operation_scope_required', + }) + await fx.run('node_launch', async (operation) => { + await assert.rejects( + fx.service.issueLaunch( + fx.input(operation, { + commandId: 'launch-one', + expectedManifestRevision: '1', + payloadDigest: '', + manifestDigest, + }), + ), + { code: 'node_publication_required' }, + ) + }) + await fx.run('node_publish_initial', async (operation) => { + for (const patch of [ + { capturedFence: { ...capturedFence, incarnation: 'f'.repeat(32) } }, + { expectedManifestRevision: '1' }, + { revision: '01' }, + { payloadDigest: 'not-a-digest' }, + { presetId: 'arbitrary-preset' }, + ]) { + await assert.rejects(fx.service.issueInitialPublish(fx.input(operation, patch)), { + code: 'node_command_invalid', + }) + } + }) + }, +) -test('a command cannot be reissued, relabeled or replaced after an unknown attempt', async (t) => { - const fx = await fixture(t) +commandTest('a command cannot be reissued, relabeled or replaced after an unknown attempt', async (fx) => { await fx.service.initializeCommands() await fx.run('node_publish_initial', async (operation) => { const first = await fx.service.issueInitialPublish(fx.input(operation)) @@ -237,7 +260,7 @@ test('a command cannot be reissued, relabeled or replaced after an unknown attem } await operation.markUncertain() await assert.rejects(fx.service.recordInitialPublishReceipt(first, fx.publication(first)), { - code: 'node_operation_invalid', + code: 'node_operation_scope_required', }) return { vmOperationUncertain: true } }) @@ -247,75 +270,80 @@ test('a command cannot be reissued, relabeled or replaced after an unknown attem assert.equal((await new SharedVmOperations(fx.peer).tryAcquire({ vmId })).ok, false) }) -test('receipt validation rejects secret fields and mismatched intent without committing a partial receipt', async (t) => { - const fx = await fixture(t) - await fx.service.initializeCommands() - await fx.run('node_publish_initial', async (operation) => { - const grant = await fx.service.issueInitialPublish(fx.input(operation)) - for (const receipt of [ - fx.publication(grant, { access_token: 'synthetic-secret-must-not-persist' }), - fx.publication(grant, { intent_digest: 'f'.repeat(64) }), - fx.publication(grant, { intent: { ...grant.intent, owner_id: 'other-owner' } }), - fx.publication(grant, { manifest_revision: '2' }), - ]) { - await assert.rejects(fx.service.recordInitialPublishReceipt(grant, receipt), { - code: 'node_command_receipt_invalid', - }) - assert.equal((await fx.peer.query(`SELECT receipt FROM ${fx.q}.command_grants`)).rows[0].receipt, null) - } - const saved = await fx.service.recordInitialPublishReceipt(grant, fx.publication(grant)) - assert.deepEqual(await fx.service.recordInitialPublishReceipt(grant, fx.publication(grant)), saved) - await assert.rejects( - fx.service.recordInitialPublishReceipt(grant, fx.publication(grant, { manifest_digest: 'f'.repeat(64) })), - { code: 'node_command_conflict' }, - ) - }) -}) +commandTest( + 'receipt validation rejects secret fields and mismatched intent without committing a partial receipt', + async (fx) => { + await fx.service.initializeCommands() + await fx.run('node_publish_initial', async (operation) => { + const grant = await fx.service.issueInitialPublish(fx.input(operation)) + for (const receipt of [ + fx.publication(grant, { access_token: 'synthetic-secret-must-not-persist' }), + fx.publication(grant, { intent_digest: 'f'.repeat(64) }), + fx.publication(grant, { intent: { ...grant.intent, owner_id: 'other-owner' } }), + fx.publication(grant, { manifest_revision: '2' }), + ]) { + await assert.rejects(fx.service.recordInitialPublishReceipt(grant, receipt), { + code: 'node_command_receipt_invalid', + }) + assert.equal((await fx.peer.query(`SELECT receipt FROM ${fx.q}.command_grants`)).rows[0].receipt, null) + } + const saved = await fx.service.recordInitialPublishReceipt(grant, fx.publication(grant)) + assert.deepEqual(await fx.service.recordInitialPublishReceipt(grant, fx.publication(grant)), saved) + await assert.rejects( + fx.service.recordInitialPublishReceipt(grant, fx.publication(grant, { manifest_digest: 'f'.repeat(64) })), + { code: 'node_command_conflict' }, + ) + }) + }, +) -test('the node reader sees only live commands and release waits for its bounded authorization lock', async (t) => { - const fx = await fixture(t) - await fx.service.initializeCommands() - const reader = await fx.reader() - await assert.rejects(reader.query(`SELECT * FROM ${fx.q}.command_grants`), { code: '42501' }) - await fx.run('node_publish_initial', async (operation) => { - const grant = await fx.service.issueInitialPublish(fx.input(operation)) - const rows = (await reader.query(`SELECT action,epoch::text,intent_digest FROM ${fx.q}.authorized_commands`)).rows - assert.deepEqual(rows, [{ action: 'publish-initial', epoch: '1', intent_digest: grant.intentDigest }]) - await reader.query('BEGIN READ ONLY') - await reader.query('SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))', [ - nodeGrantNamespace(fx.service.schema), - nodeGrantResource(nodeId, vmId), - ]) - const releasing = operation.release() - try { - assert.equal(await Promise.race([releasing.then(() => 'released'), delay(75).then(() => 'waiting')]), 'waiting') - } finally { - await reader.query('COMMIT') - } - assert.equal((await releasing).released, true) - assert.equal((await reader.query(`SELECT * FROM ${fx.q}.authorized_commands`)).rowCount, 0) - }) -}) +commandTest( + 'the node reader sees only live commands and release waits for its bounded authorization lock', + async (fx) => { + await fx.service.initializeCommands() + const reader = await fx.reader() + await assert.rejects(reader.query(`SELECT * FROM ${fx.q}.command_grants`), { code: '42501' }) + await fx.run('node_publish_initial', async (operation) => { + const grant = await fx.service.issueInitialPublish(fx.input(operation)) + const rows = (await reader.query(`SELECT action,epoch::text,intent_digest FROM ${fx.q}.authorized_commands`)).rows + assert.deepEqual(rows, [{ action: 'publish-initial', epoch: '1', intent_digest: grant.intentDigest }]) + await reader.query('BEGIN READ ONLY') + await reader.query('SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))', [ + nodeGrantNamespace(fx.service.schema), + nodeGrantResource(nodeId, vmId), + ]) + const releasing = operation.release() + try { + assert.equal(await Promise.race([releasing.then(() => 'released'), delay(75).then(() => 'waiting')]), 'waiting') + } finally { + await reader.query('COMMIT') + } + assert.equal((await releasing).released, true) + assert.equal((await reader.query(`SELECT * FROM ${fx.q}.authorized_commands`)).rowCount, 0) + }) + }, +) -test('enabled command state corruption fails closed instead of performing runtime DDL or releasing ownership', async (t) => { - const fx = await fixture(t) - await fx.service.initializeCommands() - await fx.peer.query(`DROP TABLE ${fx.q}.command_grants CASCADE`) - await assert.rejects( - fx.run('node_publish_initial', (operation) => fx.service.issueInitialPublish(fx.input(operation))), - { code: 'node_command_authority_unavailable' }, - ) - assert.equal( - (await fx.peer.query('SELECT to_regclass($1) AS relation', [`${fx.service.schema}.command_grants`])).rows[0] - .relation, - null, - ) - const rows = (await fx.peer.query("SELECT released_at FROM vm_operations WHERE kind='node_publish_initial'")).rows - assert.equal(rows[0].released_at, null) -}) +commandTest( + 'enabled command state corruption fails closed instead of performing runtime DDL or releasing ownership', + async (fx) => { + await fx.service.initializeCommands() + await fx.peer.query(`DROP TABLE ${fx.q}.command_grants CASCADE`) + await assert.rejects( + fx.run('node_publish_initial', (operation) => fx.service.issueInitialPublish(fx.input(operation))), + { code: 'node_command_authority_unavailable' }, + ) + assert.equal( + (await fx.peer.query('SELECT to_regclass($1) AS relation', [`${fx.service.schema}.command_grants`])).rows[0] + .relation, + null, + ) + const rows = (await fx.peer.query("SELECT released_at FROM vm_operations WHERE kind='node_publish_initial'")).rows + assert.equal(rows[0].released_at, null) + }, +) -test('business epoch changes reject persisted old command receipts and deny the node reader', async (t) => { - const fx = await fixture(t) +commandTest('business epoch changes reject persisted old command receipts and deny the node reader', async (fx) => { await fx.service.initializeCommands() const reader = await fx.reader() await fx.run('node_publish_initial', async (operation) => { From de5a47c49ed1aed06743cad3700d6622e54a5555 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:47:53 +0800 Subject: [PATCH 370/473] docs: record independent review rounds and fixed-source regression evidence --- .../2026-10-21-kubernetes-review-progress.md | 21 +- .../independent-review-06891990.md | 95 +++ .../independent-review-08a18287.md | 58 ++ .../independent-review-e9076922.md | 28 + .../review-progress/verification.json | 673 +++++++++++++++++- 5 files changed, 869 insertions(+), 6 deletions(-) create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-06891990.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-08a18287.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-e9076922.md diff --git a/docs/plans/2026-10-21-kubernetes-review-progress.md b/docs/plans/2026-10-21-kubernetes-review-progress.md index e06d27057..b8dfa58f7 100644 --- a/docs/plans/2026-10-21-kubernetes-review-progress.md +++ b/docs/plans/2026-10-21-kubernetes-review-progress.md @@ -15,7 +15,7 @@ ## 独立会话审查 -两轮审查均使用单独 Codex CLI 会话和固定 detached worktree。审查会话只读源码、测试、文档与 Git 对象，没有执行测试或访问部署环境。主会话另行动态复现并验证修复。 +五轮审查均使用单独 Codex CLI 会话和固定 detached worktree。审查会话只读源码、测试、文档与 Git 对象，没有执行测试或访问部署环境。主会话另行动态复现并验证修复。 首轮源码为 `0ffae4b0`，独立会话为 `01a15ee0-3324-7490-928c-8343beb13b5e`。发现 4 项 P1、1 项 P2。对应失败规格和定向修复如下： @@ -36,14 +36,27 @@ NUL 问题动态复现得到 PostgreSQL `22021`。修复恢复依赖分支已有 - 已有容器端点的代理网络清理可能长期保留整个代理池声明。 - watchdog bridge 修复没有取得共享代理/VM 互斥，且丢弃未知结果。 -这 4 项正在复现和修复。本报告不将第二轮审查写为通过。两轮原始发现及静态审查边界见 [首轮报告](evidence/kubernetes/review-progress/independent-review-0ffae4b0.md) 和 [第二轮报告](evidence/kubernetes/review-progress/independent-review-66a580de.md)。 +这 4 项已完成定向修复：初始化 `d2b62208` 为 118/118；代理清理 `d824fa2a` 为 58/58；watchdog 及协议观察 `befd24d1` 为 72/72。本报告不将定向结果称为最终审查通过。原始发现及静态审查边界见 [首轮报告](evidence/kubernetes/review-progress/independent-review-0ffae4b0.md) 和 [第二轮报告](evidence/kubernetes/review-progress/independent-review-66a580de.md)。 + +第三轮源码为 `068919906653acf0e6dd0f749f83cb808f9f36f1`，独立会话为 `01a15f1a-0084-78a2-917f-2792725bceea`。审查发现 5 项 P1。嵌套 VM 操作未知结果、手动 Codex 刷新和明确未开始的繁忙拒绝，分别由审查开始后的 `34854048`、`12088dde` 和 `2065da50` 修正。已结束的包初始化普通失败在 `da7700bd` 修正，95/95 通过。worker 内部刷新重试覆盖先前未知结果在 `08a18287` 修正，Go race 与 Node 52/52 定向检查通过。见 [第三轮报告](evidence/kubernetes/review-progress/independent-review-06891990.md)。 + +第四轮源码为 `08a182870b6334691a114c7b93a0f07032588ff7`，独立会话为 `01a15f31-9c1b-77e1-bbee-e2941606450d`。逐项复核第三轮 5 项指定触发路径已修正，新增 1 项 P1：首次远端目录准备已取得退出码及通道关闭，仍被发布队列和协议标为未知。远端证明传播在 `04dde1cd` 修正，固定源码 94 项通过、2 项平台跳过；协议两处分类和真实准入验证仍在进行。见 [第四轮报告](evidence/kubernetes/review-progress/independent-review-08a18287.md)。 + +`08a18287` 的后续全量结果：unit 368 文件、4182 项，4159 通过、13 失败、10 跳过；E2E 31 文件、130 项，110 通过、19 失败、1 跳过。原 7 项 unit、17 项 E2E 的规范化断言与 `0ffae4b0` 一致。新增 6 项 unit、2 项 E2E 已分别复现，涉及测试数据库初始化、未启动的内存校验、备份调度拒绝和测试 Docker 隔离。修正提交为 `f011a227`、`68d2b54d`、`43076583`、`f9f396dd`，均有定向检查；仍需新的全量检查。该源码的 Go race 为 190 通过、1 辅助跳过，vet、gofmt、733 文件格式检查和 323 文件语法检查通过。 + +第五轮源码为 `e9076922a4af1f43840e6f9db8381790c809ee21`。独立会话未发现新增 P0/P1/P2，复核了首次准备已结束和 unknown 优先的释放链。真实 runner、双 PG 连接、请求活动及远端发布相关 103 项中，101 通过、2 项平台跳过。fixture 的发布入口和后端仍受测试控制，不等于整条生产 SSH 调用链验收。见 [第五轮报告](evidence/kubernetes/review-progress/independent-review-e9076922.md)。 + +该源码全量 unit 为 369 文件、4201 项，4183 通过、8 失败、10 跳过；E2E 为 32 文件、131 项，113 通过、17 失败、1 跳过。原 7 项 unit、17 项 E2E 的规范化断言仍一致。新增 1 项 unit 期待调度发现的维护准入拒绝抛出异常；`60d63596` 更正为验证零发现读取、零回调、零 job、零 claim 且无持久化失败，相关 8 项通过。该源码 Go race 为 190 通过、1 辅助跳过；vet、gofmt、735 文件格式检查、302 个 `src` MJS 语法检查通过。之后的节点发布、命令授权和 HA 接线未包含在该全量结果中。 ## 后续固定源码检查 - `f4a685b98f31d467b6a2c8c5a8b7cf7749bf15c5`：watchdog 共享恢复预算、故障准入及相关回归 99/99。 - `bf7615f8be9adf4f569c2a7d802be514a4c58e64`：共享熔断及原准入、面板、备份和重试 142/142。失败次数在 PostgreSQL 累计；半开状态的探测绑定持久 admission；未知结果不随时间释放；旧结果不能覆盖重置后的代次。 - `e1e5d2a9e6cea391fb96aacc6b0757fa3ad0360f`：KSG2 客户端、PG 绑定及相关回归 156/156。实际 Go gate、合成 Unix kernel 与两条 PG 连接验证：正常持久完成后可释放；内部错误仍保留。发送前绑定原 Fence；响应后另取原 ticket 的节点持久完成状态。绑定新 job 与凭据刷新 VM 操作使用同一资源锁。 +- `34854048`：私有 PG 节点管理授权及初始 allocate，Node 63/63；Go race 182 通过、1 辅助跳过；Darwin 编译通过。受限节点角色只读授权，操作结束或未知时撤销 grant；未授权管理 RPC 默认拒绝。尚未接真实 VM 创建及节点文件发布。 +- `06891990`：共享刷新接线 251/251；`2065da50` 的明确未开始修正 166/166。新增账号互斥持久记录，表数为 42，业务迁移为 18。私有节点授权不属于业务备份恢复范围。 +- `c05da6f2`：远端发布原始 Promise 等待，203 通过、2 平台跳过。协议两处补充接线见 `63e641b5`；新的合成测试在此前源码上分别复现继续选择 VM、调度提前开始和发布拒绝被吞。修正后持久准入及两处协议等待的 3 项检查通过。 -上述检查只使用专用测试 PostgreSQL、合成请求、模拟节点或测试自己的子进程。KSG2 集成使用合成 executor 身份，没有调用真实 Docker Engine；它不能证明可信启动或权限隔离。生产 capability 来源、节点文件发布和管理授权、凭据刷新接线、多节点运行及故障接管仍在实现。旧预编译 payload 继续复用。 +定向协议检查使用专用测试 PostgreSQL、合成请求、模拟节点或测试自己的子进程。KSG2 集成使用合成 executor 身份，没有调用真实 Docker Engine；它不能证明可信启动或权限隔离。旧 E2E harness 的 mock gateway 仍会创建真实本机 Docker 网络，已在 `f9f396dd` 隔离；此前资源按精确 fixture、network ID、无端点和无 helper 证据核对后，仅删除 10 个测试网络。6 个带业务端点的网络和 3 个固定名网络保留。生产 capability 来源、节点文件发布和管理授权、多节点运行及故障接管仍在实现。旧预编译 payload 继续复用。 -结构化统计、日志摘要与边界见 [验证证据](evidence/kubernetes/review-progress/verification.json)。任务测试资源按精确所有权清单清理；专用 PostgreSQL 与共享 Go 缓存在后续验证结束前保留。没有修改生产环境。 +结构化统计、日志摘要与边界见 [验证证据](evidence/kubernetes/review-progress/verification.json)。已精确删除 9 个完成检查的 root 工作树或源码副本；另有固定检查脚本自行清理的目录。专用 PostgreSQL 与共享 Go 缓存在后续验证结束前保留。没有修改生产环境。 diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-06891990.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-06891990.md new file mode 100644 index 000000000..ba667b20f --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-06891990.md @@ -0,0 +1,95 @@ +发现 **5 项 P1**，未发现有证据支持的 P0。以下均为静态源码结论，合成复现步骤交主会话执行。 + +核对提交为 `068919906653acf0e6dd0f749f83cb808f9f36f1`，基线为 `66a580de17670a954638fbfb46fee25740adab78`。开始和结束时 HEAD 一致，工作区干净。`git diff --check` 无输出。 + +1. **P1：Claude worker 的内部重试仍会覆盖刷新结果不明。** + + 位置：`worker/internal/oauth/refresher.go:171`、`:175`、`:181`；`worker/internal/oauthcmd/oauthcmd.go:89`；[src/lib/transport/slot-oauth.mjs:236](https://github.com/liubing0806/vm2api/blob/068919906653acf0e6dd0f749f83cb808f9f36f1/src/lib/transport/slot-oauth.mjs#L236)。 + + 触发条件：首次刷新请求已被上游接收，刷新令牌（RT）可能已经轮换，但响应中断；worker 随后使用原 RT 重试，取得明确的 400/401 拒绝。 + + 调用链：共享刷新服务 → `runSlotOauthRaw` → worker `Ensure` → `requestWithRetry`。worker 只返回最后一次错误。Node 根据最后的退出码、HTTP 状态和错误码设置 `attemptFinished:true`。`shared-credential-refresh.mjs:162` 随后完成后台任务并释放 VM 操作记录。首次刷新结果不明没有保留。 + + 影响：账号恢复接受新执行或刷新，而首次刷新是否消耗 RT 仍未确认。 + + 最小修复：worker 累积每次请求的不确定性；传输结果不明后停止重试，并显式返回未知状态。 + + 合成复现：首次模拟上游接收请求后断开响应，第二次返回 `invalid_grant`。检查 worker 输出以及共享任务 `finished_at`、VM 操作 `released_at`。预期保留未知；当前源码路径会按明确失败完成。 + +2. **P1：package 的 Codex 探测可在嵌套刷新未知后释放账号互斥。** + + 位置：[src/lib/vm/shared-vm-operations.mjs:210](https://github.com/liubing0806/vm2api/blob/068919906653acf0e6dd0f749f83cb808f9f36f1/src/lib/vm/shared-vm-operations.mjs#L210)、`src/lib/oauth/shared-credential-refresh.mjs:141`、[src/lib/oauth/openai-quota.mjs:305](https://github.com/liubing0806/vm2api/blob/068919906653acf0e6dd0f749f83cb808f9f36f1/src/lib/oauth/openai-quota.mjs#L305)。 + + 触发条件：导入带有效账号标识、AT 和 RT 的 Codex package；启动成功；额度探测返回 401；刷新请求超时或断连。 + + 调用链：`panel-routes.mjs:4242` 的外层 VM 操作 → `bringUpImportedVm:895` → `buildProbeOne` → 共享刷新。嵌套 `SharedVmOperations.run` 直接返回回调结果，不处理 `vmOperationUncertain`。刷新服务将后台任务标为未知，但没有标记继承的 VM 操作。额度层把未知转换为普通 `rotateError`，package 再转换为 `probe.ok:false`。 + + 外层回调正常结束后，VM 和账号互斥被释放。执行准入只检查 `vm_operations`，因此同 VM 或同账号另一 VM 可以取得新准入。未知后台任务仍存在，但它不能阻止该准入。 + + 最小修复：嵌套 VM 操作在返回未知结果或发生效果不明异常时，直接标记继承的操作未知；额度层同时保留未知标志。 + + 合成复现：在实际 `SharedVmOperations.run` 内调用 `queryOpenaiQuota`，模拟额度 401、刷新断连。外层正常返回后，用第二条 PG 连接尝试同账号新准入。当前路径可释放 VM 操作；预期应拒绝新准入。 + +3. **P1：已启用的手动 Codex 刷新仍绕过共享刷新与账号互斥。** + + 位置：[src/lib/admin/panel-routes.mjs:5272](https://github.com/liubing0806/vm2api/blob/068919906653acf0e6dd0f749f83cb808f9f36f1/src/lib/admin/panel-routes.mjs#L5272)、同文件 `:5282`。 + + 触发条件：Search 或 quota 已持有该账号的共享刷新操作，或者此前刷新仍为未知；用户调用 `POST /api/panel/vms/:id/oauth/refresh`。 + + 该入口直接调用 `refreshCodexAccessToken`，成功后直接写凭据。它不查询共享任务，也不取得 VM/账号互斥。外层面板包装器只等待持久化，没有补充互斥。失败分支也没有使用新增的 `attemptFinished`。 + + 影响：同一 RT 可以并发提交；迟到结果可以覆盖共享刷新发布的凭据；手动刷新超时后仍允许再次刷新。这是当前可调用入口的遗漏。 + + 最小修复：Kubernetes 模式下改用 `refreshSharedCodexCredential`，统一处理发布、明确失败和未知结果。 + + 合成复现：保持一个共享刷新回调未完成，再调用手动刷新 handler；在模块边界注入合成刷新函数。断言原始刷新调用次数应仍为一次。另覆盖已有未知任务的情况。 + +4. **P1：package 把已明确结束的 bootstrap 普通失败标为未知，长期占用代理池。** + + 位置：[src/lib/admin/panel-routes.mjs:888](https://github.com/liubing0806/vm2api/blob/068919906653acf0e6dd0f749f83cb808f9f36f1/src/lib/admin/panel-routes.mjs#L888)、同文件 `:669`、`:673`。 + + 触发条件：package 启动成功；bootstrap 前台命令正常退出，但 hello 或 usage 业务检查失败；辅助操作和清理均已结束；resident 关闭。 + + 原始 bootstrap 可以返回 `{ok:false, attemptFinished:true, vmOperationUncertain:false}`。统一完成判定接受该结果，但外层 `vmEffect` 仍因 `ok!==true` 标记 VM 和代理池未知。 + + 影响：没有未结束执行的普通初始化失败，也会阻止后续代理管理和全局维护。 + + 最小修复：该入口依据统一执行完成证明决定互斥释放，另行保留业务失败结果。 + + 合成复现：在 package fixture 注入上述 bootstrap 返回值。请求结束后检查 VM claim 和 `proxy_operation:pool`。预期均可释放；当前路径保留未知。 + +5. **P1：刷新因其他活动执行被拒绝启动后，runner 将已结束作业的准入永久标为未知。** + + 位置：`src/lib/oauth/shared-credential-refresh.mjs:184`；[src/lib/transport/kernel-router.mjs:317](https://github.com/liubing0806/vm2api/blob/068919906653acf0e6dd0f749f83cb808f9f36f1/src/lib/transport/kernel-router.mjs#L317)；`src/lib/pool/failover-runner.mjs:526`、`:1150`。 + + 触发条件：同账号允许至少两个并发请求；A 的模型作业已明确结束并返回 401；B 的准入仍活动。 + + 刷新获取 VM 操作时，`shared-vm-operations.mjs:97` 因 B 返回 `vm_execution_active`。共享刷新服务确认回调未进入，并将任务完成为 `refresh_not_started`，但继续抛出普通异常。 + + kernel router 未保留 A 已结束的结果。异常进入 runner 后被标为执行未知；`recoverCredential` 也会对刷新异常无条件标记未知。随后普通 release 无法释放 A 的准入。 + + 影响：正常并发下的刷新拒绝会持久占用账号容量，并阻止维护。此触发条件发生在正常开放状态，超出已知的 DRAINING 保守边界。 + + 最小修复：传播明确的刷新未启动结果；在原模型作业已结束且刷新未启动时，保留可释放的准入状态。 + + 合成复现：建立 A、B 两条同账号准入，完成 A 的 job，保持 B 活动，再调用 A 的 `recoverCredential`。确认 worker 调用次数为零。释放 B 后检查 A 是否仍有 `uncertain_at`。 + +上轮四项的指定触发路径复核如下，状态仅代表源码证据： + +| 上轮项目 | 状态与证据 | +|---|---| +| usage 超时被重试成功覆盖 | 已修正。`official-cc-bootstrap.mjs:1091` 遇到未确认结束立即停止重试；`:1336` 转为未知；`:1532` 返回统一完成状态；`:1661` 决定任务和 VM 操作保留。 | +| package/import 辅助错误提前释放 | 指定未知路径已修正。原始函数在 `:1351`、`:1363`、`:1401` 汇总辅助未知；package `panel-routes.mjs:890` 和 import `import-commit.mjs:304` 使用统一判定。普通已结束失败另见发现 4。 | +| active endpoints 正常清理变永久 unknown | 指定路径已修正。`egress.mjs:1103` 按 ID 和名称复核网络；`:1117` 保留仍有端点的网络；`:1125` 处理明确的 active endpoints 删除拒绝。helper 结束证明仍单独检查。 | +| watchdog bridge helper 缺 claims 和未知传播 | 已修正。`slot-recovery.mjs:702` 先取得代理池 claim，`:706` 补充绑定 VM，`:716` 取得 VM claims 后执行；`:667` 返回未知。`kernel-watchdog.mjs:227` 据此保留后台任务。 | + +覆盖范围还包括： + +- 共享刷新使用 PG 事务锁及前次 execution ID 条件去重。新准入和新 job 绑定均与账号/VM 操作使用相同资源锁。已结束 job 例外核对了数据库、owner、账号、VM、未释放及无 pending job。 +- 成功发布位于结束事务之前。server 的 Claude 发布回调先写 VM 投影并等待持久化。正常独立刷新路径在任务和 VM 操作结束后才完成 activity。 +- 共享结果写入凭据哈希和受限错误码。本轮检查的 metadata 路径未发现 token 内容泄露。 +- KSG2 客户端按值冻结原 Fence；发送前要求匹配的持久绑定确认；响应后使用原 ticket 查询节点持久终态。PG 拒绝身份不符或缺失完成证明。显式 fenced worker 路径拒绝旧传输回退。未发现额外可证实问题。 + +未执行测试、网络、服务、数据库或运行环境操作。真实 PG 并发、worker 制品行为、Docker 端点和跨 Pod 时序均未动态验证。DRAINING 拒绝暂保守保留未知仍记为已知边界。生产启动权限、管理 grant 和节点发布能力仍属后续范围。 + +`ControlPlaneLock`、`replicas: 1` 和 Recreate 仍保留。本检查点不代表高可用完成，**本报告不批准发布**。 diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-08a18287.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-08a18287.md new file mode 100644 index 000000000..ff9c7e5fd --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-08a18287.md @@ -0,0 +1,58 @@ +新增 **1 项 P1**。未发现其他有证据支持的 P0/P1/P2。 + +审查提交：`08a182870b6334691a114c7b93a0f07032588ff7`。增量基线：`068919906653acf0e6dd0f749f83cb808f9f36f1`。开始和结束时 HEAD 一致，工作区干净，`git diff --check` 无输出。 + +**新增 P1：已明确结束的远端准备失败，被 telemetry touch 路径持久标为未知。** + +位置： + +- [src/lib/cluster/remote-slot-files.mjs:228](https://github.com/liubing0806/vm2api/blob/08a182870b6334691a114c7b93a0f07032588ff7/src/lib/cluster/remote-slot-files.mjs#L228) +- [src/lib/protocol/handle-protocol.mjs:1358](https://github.com/liubing0806/vm2api/blob/08a182870b6334691a114c7b93a0f07032588ff7/src/lib/protocol/handle-protocol.mjs#L1358) +- [src/lib/pool/failover-runner.mjs:1167](https://github.com/liubing0806/vm2api/blob/08a182870b6334691a114c7b93a0f07032588ff7/src/lib/pool/failover-runner.mjs#L1167) + +触发条件：节点连接后的首次目录准备尚未完成缓存；请求在提交模型作业前发布 telemetry touch；远端 `mkdir/chmod/chown` 因权限或磁盘错误正常退出，并已收到非零退出码和通道关闭。 + +可证实调用链： + +1. `touchTelemetrySession` → `pushTelemetryTouch` → `queueSlotPush` → `pushSlotFiles` → `prepareRemoteDir`。 +2. [ssh-link.mjs:167](https://github.com/liubing0806/vm2api/blob/08a182870b6334691a114c7b93a0f07032588ff7/src/lib/cluster/ssh-link.mjs#L167) 保存退出码，收到 `close` 后返回。[remote-fs.mjs:115](https://github.com/liubing0806/vm2api/blob/08a182870b6334691a114c7b93a0f07032588ff7/src/lib/cluster/remote-fs.mjs#L115) 将非零退出转换为普通异常。此时前台命令已经结束，`openSftp` 尚未执行。 +3. queue 对所有异常无条件添加 `vmOperationUncertain` 和 `executionUncertain`。 +4. protocol 再无条件标记执行准入、API key 准入和请求活动未知。runner 在 `callEntered=false` 时仍接受该未知标志并停止。 +5. 普通释放不能清除 `execution_admissions.uncertain_at`；protocol 的释放条件也排除未知 API key 准入。 + +影响：没有模型作业、没有未结束远端命令的普通失败，会持续占用账号和 VM 容量。并发上限为 1 时，后续请求被拒绝。请求活动未知还会阻止全局维护。HTTP 返回或结束不会清除这些 PostgreSQL 状态。 + +最小修复：保留远端命令的明确结束证明，并贯穿 queue 和 protocol。已确认结束的准备失败按普通失败处理；缺少结束证明的发布失败继续保留未知准入。仅修改 queue 不够，protocol 当前仍会无条件重新添加未知状态。 + +交主会话的合成复现，本轮未执行： + +1. 使用隔离 PostgreSQL、合成节点 VM，以及并发上限为 1 的执行准入和 managed API key。 +2. 保留实际 touch、queue、`prepareRemoteDir` 和 runner。合成 SSH 通道依次发出 `exit(1)`、`close`，错误内容使用普通合成字符串。 +3. 断言 SFTP 未打开、模型调用次数为零、远端退出已确认。 +4. 请求结束后检查执行准入、API key 准入和请求活动。当前源码路径会留下未知且未释放的记录；预期普通失败可以释放。另保留断连场景，确认缺少结束证明时仍保留准入。 + +**上一轮五项逐项复核** + +以下“已修正”仅指指定触发路径的静态源码结论。 + +| 项目 | 状态与依据 | +|---|---| +| 1. Go worker 首次刷新未知被后续明确失败覆盖 | **已修正。** `worker/internal/oauth/refresher.go:197` 遇到未知立即停止重试；`:142` 不再用后续文件观察覆盖未知。`oauthcmd/oauthcmd.go:348` 输出未知标志，`src/lib/transport/slot-oauth.mjs:238` 将其排除出完成证明。 | +| 2. package Codex quota 嵌套未知被吞后释放 | **已修正。** `src/lib/vm/shared-vm-operations.mjs:223` 在嵌套返回未知或抛出未知异常时直接标记继承的操作。额度层仍会压平返回值，但外层正常返回不能清除该操作的未知状态。 | +| 3. manual Codex refresh 绕过共享账号声明 | **已修正。** `src/lib/admin/panel-routes.mjs:5262` 取得 VM 操作；`:5283` 调用共享 Codex 刷新；`:5289` 保留未知。嵌套共享刷新会核对并补充账号声明。 | +| 4. package bootstrap 普通已结束失败误标全 pool unknown | **已修正。** `src/lib/admin/panel-routes.mjs:889` 使用 `officialCcBootstrapFinished` 作为执行完成判定，业务 `ok:false` 保留在 `value`，不会再单独触发原来的未知判定。 | +| 5. 同账号另一执行拒绝刷新后，已结束首 job 永久 unknown | **已修正。** `shared-credential-refresh.mjs:198` 确认未启动任务完成后传播标志；`kernel-router.mjs:322` 保留原 job 已结束证明；`failover-runner.mjs:526`、`:1083`、`:1143` 据此停止并允许普通释放。 | + +**已覆盖与未验证范围** + +已读取增量及必要完整调用链，覆盖： + +- private node authority 的 schema、meta、view、feature marker，以及业务 restore 的依赖和 epoch 隔离。 +- `SECURITY DEFINER`、固定 search_path、节点 reader 权限、grant 发行与撤销锁序、Go pgx 授权窗口、默认拒绝管理 RPC。 +- allocation receipt 的校验、持久化、同 intent 重读，以及节点 management binding 的持久检查。 +- SFTP queue 原始 Promise，以及 start、reload、worker、supervisor、protocol、panel、routing 的等待和异常消费。 +- 共享 circuit、KSG2 原始 Fence、持久绑定及节点完成证明；另复核 active endpoints 清理和 watchdog bridge claims。 + +未执行测试、服务、网络、数据库、Docker、Kubernetes、SSH或生产操作。真实并发、断连时序、磁盘持久化和 worker 制品行为均未动态验证。角色安装按测试或管理员显式流程评估；未将尚未启用的生产 grant、初始创建路径或多节点 HA 记为缺陷。 + +`ControlPlaneLock`、`replicas: 1` 和 `Recreate` 仍保留。**本报告不批准发布。** \ No newline at end of file diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-e9076922.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-e9076922.md new file mode 100644 index 000000000..379541427 --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-e9076922.md @@ -0,0 +1,28 @@ +本增量未发现可证的新增 P0/P1/P2。上一轮 P1 在指定的首次 prepare 失败场景下，源码已修正。动态行为未验证，本审查不批准发布。 + +已核对 HEAD 为 `e9076922a4af1f43840e6f9db8381790c809ee21`，增量基线为 `08a182870b6334691a114c7b93a0f07032588ff7`。审查前后工作区均干净。 + +**上一轮 P1 复核** + +- **完成证明范围正确。** `execCollect` 等待 channel close 后返回；`runRemote` 仅为安全整数非零退出码附加 `remoteCommandEnded`。只有首次 `prepareRemoteDir` 失败，且两种 uncertainty 标记均无矛盾时，才附加 `publicationFinished`。后续 SFTP 错误不会经过该赋值处。[ssh-link.mjs:149](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/cluster/ssh-link.mjs#L149)、[remote-fs.mjs:115](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/cluster/remote-fs.mjs#L115)、[remote-slot-files.mjs:95](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/cluster/remote-slot-files.mjs#L95) +- **unknown 不会被完成标记覆盖。** queue 保留矛盾标记；supervisor 等待所有 publication，并优先抛出 unknown 失败；protocol 两处均检查完成标记及 uncertainty。[remote-slot-files.mjs:231](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/cluster/remote-slot-files.mjs#L231)、[supervisor.mjs:825](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/transport/rust-kernel-supervisor.mjs#L825)、[protocol.mjs:1082](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/protocol/handle-protocol.mjs#L1082)、[protocol.mjs:1362](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/protocol/handle-protocol.mjs#L1362) +- **资源释放链符合该场景。** fleet sync 失败发生在 API key 预约之前。touch 的确定准备失败发生在真实 FailoverRunner 调用后端之前，不会标记执行未知，`finally` 会等待释放执行预约。protocol 正常返回错误结果后释放 key；HTTP 外层转换异常为响应，再等待响应及持久化完成请求活动。unknown 分支仍保留对应资源。[runner.mjs:956](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/pool/failover-runner.mjs#L956)、[runner.mjs:1166](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/pool/failover-runner.mjs#L1166)、[runner.mjs:1306](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/pool/failover-runner.mjs#L1306)、[protocol.mjs:485](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/protocol/handle-protocol.mjs#L485)、[server.mjs:1242](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/server.mjs#L1242)、[server.mjs:1452](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/server.mjs#L1452) + +**其他已覆盖项** + +- 内存非法值及 KVM 校验在 Docker I/O 前返回 `vmOperationNotStarted`；Standalone 异常路径不再无条件访问数据库而覆盖原错误。[memory.mjs:180](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/vm/docker-slot-memory.mjs#L180)、[panel-routes.mjs:656](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/admin/panel-routes.mjs#L656) +- schedule 仅忽略 discovery 回调开始前的维护准入拒绝；已 admitted 的错误仍向上传播并记录 failure。[vm-test-schedule.mjs:380](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/src/lib/admin/vm-test-schedule.mjs#L380) +- Codex unit 新增数据库初始化入口、异步等待及 SQL 读回，没有放宽原断言。harness 的 loader 仅在测试子进程加载；策略函数继续引用源码，Docker/egress 使用替身，geo 返回不可用。[harness.mjs:122](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/test/harness.mjs#L122)、[mock-runtime.mjs:1](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/test/fixtures/mock-gateway-runtime.mjs#L1) +- DNS helper 使用独立临时目录，清理按该 fixture 的代理配置和执行收据确认身份。[egress.test.mjs:543](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/test/unit/egress.test.mjs#L543) + +**未验证边界** + +本轮未运行测试或任何服务。六个 publication 场景使用真实 runner 和共享预约，但发布入口、scheduler、后端及活动完成动作仍由 fixture 控制，不能作为整条生产调用链的动态证明。[fixture:25](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/test/fixtures/protocol-publication.mjs#L25)、[fixture:204](https://github.com/liubing0806/vm2api/blob/e9076922a4af1f43840e6f9db8381790c809ee21/test/fixtures/protocol-publication.mjs#L204) + +主会话若补动态证据，应使用临时测试数据库及合成 SSH/SFTP，连接未替换的发布入口，分别验证： + +1. 首次 prepare 非零退出并 close：SFTP、模型调用均为零，预约释放、请求活动完成。 +2. 缺失 close、后续 SFTP 失败或完成标记与 unknown 矛盾：资源保留。 +3. 多节点“确定失败＋unknown”：等待全部结果，聚合结果保持 unknown。 + +`ControlPlaneLock`、`replicas: 1`、`Recreate` 均保留。本结论不覆盖其他 worktree 的 Go publish/new grant，也不证明高可用或生产能力已启用。 \ No newline at end of file diff --git a/docs/plans/evidence/kubernetes/review-progress/verification.json b/docs/plans/evidence/kubernetes/review-progress/verification.json index 33f0ae1cc..efee4335b 100644 --- a/docs/plans/evidence/kubernetes/review-progress/verification.json +++ b/docs/plans/evidence/kubernetes/review-progress/verification.json @@ -229,12 +229,44 @@ ], "delta_review": { "source_sha": "66a580de17670a954638fbfb46fee25740adab78", - "status": "completed_4_P1_reproduction_and_fixes_in_progress", + "status": "4_P1_targeted_fixes_verified_pending_followup", "session_id": "01a15f03-648a-7d92-bf3d-f4d20d4cf03e", "findings": { "P1": 4 }, "raw_report_sha256": "771838ebe137f75df8d9456bfa7590672fe90fa9562dcdda33219f22d0ef99bc" + }, + "third_review": { + "source_sha": "068919906653acf0e6dd0f749f83cb808f9f36f1", + "session_id": "01a15f1a-0084-78a2-917f-2792725bceea", + "status": "completed_5_P1_targeted_fixes_in_progress", + "raw_report_sha256": "d4fef6ad11fcf21bef643d0c3c29ad4d0657246b46c1305c085587482aa370bc", + "findings": [ + { + "finding": "worker refresh retry replaces unknown attempt", + "status": "reproduction_and_fix_in_progress" + }, + { + "finding": "nested VM operation drops unknown", + "fix_sha": "34854048", + "status": "targeted_verified_pending_final_review" + }, + { + "finding": "manual Codex refresh bypass", + "fix_sha": "12088dde", + "status": "targeted_verified_pending_final_review" + }, + { + "finding": "finished package bootstrap business failure retains claims", + "fix_sha": "da7700bd", + "status": "targeted_verified_pending_final_review" + }, + { + "finding": "refresh not-started busy refusal retains finished model admission", + "fix_sha": "2065da50", + "status": "targeted_verified_pending_final_review" + } + ] } }, "additional_fixed_source_checks": [ @@ -285,6 +317,150 @@ "failures": [], "log_sha256": "675fc3b7320b857a06df520380f210a4b72a5ee7f9c487f0f97dcf03e40e0402" } + }, + { + "source_sha": "befd24d1", + "scope": "bridge ownership, unknown results, closed-bootstrap activity and protocol observer", + "result": { + "files": 1, + "totals": { + "tests": 72, + "pass": 72, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "411e871fc29059dc4049ba035dd26d1213d2f641137766ab4d25bc483d1bfc8f" + } + }, + { + "source_sha": "34854048", + "scope": "private PG management authorization, node allocation, nested unknown and immutable operation handles", + "result": { + "files": 1, + "totals": { + "tests": 63, + "pass": 63, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "9361e6fbee5ddbf15cafd73341fccca62165e3a9b4f49d1d1b556e20501623cf" + } + }, + { + "source_sha": "06891990", + "scope": "shared refresh integration", + "result": { + "files": 1, + "totals": { + "tests": 251, + "pass": 251, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "ff422ddfcb7c59557ae7c054f4fbcc51339f40b68a19541dc25d388e3e2d7ac0" + } + }, + { + "source_sha": "2065da50", + "scope": "explicit refresh not-started refusal", + "result": { + "files": 1, + "totals": { + "tests": 166, + "pass": 166, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "1839299616616f9b33ea009df0ee79e415f0c6a2259b626baea8c5518187d69c" + } + }, + { + "source_sha": "d824fa2a", + "scope": "known attached proxy networks", + "result": { + "files": 1, + "totals": { + "tests": 58, + "pass": 58, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "c15c3ba3934d9e96c2638d7c9d73ac080096f72fb0a4e2b5e44ad9d630fafd2a" + } + }, + { + "source_sha": "d2b62208", + "scope": "raw bootstrap execution proof across producers", + "result": { + "files": 1, + "totals": { + "tests": 118, + "pass": 118, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "e096552f69988b162f9de37793202a31bc4826bc6b4ab60be0e89b027e04acdb" + } + }, + { + "source_sha": "c05da6f2", + "scope": "original remote file publication promises", + "result": { + "files": 1, + "totals": { + "tests": 205, + "pass": 203, + "fail": 0, + "cancelled": 0, + "skipped": 2 + }, + "failures": [], + "log_sha256": "e0e447c998c5b6e32eebb032fa0540f9ffc99dd901e6ee96639f304df5c2c406" + } + }, + { + "source_sha": "12088dde", + "scope": "manual refresh ownership and publication", + "result": { + "files": 1, + "totals": { + "tests": 90, + "pass": 90, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "80c41d2583539be28bca47a2916cecc90ded45fae39f5312fda8a7c4ba41c6ce" + } + }, + { + "source_sha": "da7700bd", + "scope": "known ended bootstrap business failure releases claims", + "result": { + "files": 1, + "totals": { + "tests": 95, + "pass": 95, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "a68d5427ffb6a6901c0dfb6495cb8c27136048a780cd2e03c8aabb3ed81f60b6" + } } ], "environment_boundary": "Dedicated task PostgreSQL plus synthetic/mock workers only. No production changes. Node-side and later controller increments have not been deployed to the singleton image.", @@ -293,5 +469,498 @@ "captured node execution binding", "trusted node-owned mutation and permissions", "multi-node active/active and fault takeover acceptance" - ] + ], + "completed_root_resource_cleanup": [ + { + "path": "/tmp/vm2api-648-independent-review-0ffae4b0", + "kind": "completed independent review worktree", + "source_sha": "0ffae4b04c2429f472bef5a42add4a00a1ed7fcc", + "removed": true + }, + { + "path": "/tmp/vm2api-648-independent-review-66a580de", + "kind": "completed independent review worktree", + "source_sha": "66a580de17670a954638fbfb46fee25740adab78", + "removed": true + }, + { + "path": "/tmp/vm2api-648-check-bf7615f8", + "kind": "completed root fixed-source test archive", + "removed": true + }, + { + "path": "/tmp/vm2api-648-check-f4a685b9", + "kind": "completed root fixed-source test archive", + "removed": true + }, + { + "path": "/tmp/vm2api-648-check-0ffae4b0", + "kind": "completed root fixed-source test archive", + "removed": true + }, + { + "path": "/tmp/vm2api-648-check-2923a65f", + "kind": "completed root fixed-source test archive", + "removed": true + }, + { + "path": "/tmp/vm2api-648-check-4e5cc468", + "kind": "completed root fixed-source test archive", + "removed": true + } + ], + "later_full_suite_checkpoints": [ + { + "source_sha": "08a182870b6334691a114c7b93a0f07032588ff7", + "suites": { + "unit": { + "files": 368, + "totals": { + "tests": 4182, + "pass": 4159, + "fail": 13, + "cancelled": 0, + "skipped": 10 + }, + "failures": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "error_sha256": "1afab608e2476a88783807d547c9b239ad4ac831679731a3ed4d44812b0c31d2", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "error_sha256": "7862d1144e6748d63aa6e98dabf5a9fa7d8d1582f7fbfe851d978eb15f6d5c92", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "error_sha256": "81b2ee3a4bb8b1ab4f351bede4398032dc9637c683b82b146630fd6857e94c2e", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "error_sha256": "01732f4307f58f479e9b3d8329d92a93d9c748c58028276022257d3645e77e87", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/codex-model-mapping.test.mjs", + "test": "disabled GPT mapping target returns model_disabled", + "error_sha256": "f7875024408f543ca8854c2a3fd80868c8e8ce86600e0e1b1ff1939131eb3861", + "matches_previous_verified_error": false + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/egress.test.mjs", + "test": "egress config carries dns_upstream only when configured", + "error_sha256": "7b37bbe628b2b4bd13d3914908e2b78d68b45d15a6c436810675f424167ed80b", + "matches_previous_verified_error": false + }, + { + "file": "test/unit/panel-vm-memory-raise.test.mjs", + "test": "PATCH vm memory rejects kvm slots", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_previous_verified_error": false + }, + { + "file": "test/unit/panel-vm-memory-raise.test.mjs", + "test": "PATCH vm memory rejects invalid values without persisting", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_previous_verified_error": false + }, + { + "file": "test/unit/vm-test-chat-codex.test.mjs", + "test": "resolveTestModels refresh merges ChatGPT ids and ignores 401", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_previous_verified_error": false + }, + { + "file": "test/unit/vm-test-chat-codex.test.mjs", + "test": "syncCodexCatalog refreshes OAuth on 401 then merges GPT ids", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_previous_verified_error": false + } + ], + "log_sha256": "d96efb13e0af2a390937493a9ab08d4de83ee6cf3c635ade4a009eac8e7c17fb" + }, + "e2e": { + "files": 31, + "totals": { + "tests": 130, + "pass": 110, + "fail": 19, + "cancelled": 0, + "skipped": 1 + }, + "failures": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "error_sha256": "a25cb59468e4decc45b1d8b3a66b0a4fbd2195af9f498da39aa83e92169825b5", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "error_sha256": "84e06146cf59485c3b46e05070f6a9a184ecb010a57ef8ab1b62a3ebae90c36a", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "error_sha256": "dfd84d60931072a864eb94f48ce13d4223e963ae919898236f1859bedc8d917e", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "error_sha256": "6ea819807eaee98e6a4c38817c9f011fb426e6a5ff499fa441d8f7bb07371461", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "error_sha256": "c23279127cad70a092d0cc7bb6ac400e05aff03e4614d1eaaa40252878c221c2", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "error_sha256": "97ae4156650fa0de2f613aa7753666781545cb2fe7818350a6df904ab696cdbc", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "error_sha256": "a2c6319267f1b752b63d96a41e15b1a5133952e88f0c145bc96dda53fd31cbe5", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "error_sha256": "ba0ac5e90ea008839723c328e8b172228820837ab895e425d2190c1aa5caa708", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/maintenance-backup.e2e.test.mjs", + "test": "Kubernetes scheduled backup uses private claims and leaves no active business task in its snapshot", + "error_sha256": "ca7bd47274e39057429b9c2bce5b439e7fd9dee7ce0a7ef0ade091c316b0f57e", + "matches_previous_verified_error": false + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "error_sha256": "8353154a58e84007d91c2d217c516dc771e359ddb43d527c7db6734ab5cff6ca", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "error_sha256": "085c13ab1e54fb324bce5520b12006fbf7f84b4a69a1c5cf9e09727b5b33da99", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "error_sha256": "0be25feaedaa697c09835d15c8cfdae694de2710330cadfadcbdfa146625a099", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/panel-api.e2e.test.mjs", + "test": "sessionKey import rejected when bound proxy is dead", + "error_sha256": "6247b85e10b2439da25afb216e49541de2a1274efa433f286aa90a60b2624b8c", + "matches_previous_verified_error": false + } + ], + "log_sha256": "c5b560e503ae7a69ee95ef8df51e3ca2fc48810aca1a1c118cc9f65f53613afb" + } + }, + "go_race": { + "pass": 190, + "fail": 0, + "skipped": 1 + }, + "go_vet_exit": 0, + "gofmt_exit": 0, + "biome_files": 733, + "syntax_files": 323, + "scope": "Source archive; no HA or later source validation. New Node failures reproduced and corrected after this SHA." + }, + { + "source_sha": "e9076922a4af1f43840e6f9db8381790c809ee21", + "suites": { + "unit": { + "files": 369, + "totals": { + "tests": 4201, + "pass": 4183, + "fail": 8, + "cancelled": 0, + "skipped": 10 + }, + "failures": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "error_sha256": "1afab608e2476a88783807d547c9b239ad4ac831679731a3ed4d44812b0c31d2", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "error_sha256": "7862d1144e6748d63aa6e98dabf5a9fa7d8d1582f7fbfe851d978eb15f6d5c92", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "error_sha256": "81b2ee3a4bb8b1ab4f351bede4398032dc9637c683b82b146630fd6857e94c2e", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "error_sha256": "01732f4307f58f479e9b3d8329d92a93d9c748c58028276022257d3645e77e87", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_previous_verified_error": true + }, + { + "file": "test/unit/maintenance-monitor-startup.test.mjs", + "test": "DRAINING rejects monitor configuration and VM discovery before any callback or claim", + "error_sha256": "237cd1985976a0becaf5907965d895cf9dbb49c4eb42c95458e151e2445f780e", + "matches_previous_verified_error": false + } + ], + "log_sha256": "8453d2b40223fc584cf4efdc5a3f0245e86e3d32a7c2e93e0efc2bede44731d9" + }, + "e2e": { + "files": 32, + "totals": { + "tests": 131, + "pass": 113, + "fail": 17, + "cancelled": 0, + "skipped": 1 + }, + "failures": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "error_sha256": "a25cb59468e4decc45b1d8b3a66b0a4fbd2195af9f498da39aa83e92169825b5", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "error_sha256": "84e06146cf59485c3b46e05070f6a9a184ecb010a57ef8ab1b62a3ebae90c36a", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "error_sha256": "dfd84d60931072a864eb94f48ce13d4223e963ae919898236f1859bedc8d917e", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "error_sha256": "6ea819807eaee98e6a4c38817c9f011fb426e6a5ff499fa441d8f7bb07371461", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "error_sha256": "c23279127cad70a092d0cc7bb6ac400e05aff03e4614d1eaaa40252878c221c2", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "error_sha256": "97ae4156650fa0de2f613aa7753666781545cb2fe7818350a6df904ab696cdbc", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "error_sha256": "a2c6319267f1b752b63d96a41e15b1a5133952e88f0c145bc96dda53fd31cbe5", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "error_sha256": "ba0ac5e90ea008839723c328e8b172228820837ab895e425d2190c1aa5caa708", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "error_sha256": "8353154a58e84007d91c2d217c516dc771e359ddb43d527c7db6734ab5cff6ca", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "error_sha256": "085c13ab1e54fb324bce5520b12006fbf7f84b4a69a1c5cf9e09727b5b33da99", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "error_sha256": "0be25feaedaa697c09835d15c8cfdae694de2710330cadfadcbdfa146625a099", + "matches_previous_verified_error": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_previous_verified_error": true + } + ], + "log_sha256": "6cb118f0a4880c12427738537f9278c40eef22bb1ffc0d4f87691de1fc727467" + } + }, + "go_race": { + "pass": 190, + "fail": 0, + "skipped": 1 + }, + "go_vet_exit": 0, + "gofmt_exit": 0, + "biome_files": 735, + "syntax_files": 302, + "syntax_scope": "src/**/*.mjs", + "later_correction": { + "source_sha": "60d63596", + "kind": "test expectation only", + "reason": "Discovery refusal before callback is no longer a persistence failure; asserts zero discovery reads, callbacks, jobs and claims.", + "targeted_pass": 8 + }, + "scope": "Static source plus synthetic/isolated tests. No HA deployment or later source validation." + } + ], + "independent_reviews": [ + { + "source_sha": "08a182870b6334691a114c7b93a0f07032588ff7", + "session_id": "01a15f31-9c1b-77e1-bbee-e2941606450d", + "base_sha": "068919906653acf0e6dd0f749f83cb808f9f36f1", + "report": "independent-review-08a18287.md", + "findings": { + "P1": 1 + }, + "boundary": "Read-only static review, no tests or environment access. No release approval." + }, + { + "source_sha": "e9076922a4af1f43840e6f9db8381790c809ee21", + "base_sha": "08a182870b6334691a114c7b93a0f07032588ff7", + "session_id": "01a15f4b-4370-71b1-bf5c-c4630b14ade8", + "report": "independent-review-e9076922.md", + "new_P0_P1_P2": 0, + "boundary": "Read-only static source; no tests, services, production or release approval." + } + ], + "root_cleanup_removed_count": 9 } From ceb410e2e9172469e8db4264f6769b0fa6eaaf65 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:43:07 +0800 Subject: [PATCH 371/473] test: require captured read-only node health and durable credential identity --- worker/internal/slotgate/node_health_test.go | 194 +++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 worker/internal/slotgate/node_health_test.go diff --git a/worker/internal/slotgate/node_health_test.go b/worker/internal/slotgate/node_health_test.go new file mode 100644 index 000000000..d2b518318 --- /dev/null +++ b/worker/internal/slotgate/node_health_test.go @@ -0,0 +1,194 @@ +package slotgate + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +type nodeHealthReply struct { + OK bool `json:"ok"` + Error string `json:"error"` + Fence Fence `json:"fence"` + ManifestRevision string `json:"manifest_revision"` + ManifestDigest string `json:"manifest_digest"` + Health map[string]any `json:"health"` +} + +func capturedHealthRequest(state AuthorityState) map[string]any { + return map[string]any{"op": "health", "vm_id": "v", "fence": state.Fence, "manifest_revision": "1", "manifest_digest": state.InitialManifest.Receipt.ManifestDigest} +} + +func readNodeHealth(t *testing.T, addr string, request map[string]any) nodeHealthReply { + t.Helper() + c, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS13}) + if err != nil { + t.Fatal(err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(5 * time.Second)) + data, _ := json.Marshal(request) + fmt.Fprintf(c, "KSG2 %s\n%s\n", testToken, data) + var reply nodeHealthReply + if err = json.NewDecoder(bufio.NewReader(c)).Decode(&reply); err != nil { + t.Fatal(err) + } + return reply +} + +func activeHealthGate(t *testing.T) (*publicationFixture, string, *Server, AuthorityState) { + t.Helper() + f := publishingFixture(t) + f.publish(t) + state, _ := f.a.State("v") + if _, err := f.l.LaunchManifest(context.Background(), launchIntentForTest(state, "health-launch"), allowNodeCommandForTest); err != nil { + t.Fatal(err) + } + addr, s := publishingGate(t, f) + state, _ = s.authority.State("v") + return f, addr, s, state +} + +func TestNodeHealthUsesCapturedManifestAndFixedKernelReadWithoutExecution(t *testing.T) { + f, addr, s, state := activeHealthGate(t) + token, err := readPublishedInitial(f.root, "v", state.Fence, f.l.policy, *state.InitialManifest, false) + if err != nil { + t.Fatal(err) + } + var calls atomic.Int64 + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.Method != "GET" || r.URL.RequestURI() != "/internal/health" || r.Header.Get("x-internal-token") != token || r.Header.Get("x-kin-internal-token") != token { + t.Error("health did not use the fixed authenticated node route") + } + body, _ := io.ReadAll(r.Body) + if len(body) != 0 { + t.Error("health sent caller body") + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"ok":true,"healthy":true,"ready_slots":1,"version":"synthetic","credential_state":"ready","access_token":"synthetic-secret-do-not-return","nested":{"refresh_token":"synthetic-secret-do-not-return"}}`) + })) + before, _ := s.authority.State("v") + reply := readNodeHealth(t, addr, capturedHealthRequest(state)) + if !reply.OK || reply.Fence != state.Fence || reply.ManifestRevision != "1" || reply.ManifestDigest != state.InitialManifest.Receipt.ManifestDigest { + t.Fatalf("health lacks original captured proof: %+v", reply) + } + credential, ok := reply.Health["credential"].(map[string]any) + if !ok || credential["account_uuid"] != "synthetic-account" || credential["has_access"] != true || credential["has_refresh"] != true { + t.Fatalf("node credential identity missing: %+v", reply.Health) + } + for _, key := range []string{"expires_at", "generation"} { + if _, ok := credential[key]; !ok { + t.Fatalf("missing credential summary %s", key) + } + } + encoded, _ := json.Marshal(reply) + if strings.Contains(string(encoded), "synthetic-secret-do-not-return") || strings.Contains(string(encoded), "synthetic-access") || strings.Contains(string(encoded), token) { + t.Fatal("health returned secret contents") + } + after, _ := s.authority.State("v") + if calls.Load() != 1 || len(after.Commands) != len(before.Commands) { + t.Fatal("read-only health created an execution or repeated a kernel request") + } +} + +func TestNodeInitialPublishRequiresAndPersistsCredentialIdentity(t *testing.T) { + without := []byte(`{"credential":{"type":"oauth","access_token":"synthetic"}}`) + if _, err := initialCredentialBytes(without); err == nil { + t.Fatal("initial publication accepts credentials without authoritative account identity") + } + with := []byte(`{"credential":{"type":"oauth","access_token":"synthetic","account_uuid":"synthetic-account","org_uuid":"synthetic-org","email":"synthetic@example.test"}}`) + data, err := initialCredentialBytes(with) + if err != nil { + t.Fatal(err) + } + var document struct { + OAuth struct { + Account string `json:"accountUuid"` + Org string `json:"orgUuid"` + Email string `json:"email"` + } `json:"claudeAiOauth"` + } + if err = json.Unmarshal(data, &document); err != nil { + t.Fatal(err) + } + if document.OAuth.Account != "synthetic-account" || document.OAuth.Org != "synthetic-org" || document.OAuth.Email != "synthetic@example.test" { + t.Fatal("publication did not use the existing credential identity schema") + } +} + +func TestNodeHealthRejectsUnknownManagementAndRechecksFenceAfterKernelRead(t *testing.T) { + for _, phase := range []string{"unknown-management", "revoked-during-read"} { + t.Run(phase, func(t *testing.T) { + f, addr, s, state := activeHealthGate(t) + var calls atomic.Int64 + if phase == "unknown-management" { + if _, err := s.authority.BeginCommand("v", state.Fence, "unknown-management", CommandMutation); err != nil { + t.Fatal(err) + } + } + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if _, err := s.authority.Revoke("v", state.Fence); err != nil { + t.Error(err) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"ok":true,"healthy":true}`) + })) + reply := readNodeHealth(t, addr, capturedHealthRequest(state)) + if reply.OK { + t.Fatal("unknown/revoked management returned running proof") + } + if phase == "unknown-management" && calls.Load() != 0 { + t.Fatal("unknown management reached kernel") + } + }) + } +} + +func TestNodeHealthRejectsStaleManifestAndCredentialIdentityChange(t *testing.T) { + for _, change := range []string{"fence", "manifest", "identity"} { + t.Run(change, func(t *testing.T) { + f, addr, _, state := activeHealthGate(t) + var calls atomic.Int64 + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"ok":true,"healthy":true}`) + })) + request := capturedHealthRequest(state) + switch change { + case "fence": + other := state.Fence + other.Incarnation = strings.Repeat("a", 32) + request["fence"] = other + case "manifest": + request["manifest_digest"] = strings.Repeat("a", 64) + case "identity": + file := filepath.Join(f.private(), "claude", "credentials.json") + if err := os.Chmod(file, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(`{"type":"oauth","claudeAiOauth":{"accessToken":"synthetic-other","accountUuid":"different-account","kinGeneration":2}}`), 0o600); err != nil { + t.Fatal(err) + } + } + if reply := readNodeHealth(t, addr, request); reply.OK { + t.Fatal("stale proof or changed credential account was accepted") + } + if change != "identity" && calls.Load() != 0 { + t.Fatal("stale fence/manifest reached kernel") + } + }) + } +} From 7ae6c58a38384ccdfd29b4e6305222dd232256aa Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:48:41 +0800 Subject: [PATCH 372/473] feat: read captured node health with durable credential identity --- docs/plans/2026-10-22-node-health.md | 29 ++ .../2026-10-22-node-initial-publication.md | 6 +- worker/internal/slotgate/dial_linux.go | 7 +- worker/internal/slotgate/dial_other.go | 7 +- .../internal/slotgate/docker_launch_test.go | 2 +- worker/internal/slotgate/ksg2.go | 39 +- worker/internal/slotgate/management_intent.go | 13 +- worker/internal/slotgate/node_health.go | 351 ++++++++++++++++++ worker/internal/slotgate/node_health_test.go | 86 +++++ .../internal/slotgate/publish_files_linux.go | 19 + .../internal/slotgate/publish_files_other.go | 4 + worker/internal/slotgate/publish_initial.go | 24 +- .../internal/slotgate/publish_initial_test.go | 10 +- 13 files changed, 573 insertions(+), 24 deletions(-) create mode 100644 docs/plans/2026-10-22-node-health.md create mode 100644 worker/internal/slotgate/node_health.go diff --git a/docs/plans/2026-10-22-node-health.md b/docs/plans/2026-10-22-node-health.md new file mode 100644 index 000000000..113a4c531 --- /dev/null +++ b/docs/plans/2026-10-22-node-health.md @@ -0,0 +1,29 @@ +# 捕获执行身份的只读节点 health + +本批在节点初始发布之后新增只读 KSG2 health。失败规格为 `967711c8`：原节点尚不识别此 RPC；原初始凭据发布允许缺少唯一账号身份。仅使用 fake kernel、fake Engine、临时目录和 Unix/TLS 监听，未接触真实节点、Docker、SSH 或 provider。 + +## 固定请求与公开响应 + +请求固定为 `{op:'health',vm_id,fence,manifest_revision,manifest_digest}`。manifest revision 为规范字符串 `1`，fence 必须是历史 launch receipt 捕获的 ACTIVE Fence，包含原 immutable executor。不能提交 URL、HTTP 路径、headers、身份、凭据、command/ticket 或 management intent。 + +响应为 `{ok:true,fence,manifest_revision,manifest_digest,health}`。节点从不返回最新 state 替换原 Fence，不通过 prepare/forward，不创建 command，不刷新凭据或请求外部 provider。 + +节点在请求前、固定 GET 发送边界和响应后核对原 ACTIVE Fence、manifest 和 pending management。任何 pending/unknown mutation 或 REVOKING 状态均拒绝。GET 的响应读取不持 Authority 全局锁；只在初始小请求写入时校验原状态。固定路径是捕获 incarnation 的 `run/kernel.sock`，HTTP 路由仅 `GET /internal/health`，内部 token 从节点固定文件取得。上下文和 socket deadline 均限制为两秒，不回退到其他路径。 + +kernel 必须返回完整、受限的 HTTP 200 application/json；截断、trailer、重复字段、额外 JSON 和错误字段类型被拒绝。只保留明确的公开布尔值、计数、engine/provider/version 字段。未知字段、嵌套 provider 扩展和 headers 不返回。非空内部 unhealthy_reason 变为固定 `kernel_unhealthy`，避免原始错误描述泄露认证或 endpoint 内容。 + +## 账号身份与凭据摘要 + +已有 payload health 取证 `/tmp/vm2api-648-kernel-busy-capabilities.json` 仅有 credential_state，不提供 account_uuid。本接口不从该状态猜测账号。 + +初始发布现在要求 account_uuid，并允许 org_uuid/email。它们是受可信 grant 批准且纳入摘要的输入。节点按既有凭据 schema 写入，并在私有 manifest 保存身份。health 读取 node-owned 凭据目录中的普通文件，拒绝 symbolic/hard link；当前凭据的 accountUuid/account_uuid 必须与私有 manifest 一致。kernel 若明确报告了不同账号，也被拒绝。 + +公开 credential 固定含 account_uuid、has_access、has_refresh、expires_at、generation，另可含 needs_refresh、org_uuid、email。identity 只含 account_uuid/org_uuid/email。不会返回 access/refresh/API/internal token、原始凭据或单个 secret 文件摘要。 + +generation 保留原始数字文本或受限 opaque string，避免 JavaScript 安全整数边界导致精度丢失。时间按现有 store 的 seconds/milliseconds 规则归一到毫秒文本；无效或溢出的 expiry 被拒绝。needs_refresh 只调用现有纯判断函数，不触发刷新。响应后再次检查凭据 bytes 和 config/token 摘要；读取期间发生轮换或状态变化时，整次观察被拒绝。 + +## 证明边界 + +这是原捕获身份的一次节点观察，不是某个推理 job 的终止证明。ready_slots、正常 health、HTTP 取消或连接关闭不能释放 unknown execution。controller 仍须在提交前用 PG 准入，并等待原 command 的持久完成。 + +尚未提供节点 credential refresh、运行期 config 修改、controller 断连后独立 drain、无 ticket recovery receipt 或完整 HA 故障接管。这些不通过 health 隐式执行。旧 SSH/SFTP/Docker 权限迁移仍未完成，因此不能据此开启生产多副本 HA。 diff --git a/docs/plans/2026-10-22-node-initial-publication.md b/docs/plans/2026-10-22-node-initial-publication.md index 4e73eb182..dda4835cd 100644 --- a/docs/plans/2026-10-22-node-initial-publication.md +++ b/docs/plans/2026-10-22-node-initial-publication.md @@ -25,7 +25,9 @@ publish 固定 `0→1`、`payload_digest=SHA256(payload bytes)`、空 `manifest_ ## 固定内容和私有文件 -payload 只接受 `{credential:{type,access_token?,refresh_token?,api_key?,expires_at?,scopes?}}`。type 为 `oauth`、`setup-token`、`official-setup-token`、`apikey`。未知字段、重复字段、额外 JSON、文件名、路径、mode、UID、URL、env、argv 和原始 settings/config 均拒绝。单 token 上限 16 KiB；scopes 最多 32 项，每项 128 字符。setup-token 不接受 refresh token；API key 不与 OAuth 字段混用。 +payload 只接受 `{credential:{type,account_uuid,org_uuid?,email?,access_token?,refresh_token?,api_key?,expires_at?,scopes?}}`。type 为 `oauth`、`setup-token`、`official-setup-token`、`apikey`。account_uuid 必选，org_uuid/email 为受限公开字符串。未知字段、重复字段、额外 JSON、文件名、路径、mode、UID、URL、env、argv 和原始 settings/config 均拒绝。单 token 上限 16 KiB；scopes 最多 32 项，每项 128 字符。setup-token 不接受 refresh token；API key 不与 OAuth 字段混用。 + +身份元数据来自可信 initial grant 批准的完整 payload，参与 payload/manifest digest，并写入节点私有 manifest。OAuth 按现有 `credential.Store` schema 写 `claudeAiOauth.accountUuid/orgUuid/email`；API key 按该 store 的 document fallback 保存同名元数据。没有唯一账号身份的旧发布输入被拒绝，不能从 Pod 后续 health 参数补充或猜测账号。 节点生成固定 kernel config、256-bit 随机 internal token 和固定 `.credentials.json → credentials.json` link。内部 token 不返回 controller。批准的 image ID、network ID、UID/GID 来自 node-owned DockerLaunchPolicy；RPC 不能改变这些值。 @@ -49,6 +51,6 @@ managed forward 只允许 kernel `POST /internal/v1/messages`，无 query，body 失败规格 `9c6fad09` 先复现未记录目录可以 launch，以及缺独立 publish 授权入口。后续专用测试覆盖完成 receipt、重放、错误 Fence/revision/大小/摘要、截断上传、PG authorizer hook 拒绝、rename 前 revoke、symbolic/hard link、目录冲突、journal fault/restart、不同 VM 并行、manifest-bound launch、node token 覆盖和管理 HTTP 拒绝。 -这些测试不代替配套 PG service 与 Go reader 的双连接组合验证，也不代替现成 payload 的实际启动与推理验收。未提供 health/credential-summary RPC、运行期 credential refresh/config 热更新、新 generation 再发布、native Mac/KVM/Codex executor 或自动未知状态恢复。 +这些测试不代替配套 PG service 与 Go reader 的双连接组合验证，也不代替现成 payload 的实际启动与推理验收。后续只读 health 见同目录 `2026-10-22-node-health.md`。未提供运行期 credential refresh/config 热更新、新 generation 再发布、native Mac/KVM/Codex executor 或自动未知状态恢复。 runtime 的旧 SSH/SFTP/Docker 权限及已建立会话尚未撤销。节点服务、批准 image/network、受限 PG role 和文件权限仍需另行授权的可信初始化。因此本批不能作为生产开启多副本 HA 的条件。 diff --git a/worker/internal/slotgate/dial_linux.go b/worker/internal/slotgate/dial_linux.go index ef46c0922..6d88450a4 100644 --- a/worker/internal/slotgate/dial_linux.go +++ b/worker/internal/slotgate/dial_linux.go @@ -3,6 +3,7 @@ package slotgate import ( + "context" "fmt" "net" "syscall" @@ -15,6 +16,10 @@ const oPath = 0x200000 // in run/, so the final component is opened O_PATH|O_NOFOLLOW, checked to be a // socket, and connected through its /proc fd with no second path lookup. func dialSlotSocket(path string) (net.Conn, error) { + return dialSlotSocketContext(context.Background(), path) +} + +func dialSlotSocketContext(ctx context.Context, path string) (net.Conn, error) { fd, err := syscall.Open(path, oPath|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) if err != nil { return nil, err @@ -27,5 +32,5 @@ func dialSlotSocket(path string) (net.Conn, error) { if st.Mode&syscall.S_IFMT != syscall.S_IFSOCK { return nil, fmt.Errorf("%s is not a socket", path) } - return net.Dial("unix", fmt.Sprintf("/proc/self/fd/%d", fd)) + return (&net.Dialer{}).DialContext(ctx, "unix", fmt.Sprintf("/proc/self/fd/%d", fd)) } diff --git a/worker/internal/slotgate/dial_other.go b/worker/internal/slotgate/dial_other.go index 8f3fe9734..85b0fbc82 100644 --- a/worker/internal/slotgate/dial_other.go +++ b/worker/internal/slotgate/dial_other.go @@ -3,6 +3,7 @@ package slotgate import ( + "context" "fmt" "net" "os" @@ -10,6 +11,10 @@ import ( // Nodes are Linux; this keeps the package testable elsewhere (lstat, then dial). func dialSlotSocket(path string) (net.Conn, error) { + return dialSlotSocketContext(context.Background(), path) +} + +func dialSlotSocketContext(ctx context.Context, path string) (net.Conn, error) { st, err := os.Lstat(path) if err != nil { return nil, err @@ -17,5 +22,5 @@ func dialSlotSocket(path string) (net.Conn, error) { if st.Mode()&os.ModeSocket == 0 { return nil, fmt.Errorf("%s is not a socket", path) } - return net.Dial("unix", path) + return (&net.Dialer{}).DialContext(ctx, "unix", path) } diff --git a/worker/internal/slotgate/docker_launch_test.go b/worker/internal/slotgate/docker_launch_test.go index dfa19e790..a93e0e86a 100644 --- a/worker/internal/slotgate/docker_launch_test.go +++ b/worker/internal/slotgate/docker_launch_test.go @@ -146,7 +146,7 @@ func launchFixtureConfigured(t *testing.T, published bool, network string) (*Aut } } } else { - payload := []byte(`{"credential":{"type":"oauth","access_token":"synthetic-access","refresh_token":"synthetic-refresh"}}`) + payload := []byte(`{"credential":{"type":"oauth","account_uuid":"synthetic-account","access_token":"synthetic-access","refresh_token":"synthetic-refresh"}}`) intent := publicationIntentForTest(allocation, policy, payload) publisher := initialPublisher{authority: a, root: root, nodeID: allocation.NodeID, policy: policy, authorize: allowNodeCommandForTest} if _, err = publisher.Publish(context.Background(), intent, len(payload), strings.NewReader(string(payload))); err != nil { diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index c1d2c7a7f..2dfe30d75 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -28,17 +28,19 @@ var errGateTicketInvalid = errors.New("ticket_invalid") // commands continue with one parsed HTTP/1 request and an HTTP response after // OK\n. KSG1 raw splicing is never accepted when fenced mode is enabled. type gate2Request struct { - Op string `json:"op"` - VMID string `json:"vm_id"` - Fence Fence `json:"fence"` - CommandID string `json:"command_id,omitempty"` - Kind string `json:"kind,omitempty"` - Ticket string `json:"ticket,omitempty"` - AllocationID string `json:"allocation_id,omitempty"` - OperationID string `json:"operation_id,omitempty"` - OwnerID string `json:"owner_id,omitempty"` - Management *ManagementIntent `json:"management,omitempty"` - ContentLength int `json:"content_length,omitempty"` + Op string `json:"op"` + VMID string `json:"vm_id"` + Fence Fence `json:"fence"` + CommandID string `json:"command_id,omitempty"` + Kind string `json:"kind,omitempty"` + Ticket string `json:"ticket,omitempty"` + AllocationID string `json:"allocation_id,omitempty"` + OperationID string `json:"operation_id,omitempty"` + OwnerID string `json:"owner_id,omitempty"` + Management *ManagementIntent `json:"management,omitempty"` + ContentLength int `json:"content_length,omitempty"` + ManifestRevision string `json:"manifest_revision,omitempty"` + ManifestDigest string `json:"manifest_digest,omitempty"` } type gate2State struct { @@ -112,6 +114,10 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. io.WriteString(conn, "ERR bad_request\n") return } + if request.Op != "health" && (request.ManifestRevision != "" || request.ManifestDigest != "") { + io.WriteString(conn, "ERR bad_request\n") + return + } if request.Op == "forward" { s.forwardFenced(ctx, conn, reader, request) return @@ -122,6 +128,15 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply := map[string]any{"ok": false} var state AuthorityState switch request.Op { + case "health": + var health map[string]any + health, err = s.readHealth(ctx, request) + if err == nil { + reply["health"] = health + reply["fence"] = request.Fence + reply["manifest_revision"] = request.ManifestRevision + reply["manifest_digest"] = request.ManifestDigest + } case "allocate": if s.allocator == nil || request.Fence != (Fence{}) || request.CommandID != "" || request.Kind != "" || request.Ticket != "" { err = ErrManagementDenied @@ -218,7 +233,7 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["error"] = gate2Code(err) } else { reply["ok"] = true - if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" { + if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" && request.Op != "health" { reply["state"] = publicGateState(state) } } diff --git a/worker/internal/slotgate/management_intent.go b/worker/internal/slotgate/management_intent.go index 930f279a5..b134c03ce 100644 --- a/worker/internal/slotgate/management_intent.go +++ b/worker/internal/slotgate/management_intent.go @@ -95,9 +95,20 @@ type initialManifest struct { KernelDigest string `json:"kernel_digest"` TokenDigest string `json:"token_digest"` CredentialDigest string `json:"credential_digest"` + Identity credentialIdentity `json:"identity"` +} + +type credentialIdentity struct { + AccountUUID string `json:"account_uuid"` + OrgUUID string `json:"org_uuid,omitempty"` + Email string `json:"email,omitempty"` +} + +func validCredentialIdentity(identity credentialIdentity) bool { + return vmIDRe.MatchString(identity.AccountUUID) && (identity.OrgUUID == "" || vmIDRe.MatchString(identity.OrgUUID)) && publicHealthString(identity.Email) } func (m initialManifest) digest() string { return hashText(strings.Join([]string{"slotgate-manifest-v1", m.Receipt.Intent.digest(), "1", - m.KernelDigest, m.TokenDigest, m.CredentialDigest}, "\x00")) + m.KernelDigest, m.TokenDigest, m.CredentialDigest, m.Identity.AccountUUID, m.Identity.OrgUUID, m.Identity.Email}, "\x00")) } diff --git a/worker/internal/slotgate/node_health.go b/worker/internal/slotgate/node_health.go new file mode 100644 index 000000000..4c6b7107a --- /dev/null +++ b/worker/internal/slotgate/node_health.go @@ -0,0 +1,351 @@ +package slotgate + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "mime" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/dofastted/kin-gateway/worker/internal/credential" +) + +const nodeHealthMax = 32 << 10 + +func publicHealthString(value string) bool { + if len(value) > 512 { + return false + } + for _, c := range value { + if c < 0x20 || c == 0x7f { + return false + } + } + return true +} + +// The caller holds the Authority lock. No state is changed and no latest Fence +// is substituted for the request's original identity. +func (a *Authority) healthStateLocked(request gate2Request) (AuthorityState, error) { + state, err := a.match(request.VMID, request.Fence) + if err != nil { + return AuthorityState{}, err + } + if state.Phase != PhaseActive || state.Allocation == nil || state.InitialManifest == nil || hasPendingMutation(state) { + return AuthorityState{}, ErrAuthorityPhase + } + manifest := state.InitialManifest + if validateInitialManifest(request.VMID, state) != nil || request.ManifestRevision != "1" || request.ManifestRevision != manifest.Receipt.ManifestRevision || request.ManifestDigest != manifest.Receipt.ManifestDigest || + request.Fence.Generation != manifest.Receipt.Intent.Fence.Generation || request.Fence.Incarnation != manifest.Receipt.Intent.Fence.Incarnation { + return AuthorityState{}, ErrFenceMismatch + } + return state, nil +} + +func (a *Authority) withHealthState(request gate2Request, read func(AuthorityState) error) error { + a.mu.Lock() + defer a.mu.Unlock() + state, err := a.healthStateLocked(request) + if err != nil { + return err + } + return read(state) +} + +func (s *Server) readHealth(ctx context.Context, request gate2Request) (map[string]any, error) { + if s.launcher == nil || request.CommandID != "" || request.Kind != "" || request.Ticket != "" || request.AllocationID != "" || request.OperationID != "" || request.OwnerID != "" || request.Management != nil || request.ContentLength != 0 || !validDigest(request.ManifestDigest) { + return nil, ErrManagementDenied + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + var snapshot AuthorityState + if err := s.authority.withHealthState(request, func(state AuthorityState) error { snapshot = state; return nil }); err != nil { + return nil, err + } + if snapshot.InitialManifest.Receipt.Intent.PolicyDigest != s.launcher.policy.digest() { + return nil, ErrManagementDenied + } + token, err := readPublishedInitial(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy, *snapshot.InitialManifest, false) + if err != nil { + return nil, err + } + before, err := readPublishedCredential(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy) + if err != nil { + return nil, err + } + summary, identity, err := nodeCredentialSummary(before, snapshot.InitialManifest.Identity) + if err != nil { + return nil, err + } + socket := filepath.Join(s.cfg.Root, "vms", request.VMID, "incarnations", request.Fence.Incarnation, "run", "kernel.sock") + connection, err := dialSlotSocketContext(ctx, socket) + if err != nil { + return nil, err + } + defer connection.Close() + deadline, _ := ctx.Deadline() + if err = connection.SetDeadline(deadline); err != nil { + return nil, err + } + stop := context.AfterFunc(ctx, func() { connection.Close() }) + defer stop() + read, _ := http.NewRequest(http.MethodGet, "http://kernel/internal/health", nil) + read.Close = true + read.Header.Set("Accept", "application/json") + read.Header.Set("x-internal-token", token) + read.Header.Set("x-kin-internal-token", token) + // Serialize only this fixed, small request at the captured send boundary. + // Response reads and all parsing happen without the Authority global lock. + err = s.authority.withHealthState(request, func(_ AuthorityState) error { + if ctx.Err() != nil { + return ctx.Err() + } + return read.Write(connection) + }) + if err != nil { + return nil, err + } + response, err := http.ReadResponse(bufio.NewReader(connection), read) + if err != nil { + return nil, err + } + defer response.Body.Close() + media, _, err := mime.ParseMediaType(response.Header.Get("Content-Type")) + if err != nil || media != "application/json" || response.StatusCode != http.StatusOK || response.ProtoMajor != 1 || response.ContentLength > nodeHealthMax { + return nil, ErrInvalidAuthority + } + data, err := io.ReadAll(io.LimitReader(response.Body, nodeHealthMax+1)) + if err != nil || len(data) > nodeHealthMax || len(response.Trailer) != 0 { + return nil, ErrInvalidAuthority + } + health, err := publicKernelHealth(data, snapshot.InitialManifest.Identity) + if err != nil { + return nil, err + } + after, err := readPublishedCredential(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy) + if err != nil || !bytes.Equal(before, after) { + return nil, ErrManagementDenied + } + if _, err = readPublishedInitial(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy, *snapshot.InitialManifest, false); err != nil { + return nil, err + } + if err = s.authority.withHealthState(request, func(state AuthorityState) error { + if *state.InitialManifest != *snapshot.InitialManifest { + return ErrFenceMismatch + } + return nil + }); err != nil { + return nil, err + } + health["credential"] = summary + health["identity"] = identity + return health, nil +} + +func publicKernelHealth(data []byte, identity credentialIdentity) (map[string]any, error) { + var source map[string]json.RawMessage + if strictJSON(data, &source) != nil || source == nil { + return nil, ErrInvalidAuthority + } + out := map[string]any{} + for _, name := range []string{"ok", "healthy", "recovering", "classifier_request_context"} { + if raw, ok := source[name]; ok { + var value *bool + if json.Unmarshal(raw, &value) != nil || value == nil { + return nil, ErrInvalidAuthority + } + out[name] = *value + } + } + if _, ok := out["ok"]; !ok { + return nil, ErrInvalidAuthority + } + for _, name := range []string{"ready_slots", "closed_slots", "cli_restarts", "cli_pid"} { + if raw, ok := source[name]; ok { + value, err := strconv.ParseUint(string(raw), 10, 53) + if err != nil || strconv.FormatUint(value, 10) != string(raw) { + return nil, ErrInvalidAuthority + } + out[name] = value + } + } + for _, name := range []string{"engine", "provider", "worker_version", "version", "unhealthy_reason"} { + if raw, ok := source[name]; ok { + var value *string + if json.Unmarshal(raw, &value) != nil { + return nil, ErrInvalidAuthority + } + if value == nil { + out[name] = nil + continue + } + if !publicHealthString(*value) { + return nil, ErrInvalidAuthority + } + // Arbitrary internal error descriptions can contain endpoint or auth + // details. Expose a fixed public reason instead of returning text. + if name == "unhealthy_reason" && *value != "" { + out[name] = "kernel_unhealthy" + } else { + out[name] = *value + } + } + } + for _, name := range []string{"credential", "identity"} { + if raw, ok := source[name]; ok { + var value map[string]json.RawMessage + if json.Unmarshal(raw, &value) != nil || value == nil { + return nil, ErrInvalidAuthority + } + if account, ok := value["account_uuid"]; ok { + var id string + if json.Unmarshal(account, &id) != nil || id != identity.AccountUUID { + return nil, ErrManagementDenied + } + } + } + } + return out, nil +} + +func credentialString(source map[string]json.RawMessage, names ...string) (string, error) { + for _, name := range names { + if raw, ok := source[name]; ok { + var value string + if json.Unmarshal(raw, &value) != nil { + return "", ErrInvalidAuthority + } + return value, nil + } + } + return "", nil +} + +func credentialRaw(source map[string]json.RawMessage, names ...string) json.RawMessage { + for _, name := range names { + if raw, ok := source[name]; ok { + return raw + } + } + return nil +} + +// No float64 conversion: generation remains exact even beyond JS safe integer. +func credentialScalar(raw json.RawMessage) (string, error) { + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return "0", nil + } + if raw[0] == '"' { + var value string + if json.Unmarshal(raw, &value) != nil || value == "" || len(value) > 128 || !publicHealthString(value) { + return "", ErrInvalidAuthority + } + return value, nil + } + value := string(raw) + if len(value) > 128 || value == "" || (len(value) > 1 && value[0] == '0') { + return "", ErrInvalidAuthority + } + for _, c := range value { + if c < '0' || c > '9' { + return "", ErrInvalidAuthority + } + } + return value, nil +} + +func nodeCredentialSummary(data []byte, expected credentialIdentity) (map[string]any, map[string]any, error) { + var document map[string]json.RawMessage + if strictJSON(data, &document) != nil || document == nil { + return nil, nil, ErrInvalidAuthority + } + oauth := document + if raw, ok := document["claudeAiOauth"]; ok { + if json.Unmarshal(raw, &oauth) != nil || oauth == nil { + return nil, nil, ErrInvalidAuthority + } + } + account, err := credentialString(oauth, "accountUuid", "account_uuid") + if err != nil || account != expected.AccountUUID { + return nil, nil, ErrManagementDenied + } + org, err := credentialString(oauth, "orgUuid", "org_uuid") + if err != nil { + return nil, nil, err + } + email, err := credentialString(oauth, "email", "emailAddress", "email_address") + if err != nil { + return nil, nil, err + } + if !validCredentialIdentity(credentialIdentity{account, org, email}) { + return nil, nil, ErrInvalidAuthority + } + typ, err := credentialString(document, "type") + if err != nil { + return nil, nil, err + } + if credential.NormalizeType(typ) == credential.TypeOAuth { + if nested, e := credentialString(oauth, "type"); e != nil { + return nil, nil, e + } else if nested != "" { + typ = nested + } + } + access, err := credentialString(oauth, "accessToken", "access_token") + if err != nil { + return nil, nil, err + } + refresh, err := credentialString(oauth, "refreshToken", "refresh_token") + if err != nil { + return nil, nil, err + } + if raw, ok := document["anthropicApiKey"]; ok { + var api map[string]json.RawMessage + if json.Unmarshal(raw, &api) != nil || api == nil { + return nil, nil, ErrInvalidAuthority + } + access, err = credentialString(api, "apiKey", "api_key") + if err != nil { + return nil, nil, err + } + typ = "apikey" + } + genRaw := credentialRaw(oauth, "kinGeneration", "kin_generation", "_token_version") + if len(genRaw) == 0 { + genRaw = credentialRaw(document, "kinGeneration", "kin_generation", "_token_version") + } + generation, err := credentialScalar(genRaw) + if err != nil { + return nil, nil, err + } + expires, err := credentialScalar(credentialRaw(oauth, "expiresAt", "expires_at")) + if err != nil { + return nil, nil, err + } + expiry, err := strconv.ParseInt(expires, 10, 64) + if err != nil || expiry < 0 { + return nil, nil, ErrInvalidAuthority + } + if expiry > 0 && expiry < 10_000_000_000 { + expiry *= 1000 + expires = strconv.FormatInt(expiry, 10) + } + cred := credential.Credential{Type: typ, AccessToken: access, RefreshToken: refresh, ExpiresAt: expiry} + summary := map[string]any{"has_access": cred.Valid(), "has_refresh": strings.TrimSpace(refresh) != "", "expires_at": expires, "generation": generation, "account_uuid": account, "needs_refresh": cred.NeedsRefresh(time.Now(), 5*time.Minute)} + identity := map[string]any{"account_uuid": account} + if org != "" { + summary["org_uuid"] = org + identity["org_uuid"] = org + } + if email != "" { + summary["email"] = email + identity["email"] = email + } + return summary, identity, nil +} diff --git a/worker/internal/slotgate/node_health_test.go b/worker/internal/slotgate/node_health_test.go index d2b518318..41c2b543a 100644 --- a/worker/internal/slotgate/node_health_test.go +++ b/worker/internal/slotgate/node_health_test.go @@ -192,3 +192,89 @@ func TestNodeHealthRejectsStaleManifestAndCredentialIdentityChange(t *testing.T) }) } } + +func TestNodeHealthRejectsIncompleteInvalidAndContradictoryKernelResponses(t *testing.T) { + for _, kind := range []string{"truncated", "duplicate", "trailing", "counter", "credential-identity", "identity", "trailer", "status"} { + t.Run(kind, func(t *testing.T) { + f, addr, _, state := activeHealthGate(t) + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch kind { + case "truncated": + w.Header().Set("Content-Length", "1000") + fmt.Fprint(w, `{"ok":true}`) + case "duplicate": + fmt.Fprint(w, `{"ok":true,"ok":false}`) + case "trailing": + fmt.Fprint(w, `{"ok":true} {"error":"synthetic"}`) + case "counter": + fmt.Fprint(w, `{"ok":true,"ready_slots":-1}`) + case "credential-identity": + fmt.Fprint(w, `{"ok":true,"credential":{"account_uuid":"different-account"}}`) + case "identity": + fmt.Fprint(w, `{"ok":true,"identity":{"account_uuid":"different-account"}}`) + case "trailer": + w.Header().Set("Trailer", "X-Internal-Error") + fmt.Fprint(w, `{"ok":true}`) + w.Header().Set("X-Internal-Error", "synthetic") + case "status": + w.WriteHeader(502) + fmt.Fprint(w, `{"ok":true}`) + } + })) + if reply := readNodeHealth(t, addr, capturedHealthRequest(state)); reply.OK || reply.Health != nil { + t.Fatalf("invalid response escaped as health proof: %+v", reply) + } + }) + } +} + +func TestNodeHealthCredentialRotationInvalidatesReadAndTokenTamperNeverSends(t *testing.T) { + for _, kind := range []string{"rotation", "token"} { + t.Run(kind, func(t *testing.T) { + f, addr, _, state := activeHealthGate(t) + var calls atomic.Int64 + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if kind == "rotation" { + file := filepath.Join(f.private(), "claude", "credentials.json") + if err := os.Chmod(file, 0o600); err != nil { + t.Error(err) + } + if err := os.WriteFile(file, []byte(`{"type":"oauth","claudeAiOauth":{"accessToken":"synthetic-rotated","accountUuid":"synthetic-account","kinGeneration":2}}`), 0o600); err != nil { + t.Error(err) + } + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"ok":true}`) + })) + if kind == "token" { + if err := os.WriteFile(filepath.Join(f.private(), "config", "internal.token"), []byte(strings.Repeat("d", 64)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + if reply := readNodeHealth(t, addr, capturedHealthRequest(state)); reply.OK { + t.Fatal("credential/token change accepted") + } + if kind == "token" && calls.Load() != 0 { + t.Fatal("tampered token reached kernel") + } + }) + } +} + +func TestNodeHealthCredentialSummaryUsesExistingSchemaAndExactGeneration(t *testing.T) { + data := []byte(`{"type":"oauth","claudeAiOauth":{"accessToken":"synthetic","refreshToken":"synthetic-refresh","accountUuid":"synthetic-account","orgUuid":"synthetic-org","email":"synthetic@example.test","kinGeneration":9007199254740993,"expiresAt":1700000000}}`) + summary, identity, err := nodeCredentialSummary(data, credentialIdentity{AccountUUID: "synthetic-account"}) + if err != nil { + t.Fatal(err) + } + if summary["generation"] != "9007199254740993" || summary["expires_at"] != "1700000000000" || identity["account_uuid"] != "synthetic-account" || identity["org_uuid"] != "synthetic-org" || identity["email"] != "synthetic@example.test" { + t.Fatalf("credential summary changed stored semantics: %+v %+v", summary, identity) + } + for _, invalid := range []string{`{"type":"oauth","claudeAiOauth":{"accessToken":"synthetic"}}`, `{"type":"oauth","claudeAiOauth":{"accessToken":"synthetic","accountUuid":"synthetic-account","kinGeneration":true}}`, `{"type":"oauth","claudeAiOauth":{"accessToken":"synthetic","accountUuid":"synthetic-account","expiresAt":9223372036854775808}}`} { + if _, _, err = nodeCredentialSummary([]byte(invalid), credentialIdentity{AccountUUID: "synthetic-account"}); err == nil { + t.Fatal("invalid credential summary accepted") + } + } +} diff --git a/worker/internal/slotgate/publish_files_linux.go b/worker/internal/slotgate/publish_files_linux.go index 1210caf9b..4e5d0a2dd 100644 --- a/worker/internal/slotgate/publish_files_linux.go +++ b/worker/internal/slotgate/publish_files_linux.go @@ -342,3 +342,22 @@ func readPublishedInitial(root, vm string, fence Fence, policy DockerLaunchPolic defer base.Close() return checkInitialTree(base, policy, manifest, credentials) } + +func readPublishedCredential(root, vm string, fence Fence, policy DockerLaunchPolicy) ([]byte, error) { + parent, err := incarnationParent(root, vm, false) + if err != nil { + return nil, err + } + defer parent.Close() + base, err := openDirectoryAt(parent, fence.Incarnation, false, os.Geteuid()) + if err != nil { + return nil, err + } + defer base.Close() + dir, err := openDirectoryAt(base, "claude", false, policy.UID) + if err != nil { + return nil, err + } + defer dir.Close() + return readInitialFile(dir, "credentials.json", policy.UID, initialPublishMax) +} diff --git a/worker/internal/slotgate/publish_files_other.go b/worker/internal/slotgate/publish_files_other.go index cee190f83..ad7ac2161 100644 --- a/worker/internal/slotgate/publish_files_other.go +++ b/worker/internal/slotgate/publish_files_other.go @@ -14,3 +14,7 @@ func (*initialFileTree) Close() {} func readPublishedInitial(string, string, Fence, DockerLaunchPolicy, initialManifest, bool) (string, error) { return "", errPublishUnsupported } + +func readPublishedCredential(string, string, Fence, DockerLaunchPolicy) ([]byte, error) { + return nil, errPublishUnsupported +} diff --git a/worker/internal/slotgate/publish_initial.go b/worker/internal/slotgate/publish_initial.go index fb6bd3775..d13311eb8 100644 --- a/worker/internal/slotgate/publish_initial.go +++ b/worker/internal/slotgate/publish_initial.go @@ -18,6 +18,9 @@ type initialCredential struct { APIKey string `json:"api_key,omitempty"` ExpiresAt int64 `json:"expires_at,omitempty"` Scopes []string `json:"scopes,omitempty"` + AccountUUID string `json:"account_uuid"` + OrgUUID string `json:"org_uuid,omitempty"` + Email string `json:"email,omitempty"` } type initialPayload struct { Credential initialCredential `json:"credential"` @@ -31,6 +34,9 @@ func initialCredentialBytes(data []byte) ([]byte, error) { return nil, ErrInvalidAuthority } c := payload.Credential + if !validCredentialIdentity(credentialIdentity{c.AccountUUID, c.OrgUUID, c.Email}) { + return nil, ErrInvalidAuthority + } if len(c.AccessToken) > 16<<10 || len(c.RefreshToken) > 16<<10 || len(c.APIKey) > 16<<10 || c.ExpiresAt < 0 || len(c.Scopes) > 32 { return nil, ErrInvalidAuthority } @@ -69,6 +75,17 @@ func initialCredentialBytes(data []byte) ([]byte, error) { default: return nil, ErrInvalidAuthority } + identity := document + if c.Type != "apikey" { + identity = document["claudeAiOauth"].(map[string]any) + } + identity["accountUuid"] = c.AccountUUID + if c.OrgUUID != "" { + identity["orgUuid"] = c.OrgUUID + } + if c.Email != "" { + identity["email"] = c.Email + } return json.Marshal(document) } @@ -201,6 +218,11 @@ func (p *initialPublisher) Publish(ctx context.Context, intent ManagementIntent, return PublicationReceipt{}, err } manifest := initialManifest{Receipt: PublicationReceipt{Intent: intent, IntentDigest: intent.digest(), ManifestRevision: "1"}, KernelDigest: hashBytes(kernel), TokenDigest: hashText(token + "\n"), CredentialDigest: hashBytes(credential)} + var payload initialPayload + if err = json.Unmarshal(data, &payload); err != nil { + return PublicationReceipt{}, ErrInvalidAuthority + } + manifest.Identity = credentialIdentity{payload.Credential.AccountUUID, payload.Credential.OrgUUID, payload.Credential.Email} manifest.Receipt.ManifestDigest = manifest.digest() tree, err := stageInitialFiles(p.root, intent, p.policy, kernel, []byte(token+"\n"), credential, p.fault) if err != nil { @@ -280,7 +302,7 @@ func validateInitialManifest(vm string, state AuthorityState) error { return nil } i := m.Receipt.Intent - if state.Allocation == nil || !validManagementIntent(i) || i.Action != "publish-initial" || i.VMID != vm || i.AllocationID != state.Allocation.AllocationID || i.NodeID != state.Allocation.NodeID || i.Fence != state.Allocation.Fence || + if state.Allocation == nil || !validCredentialIdentity(m.Identity) || !validManagementIntent(i) || i.Action != "publish-initial" || i.VMID != vm || i.AllocationID != state.Allocation.AllocationID || i.NodeID != state.Allocation.NodeID || i.Fence != state.Allocation.Fence || m.Receipt.IntentDigest != i.digest() || m.Receipt.ManifestRevision != "1" || !validDigest(m.KernelDigest) || !validDigest(m.TokenDigest) || !validDigest(m.CredentialDigest) || m.Receipt.ManifestDigest != m.digest() { return ErrInvalidAuthority } diff --git a/worker/internal/slotgate/publish_initial_test.go b/worker/internal/slotgate/publish_initial_test.go index f5e354e41..2ceade03e 100644 --- a/worker/internal/slotgate/publish_initial_test.go +++ b/worker/internal/slotgate/publish_initial_test.go @@ -55,7 +55,7 @@ func publishingFixture(t *testing.T) *publicationFixture { if err := os.RemoveAll(filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation)); err != nil { t.Fatal(err) } - payload := []byte(`{"credential":{"type":"oauth","access_token":"synthetic-access","refresh_token":"synthetic-refresh","scopes":["user:inference"]}}`) + payload := []byte(`{"credential":{"type":"oauth","account_uuid":"synthetic-account","access_token":"synthetic-access","refresh_token":"synthetic-refresh","scopes":["user:inference"]}}`) intent := publicationIntentForTest(*state.Allocation, l.policy, payload) return &publicationFixture{a: a, p: &initialPublisher{authority: a, root: root, nodeID: intent.NodeID, policy: l.policy, authorize: allowNodeCommandForTest}, l: l, engine: engine, state: state, root: root, intent: intent, payload: payload} } @@ -271,10 +271,10 @@ func TestInitialPublishRejectsSymlinksHardlinksUnexpectedFilesAndChangedBytes(t func TestInitialPublishRejectsArbitraryConfigAndMalformedCredentialPayload(t *testing.T) { for _, payload := range []string{ - `{"credential":{"type":"oauth","access_token":"synthetic"},"path":"../../authority.json"}`, - `{"credential":{"type":"oauth","access_token":"synthetic","base_url":"http://unapproved"}}`, - `{"credential":{"type":"oauth","access_token":"synthetic","access_token":"changed"}}`, - `{"credential":{"type":"oauth","access_token":"synthetic"}} {}`, + `{"credential":{"type":"oauth","account_uuid":"synthetic-account","access_token":"synthetic"},"path":"../../authority.json"}`, + `{"credential":{"type":"oauth","account_uuid":"synthetic-account","access_token":"synthetic","base_url":"http://unapproved"}}`, + `{"credential":{"type":"oauth","account_uuid":"synthetic-account","access_token":"synthetic","access_token":"changed"}}`, + `{"credential":{"type":"oauth","account_uuid":"synthetic-account","access_token":"synthetic"}} {}`, `{"credential":{"type":"arbitrary","access_token":"synthetic"}}`, `{"credential":{"type":"official-setup-token","access_token":"synthetic","refresh_token":"unexpected"}}`, } { From f61e5e2b9c2d1ed87b9284ebf59f1590f5279588 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:45:13 +0800 Subject: [PATCH 373/473] test: require durable HA request links across PostgreSQL peers --- test/unit/ha-request-context.test.mjs | 122 ++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 test/unit/ha-request-context.test.mjs diff --git a/test/unit/ha-request-context.test.mjs b/test/unit/ha-request-context.test.mjs new file mode 100644 index 000000000..5c35aad18 --- /dev/null +++ b/test/unit/ha-request-context.test.mjs @@ -0,0 +1,122 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb, getActivityCoordinator, quoteIdentifier, withTransaction } from '../../src/lib/db/database.mjs' +import { coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' +import { SharedSessionTurns } from '../../src/lib/pool/shared-session-turns.mjs' +import { syntheticFence } from '../fixtures/fake-slotgate.mjs' + +async function fixture(callback) { + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const before = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + try { + return await withDatabaseActivities([db, peer], async () => { + const module = await import('../../src/lib/core/ha-request-context.mjs') + const scope = getActivityCoordinator(db).assertActivity() + const activity = { activityId: scope.id, ownerId: getActivityCoordinator(db).ownerId, epoch: scope.epoch } + return callback({ db, peer, activity, ...module }) + }) + } finally { + if (before == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = before + await getActivityCoordinator(peer)?.stop() + await getActivityCoordinator(db)?.stop() + await db.pool.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinationSchemaFor(db.schema))} CASCADE`) + await peer.close() + } +} + +const input = (requestId = 'unrelated-local-id') => ({ accountId: 'fixture-account', vmId: 'vm-fixture', requestId, + accountLimit: 5, vmLimit: 5, fableLimit: 0, rpmLimit: 0 }) +const binding = { protocol: 'KSG2', nodeId: 'fixture-node', vmId: 'vm-fixture', kind: 'kernel', fence: syntheticFence() } + +test('two PG connections see exact activity, key, turn, execution, attempt and original node-job links before submission', async () => { + await fixture(async ({ db, peer, activity, withHaRequestContext, withHaRequestAttempt }) => { + const keys = new ApiKeyStore({ db }) + const key = await keys.create({ name: 'synthetic-recovery-key' }) + await withHaRequestContext({ db, activity, requestId: 'server-request-one' }, async () => { + const keyLease = await keys.reserve(key) + assert.equal(keyLease.ok, true) + const turn = await new SharedSessionTurns(db).enqueue('fixture-session', 'not-a-join-key') + const lease = await new ExecutionAdmissionsRepo(db).tryReserve(input()) + assert.equal(lease.ok, true) + await new RequestAttemptsRepo(db).begin({ requestId: 'not-a-join-key', attemptNo: 1, + vmId: 'vm-fixture', accountId: 'fixture-account', model: 'claude-synthetic' }) + await withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'node-command-one', nodeBinding: binding })) + const request = await peer.prepare('SELECT * FROM ha_request_contexts WHERE id=$1').get('server-request-one') + assert.equal(request.activity_id, activity.activityId) + assert.equal(request.activity_owner_id, activity.ownerId) + assert.equal(String(request.activity_epoch), String(activity.epoch)) + assert.equal(request.accepting_children, true) + for (const [table, id] of [['api_key_admissions', keyLease.reservationId], ['session_turns', turn.id], ['execution_admissions', lease.admissionId]]) { + const row = await peer.prepare(`SELECT ha_request_id FROM ${table} WHERE id=$1`).get(id) + assert.equal(row.ha_request_id, 'server-request-one') + } + const job = await peer.prepare('SELECT * FROM execution_jobs WHERE request_id=$1').get('node-command-one') + assert.equal(job.ha_request_id, 'server-request-one') + assert.equal(job.attempt_no, 1) + assert.equal(job.admission_id, lease.admissionId) + assert.deepEqual(job.node_binding, binding) + const attempt = await peer.prepare('SELECT ha_request_id FROM request_attempts WHERE attempt_no=1').get() + assert.equal(attempt.ha_request_id, 'server-request-one') + }) + const ended = await peer.prepare('SELECT * FROM ha_request_contexts WHERE id=$1').get('server-request-one') + assert.equal(ended.accepting_children, false) + assert.ok(ended.handler_returned_at) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_admissions WHERE released_at IS NULL').get()).count, 1) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM api_key_admissions WHERE released_at IS NULL').get()).count, 1) + }) +}) + +test('a late child waits for the parent SQL lock and is rejected after another connection seals the request', async () => { + await fixture(async ({ db, peer, activity, withHaRequestContext }) => { + await withHaRequestContext({ db, activity, requestId: 'sealed-request' }, async () => { + let resume, locked + const waiting = new Promise((resolve) => { resume = resolve }) + const ready = new Promise((resolve) => { locked = resolve }) + const close = withTransaction(peer, async () => { + await peer.prepare('SELECT id FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get('sealed-request') + locked() + await waiting + await peer.prepare('UPDATE ha_request_contexts SET accepting_children=false WHERE id=$1').run('sealed-request') + }) + await ready + const reserve = new ExecutionAdmissionsRepo(db).tryReserve(input()) + reserve.catch(() => {}) + await new Promise((resolve) => setImmediate(resolve)) + resume() + await close + await assert.rejects(reserve, { code: 'ha_request_context_closed' }) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_admissions').get()).count, 0) + }) + }) +}) + +test('a node job cannot bind an admission from a different HA request or an absent attempt', async () => { + await fixture(async ({ db, peer, activity, withHaRequestContext, withHaRequestAttempt }) => { + let lease + await withHaRequestContext({ db, activity, requestId: 'request-a' }, async () => { + lease = await new ExecutionAdmissionsRepo(db).tryReserve(input()) + await assert.rejects(withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'missing-attempt', nodeBinding: binding })), { code: 'ha_request_attempt_missing' }) + }) + await withHaRequestContext({ db, activity, requestId: 'request-b' }, async () => { + await new RequestAttemptsRepo(db).begin({ requestId: 'request-b', attemptNo: 1, vmId: 'vm-fixture', accountId: 'fixture-account', model: 'claude-synthetic' }) + await assert.rejects(withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'cross-request', nodeBinding: binding })), { code: 'ha_request_context_mismatch' }) + }) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 0) + }) +}) + +test('HA requires its original live activity and context before reserving execution', async () => { + await fixture(async ({ db, peer, activity, withHaRequestContext }) => { + await assert.rejects(withHaRequestContext({ db, activity: { ...activity, activityId: 'other-activity' }, requestId: 'forged' }, async () => {}), { code: 'ha_request_activity_mismatch' }) + await assert.rejects(new ExecutionAdmissionsRepo(db).tryReserve(input()), { code: 'ha_request_context_required' }) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_admissions').get()).count, 0) + }) +}) From 0fa2c0418b0183a41b1520c67a69381f41a35290 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:49:06 +0800 Subject: [PATCH 374/473] test: reproduce restore blocked by later request-context foreign key --- test/unit/backup-added-foreign-key.test.mjs | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/unit/backup-added-foreign-key.test.mjs diff --git a/test/unit/backup-added-foreign-key.test.mjs b/test/unit/backup-added-foreign-key.test.mjs new file mode 100644 index 000000000..cb27973e5 --- /dev/null +++ b/test/unit/backup-added-foreign-key.test.mjs @@ -0,0 +1,30 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { getDb } from '../../src/lib/db/database.mjs' +import { SettingsRepo } from '../../src/lib/db/repos/settings-repo.mjs' +import { BackupService } from '../../src/lib/admin/backup-service.mjs' + +test('an older snapshot restores after a later same-schema parent adds an FK to a baseline table (019 shape)', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-648-restore-fk-')) + const service = new BackupService({ dataDir: path.join(root, 'data'), projectRoot: null, sharedTasks: null }) + t.after(async () => { + await service.stopScheduler() + fs.rmSync(root, { recursive: true, force: true }) + }) + const db = getDb() + const settings = new SettingsRepo(db) + await settings.set('synthetic-restore-marker', 'before') + const backup = await service.createBackup() + // The same dependency shape as 019: a pre-existing request_attempts table + // gains a foreign key to a newer request-context table outside the archive. + await db.exec(`CREATE TABLE fixture_later_request_context(id text PRIMARY KEY); + ALTER TABLE request_attempts ADD COLUMN fixture_request_context text + REFERENCES fixture_later_request_context(id)`) + await settings.set('synthetic-restore-marker', 'after') + assert.equal((await service.restoreBackup(backup.id)).ok, true) + assert.equal(await settings.get('synthetic-restore-marker'), 'before') + assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(`${db.schema}.fixture_later_request_context`)).name, null) +}) From 6a65d3d3cd7ea828e54ac4e42c9f2faa3e6975e0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:50:53 +0800 Subject: [PATCH 375/473] feat: persist exact HA request and execution ownership links --- docs/plans/2026-10-22-ha-request-recovery.md | 97 ++++++++++++ src/lib/core/ha-request-context.mjs | 89 +++++++++++ .../019_ha_request_contexts.sql | 24 +++ src/lib/db/repos/api-key-admissions-repo.mjs | 8 +- .../db/repos/execution-admissions-repo.mjs | 39 ++++- src/lib/db/repos/request-attempts-repo.mjs | 55 ++++--- src/lib/pool/failover-runner.mjs | 29 ++-- src/lib/pool/shared-execution-admissions.mjs | 3 + src/lib/pool/shared-session-turns.mjs | 11 +- src/lib/protocol/handle-protocol.mjs | 3 + test/unit/backup-added-foreign-key.test.mjs | 5 +- test/unit/backup-service.test.mjs | 9 +- test/unit/database.test.mjs | 7 +- test/unit/ha-request-context.test.mjs | 149 ++++++++++++++++-- 14 files changed, 458 insertions(+), 70 deletions(-) create mode 100644 docs/plans/2026-10-22-ha-request-recovery.md create mode 100644 src/lib/core/ha-request-context.mjs create mode 100644 src/lib/db/postgres-migrations/019_ha_request_contexts.sql diff --git a/docs/plans/2026-10-22-ha-request-recovery.md b/docs/plans/2026-10-22-ha-request-recovery.md new file mode 100644 index 000000000..5406d8609 --- /dev/null +++ b/docs/plans/2026-10-22-ha-request-recovery.md @@ -0,0 +1,97 @@ +# HA request correlation and recovery prerequisites + +## Implemented correlation + +This increment only links normal Claude Messages requests in `kubernetes-ha`. +It never recovers, releases or bills an abandoned request. Standalone and singleton +Kubernetes keep nullable links and do not require the new wrapper. + +The server generates a unique request ID independently of a caller's +`x-request-id`, admits the maintenance activity with that ID, and inside +`gate.run` calls: + +```js +await withHaRequestContext({db, requestId, activity: gate}, callback) +// gate exposes {activityId, ownerId, epoch} +``` + +The wrapper verifies the actual admitted activity scope and persists +`ha_request_contexts` before entering AsyncLocalStorage. Protocol logs use this +request ID. No static foreign key references the private coordination schema. +Every new child locks the same open parent before taking its normal locks and +inserting its nullable `ha_request_id`: + +| Child | Exact association | +| --- | --- | +| API key admission | Reservation ID, key/user IDs and original reservation owner; one per HA request. | +| Session turn | Original turn ID, session key, queue sequence and turn owner. | +| Execution admission | Original admission ID, VM/account IDs, seat generation and admission owner. | +| Request attempt | The request's attempt number and selected VM/account/model. | +| Execution job | Original node command ID, admission ID, attempt number and immutable node binding. | + +The runner enters `withHaRequestAttempt(attemptNo, callback)` around the actual +attempt call. Multiple kernel hops retain separate command IDs under that +attempt. Job binding requires a matching parent, an existing attempt for the +same VM/account, and node binding before submission. Similar-looking +`request_id` values are not used to infer any association. + +Parent-row locks precede device/account/VM, session, or key locks. Wrapper return +closes `accepting_children` and records `handler_returned_at`; it does not certify +that remote work ended. Local inherited callbacks and SQL operations racing a +closed parent cannot add children. Existing execution, key/user concurrency, +session and unknown claims remain held. Normal result completion and settlement +still use their existing APIs; no owner-wide release is introduced. + +## Node prerequisites for a later recovery increment + +At the inspected baseline, a failed downstream `response.Write` returns and closes +the kernel socket. The node must instead continue observing the original kernel +response after controller loss, without replaying a request. Only a verified +normal protocol terminal and original EOF may persist completed state. + +A separate read-only command-status RPC must accept the original VM, captured +fence and command ID from PG. It must not require the dead controller's in-memory +ticket or current daemon runtime. Historical completed records remain queryable +after generation advancement. It must never return a fresh fence for relabeling. + +Not-found is not proof of non-execution. A typed seal-not-started operation must +atomically and durably tombstone the original absent or undispatched command ID +under the node authority lock. Late prepare/forward then fails. If dispatch wins +the race, the node retains unknown/pending work or its eventual original terminal +receipt. No timeout or controller-disconnection TTL may clear it. + +Before resolving abandoned children, PG recovery must close the exact parent to +stop new child binding, then enumerate the original committed command set. A new +recovery proof path may resolve uncertainty; the current ordinary release APIs +intentionally cannot clear an `uncertain_at` latch. Requests lacking the new +links, non-node executions and unresolved non-inference effects stay occupied. + +## Minimum terminal receipt for billing + +The node should atomically persist a bounded receipt with the original command +ID and fence, terminal/dispatched state, immutable receipt ID, actual model, +fixed stop reason, and an explicit usage-complete indication. Usage is a whitelist: +uncached input, output, cache-read, cache-creation and 5-minute/1-hour creation +counts, plus web-search count and supported speed/service-tier metadata. JSON and +SSE observation must merge the original protocol's usage without double counting. +Counts must survive JSON without precision loss. No response text, thinking, +tool arguments, headers or credentials are retained in that receipt. + +PG additionally needs the request's billing policy snapshot: requested/resolved +model, group multiplier, cache TTL/requested speed, and the applicable unit prices +and price version/source. A version string without historical prices cannot +reconstruct a previous tariff. Node completion alone does not prove zero usage. + +API-key settlement can reuse its reservation-row idempotency, but must bind the +chosen billable command/usage receipt explicitly. Existing behavior charges the +final request result; recovery must not silently sum every retried hop. Initially, +only an unambiguous single normal completion or an already persisted final-choice +record can be settled automatically. Missing usage, ambiguous multi-hop choice, +deleted-key handling or a failed SQL settlement requires a separate billing +resolution and cannot release reserved request quota as zero spend. + +Finally, exact execution/seat release, session-turn release and completion of the +original private activity need proof-aware, idempotent transitions after all +linked work is resolved. A request completion cannot clear unrelated background +claims. Background producers need their own activity/task/child links; current +owner IDs and partly matching schedule strings are insufficient. diff --git a/src/lib/core/ha-request-context.mjs b/src/lib/core/ha-request-context.mjs new file mode 100644 index 000000000..be065dcf8 --- /dev/null +++ b/src/lib/core/ha-request-context.mjs @@ -0,0 +1,89 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import { getActivityCoordinator, getDb } from '../db/database.mjs' +import { isKubernetesHaMode } from './control-plane-mode.mjs' + +const requests = new AsyncLocalStorage() + +function failure(code) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} + +export function currentHaRequestContext(db = getDb()) { + if (!isKubernetesHaMode()) return null + const context = requests.getStore() + if (!context) throw failure('ha_request_context_required') + if (context.db.schema !== db.schema || context.db.connectionString !== db.connectionString) + throw failure('ha_request_context_mismatch') + context.assertActive() + return context +} + +/** Must run inside the same short transaction as a new child row. */ +export async function lockHaRequestContext(db) { + const context = currentHaRequestContext(db) + if (!context) return null + if (!db.isTransaction) throw failure('ha_request_transaction_required') + const row = await db.prepare('SELECT * FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get(context.requestId) + if ( + !row || + row.activity_id !== context.activityId || + row.activity_owner_id !== context.activityOwnerId || + String(row.activity_epoch) !== String(context.activityEpoch) + ) + throw failure('ha_request_context_mismatch') + if (!row.accepting_children) throw failure('ha_request_context_closed') + return context +} + +/** Correlation only: closing this context never releases a lease or unknown work. */ +export async function withHaRequestContext({ db = getDb(), requestId, activity }, callback) { + if (!isKubernetesHaMode()) return callback() + if (requests.getStore()) throw failure('ha_request_context_nested') + const coordinator = getActivityCoordinator(db) + const scope = coordinator?.assertActivity() + if ( + !scope || + activity?.activityId !== scope.id || + activity?.ownerId !== coordinator.ownerId || + String(activity?.epoch) !== String(scope.epoch) + ) + throw failure('ha_request_activity_mismatch') + if (typeof requestId !== 'string' || !requestId.trim() || requestId.length > 128) + throw failure('ha_request_id_invalid') + const inserted = await db + .prepare(`INSERT INTO ha_request_contexts(id,activity_id,activity_owner_id,activity_epoch) + VALUES($1,$2,$3,$4) ON CONFLICT(id) DO NOTHING RETURNING id`) + .get(requestId, scope.id, coordinator.ownerId, scope.epoch) + if (!inserted) throw failure('ha_request_context_exists') + let active = true + const context = Object.freeze({ + db, + requestId, + activityId: scope.id, + activityOwnerId: coordinator.ownerId, + activityEpoch: scope.epoch, + attemptNo: null, + assertActive: () => { + if (!active) throw failure('ha_request_context_closed') + }, + }) + try { + return await requests.run(context, callback) + } finally { + active = false + const returned = await db + .prepare(`UPDATE ha_request_contexts SET accepting_children=false,handler_returned_at=clock_timestamp() + WHERE id=$1 AND activity_id=$2 AND activity_owner_id=$3 AND activity_epoch=$4 RETURNING id`) + .get(requestId, scope.id, coordinator.ownerId, scope.epoch) + if (!returned) throw failure('ha_request_context_mismatch') + } +} + +export function withHaRequestAttempt(attemptNo, callback) { + if (!isKubernetesHaMode()) return callback() + const context = requests.getStore() + if (!context) throw failure('ha_request_context_required') + context.assertActive() + if (!Number.isSafeInteger(attemptNo) || attemptNo <= 0) throw failure('ha_request_attempt_invalid') + return requests.run(Object.freeze({ ...context, attemptNo }), callback) +} diff --git a/src/lib/db/postgres-migrations/019_ha_request_contexts.sql b/src/lib/db/postgres-migrations/019_ha_request_contexts.sql new file mode 100644 index 000000000..08de36556 --- /dev/null +++ b/src/lib/db/postgres-migrations/019_ha_request_contexts.sql @@ -0,0 +1,24 @@ +CREATE TABLE ha_request_contexts ( + id text COLLATE "C" PRIMARY KEY, + activity_id text COLLATE "C" NOT NULL, + activity_owner_id text COLLATE "C" NOT NULL, + activity_epoch bigint NOT NULL CHECK (activity_epoch > 0), + accepting_children boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + handler_returned_at timestamptz +); + +-- Private coordination tables deliberately are not foreign-key dependencies: +-- a business-schema restore must not replace the maintenance authority. +ALTER TABLE api_key_admissions ADD COLUMN ha_request_id text COLLATE "C" REFERENCES ha_request_contexts(id); +ALTER TABLE session_turns ADD COLUMN ha_request_id text COLLATE "C" REFERENCES ha_request_contexts(id); +ALTER TABLE execution_admissions ADD COLUMN ha_request_id text COLLATE "C" REFERENCES ha_request_contexts(id); +ALTER TABLE request_attempts ADD COLUMN ha_request_id text COLLATE "C" REFERENCES ha_request_contexts(id); +ALTER TABLE execution_jobs ADD COLUMN ha_request_id text COLLATE "C" REFERENCES ha_request_contexts(id); +ALTER TABLE execution_jobs ADD COLUMN attempt_no integer CHECK (attempt_no > 0); + +CREATE UNIQUE INDEX idx_api_key_admissions_ha_request ON api_key_admissions(ha_request_id) WHERE ha_request_id IS NOT NULL; +CREATE INDEX idx_session_turns_ha_request ON session_turns(ha_request_id) WHERE ha_request_id IS NOT NULL; +CREATE INDEX idx_execution_admissions_ha_request ON execution_admissions(ha_request_id) WHERE ha_request_id IS NOT NULL; +CREATE INDEX idx_request_attempts_ha_request ON request_attempts(ha_request_id, attempt_no) WHERE ha_request_id IS NOT NULL; +CREATE INDEX idx_execution_jobs_ha_request ON execution_jobs(ha_request_id, attempt_no) WHERE ha_request_id IS NOT NULL; diff --git a/src/lib/db/repos/api-key-admissions-repo.mjs b/src/lib/db/repos/api-key-admissions-repo.mjs index 58b7982fd..34fbae945 100644 --- a/src/lib/db/repos/api-key-admissions-repo.mjs +++ b/src/lib/db/repos/api-key-admissions-repo.mjs @@ -2,6 +2,7 @@ import crypto from 'node:crypto' import { withTransaction } from '../database.mjs' import { ApiKeysRepo } from './api-keys-repo.mjs' import { UsersRepo } from './users-repo.mjs' +import { lockHaRequestContext } from '../../core/ha-request-context.mjs' /** Shared concurrency, request-quota reservations and the rolling RPM history. */ export class ApiKeyAdmissionsRepo { @@ -13,6 +14,7 @@ export class ApiKeyAdmissionsRepo { async reserve(keyId, ownerId, validate) { return withTransaction(this.db, async () => { + const context = await lockHaRequestContext(this.db) // Every admission locks key then owner. The owner lock serializes // requests made with different keys belonging to the same user. const record = await this.keys.getById(keyId, { forUpdate: true }) @@ -53,9 +55,9 @@ export class ApiKeyAdmissionsRepo { if (!gate.ok) return gate const reservationId = crypto.randomUUID() await this.db - .prepare(`INSERT INTO api_key_admissions (id, key_id, user_id, owner_id, created_at) - VALUES ($1, $2, $3, $4, $5)`) - .run(reservationId, keyId, record.user_id, ownerId, now) + .prepare(`INSERT INTO api_key_admissions (id, key_id, user_id, owner_id, created_at, ha_request_id) + VALUES ($1, $2, $3, $4, $5, $6)`) + .run(reservationId, keyId, record.user_id, ownerId, now, context?.requestId ?? null) return { ok: true, record, reservationId, ownerId } }) } diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 83f015185..27be49ac3 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -4,6 +4,7 @@ import { physicalContainerName, physicalVmAliases } from '../../vm/vm-physical-t import { readWatchdogState } from '../../transport/kernel-watchdog-state.mjs' import { ExecutionCircuitsRepo } from './execution-circuits-repo.mjs' import { normalizeNodeBinding, normalizeNodeCompletion, sameNodeBinding } from '../../core/node-execution-proof.mjs' +import { lockHaRequestContext } from '../../core/ha-request-context.mjs' function invalid(field) { return Object.assign(new Error(`Invalid execution admission ${field}`), { code: 'invalid_execution_admission' }) @@ -108,6 +109,7 @@ export class ExecutionAdmissionsRepo { const rpmLimit = limit(input.rpmLimit, 'rpmLimit') const family = input.family == null ? null : identifier(input.family, 'family') return withTransaction(this.db, async () => { + const context = await lockHaRequestContext(this.db) await this.assertRoutingRevision(input.routingRevision) await this.lockResources({ accountId, vmIds: [vmId] }) const operation = await this.db @@ -150,9 +152,19 @@ export class ExecutionAdmissionsRepo { input.circuit && !input.pinned ? await this.circuits.admit(accountId, admissionId, now, input.circuit) : null if (circuit && !circuit.ok) return circuit const record = await this.db - .prepare(`INSERT INTO execution_admissions (id, owner_id, request_id, account_id, vm_id, model_family, created_at, circuit_revision) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *`) - .get(admissionId, ownerId, requestId, accountId, vmId, family, now, circuit?.revision ?? null) + .prepare(`INSERT INTO execution_admissions (id, owner_id, request_id, account_id, vm_id, model_family, created_at, circuit_revision, ha_request_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *`) + .get( + admissionId, + ownerId, + context?.requestId ?? requestId, + accountId, + vmId, + family, + now, + circuit?.revision ?? null, + context?.requestId ?? null, + ) const handle = { admissionId, ownerId } return { ok: true, @@ -196,6 +208,7 @@ export class ExecutionAdmissionsRepo { async bindExecution(handle, execution) { const nodeBinding = execution?.nodeBinding === undefined ? null : normalizeNodeBinding(execution.nodeBinding) return withTransaction(this.db, async () => { + const context = await lockHaRequestContext(this.db) const before = await this.db .prepare('SELECT account_id, vm_id FROM execution_admissions WHERE id = $1 AND owner_id = $2') .get(handle.admissionId, handle.ownerId) @@ -216,6 +229,16 @@ export class ExecutionAdmissionsRepo { if (typeof execution?.requestId !== 'string' || !execution.requestId.trim()) { throw jobError('invalid_execution_job') } + if ((admission.ha_request_id ?? null) !== (context?.requestId ?? null)) + throw jobError('ha_request_context_mismatch') + if (context) { + if (!nodeBinding || !context.attemptNo) throw jobError('ha_request_attempt_missing') + const attempt = await this.db + .prepare(`SELECT id FROM request_attempts WHERE ha_request_id=$1 + AND attempt_no=$2 AND vm_id=$3 AND account_id=$4`) + .get(context.requestId, context.attemptNo, admission.vm_id, admission.account_id) + if (!attempt) throw jobError('ha_request_attempt_missing') + } if (nodeBinding && nodeBinding.vmId !== admission.vm_id) throw jobError('execution_node_binding_mismatch') const operation = await this.db .prepare(`SELECT id FROM vm_operations WHERE released_at IS NULL @@ -227,9 +250,15 @@ export class ExecutionAdmissionsRepo { .get(handle.admissionId) if (pending) throw jobError('execution_job_pending') const row = await this.db - .prepare(`INSERT INTO execution_jobs (request_id, admission_id, node_binding) VALUES ($1, $2, $3::jsonb) + .prepare(`INSERT INTO execution_jobs (request_id, admission_id, node_binding, ha_request_id, attempt_no) VALUES ($1, $2, $3::jsonb, $4, $5) ON CONFLICT (request_id) DO NOTHING RETURNING request_id`) - .get(execution.requestId, handle.admissionId, nodeBinding ? JSON.stringify(nodeBinding) : null) + .get( + execution.requestId, + handle.admissionId, + nodeBinding ? JSON.stringify(nodeBinding) : null, + context?.requestId ?? null, + context?.attemptNo ?? null, + ) if (!row) throw jobError('execution_job_exists') return { bound: true, diff --git a/src/lib/db/repos/request-attempts-repo.mjs b/src/lib/db/repos/request-attempts-repo.mjs index b704af0eb..2699af258 100644 --- a/src/lib/db/repos/request-attempts-repo.mjs +++ b/src/lib/db/repos/request-attempts-repo.mjs @@ -1,4 +1,5 @@ -import { getDb } from '../database.mjs' +import { getDb, withTransaction } from '../database.mjs' +import { lockHaRequestContext } from '../../core/ha-request-context.mjs' function parse(value, fallback = null) { if (value == null) return fallback @@ -42,8 +43,8 @@ export class RequestAttemptsRepo { request_id, attempt_no, vm_id, account_id, model, selection_reason, started_at, completed_at, upstream_status, error_scope, action, cooldown_until, downstream_committed, terminal_state, usage_json, - wait_ms, ttft_ms, latency_ms - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + wait_ms, ttft_ms, latency_ms, ha_request_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) `) this._complete = db.prepare(` UPDATE request_attempts SET @@ -61,28 +62,32 @@ export class RequestAttemptsRepo { } async begin({ requestId, attemptNo, vmId, accountId, model, selectionReason, waitMs = 0 }) { - const startedAt = new Date().toISOString() - await this._insert.run( - requestId, - attemptNo, - vmId ?? null, - accountId ?? null, - model ?? null, - selectionReason ?? null, - startedAt, - null, - null, - null, - null, - null, - 0, - 'started', - null, - waitMs ?? 0, - null, - null, - ) - return (await this.list(requestId)).find((item) => item.attempt_no === attemptNo) || null + return withTransaction(this.db, async () => { + const context = await lockHaRequestContext(this.db) + const startedAt = new Date().toISOString() + await this._insert.run( + context?.requestId ?? requestId, + attemptNo, + vmId ?? null, + accountId ?? null, + model ?? null, + selectionReason ?? null, + startedAt, + null, + null, + null, + null, + null, + 0, + 'started', + null, + waitMs ?? 0, + null, + null, + context?.requestId ?? null, + ) + return (await this.list(context?.requestId ?? requestId)).find((item) => item.attempt_no === attemptNo) || null + }) } async complete(requestId, attemptNo, result = {}) { diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 0b3dc9af3..bdd7a0117 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -25,6 +25,7 @@ import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { SharedSessionTurns } from './shared-session-turns.mjs' import { getDb } from '../db/database.mjs' import { withRefreshAdmission } from '../oauth/shared-credential-refresh.mjs' +import { withHaRequestAttempt } from '../core/ha-request-context.mjs' // Seat wait for the one-shot repair hop when the rejected hop already spent the deadline. const REPAIR_HOP_WAIT_MS = 30_000 @@ -991,19 +992,21 @@ export class FailoverRunner { if (!bound) return stickyConflict() } callEntered = true - result = await this.withCredentialAdmission(selected, () => - callAttempt({ - candidate: selected, - body, - attemptMeta, - attemptNo, - stream, - deliveryMode: deliveryMode || this.config.delivery_mode, - signal, - onCommit: () => { - committed = true - }, - }), + result = await withHaRequestAttempt(attemptNo, () => + this.withCredentialAdmission(selected, () => + callAttempt({ + candidate: selected, + body, + attemptMeta, + attemptNo, + stream, + deliveryMode: deliveryMode || this.config.delivery_mode, + signal, + onCommit: () => { + committed = true + }, + }), + ), ) executionEnded = result?.execution?.ended === true if (selected.bindExecution && (!executionEnded || result?.executionUncertain)) { diff --git a/src/lib/pool/shared-execution-admissions.mjs b/src/lib/pool/shared-execution-admissions.mjs index 02ebe6c02..0cd66522d 100644 --- a/src/lib/pool/shared-execution-admissions.mjs +++ b/src/lib/pool/shared-execution-admissions.mjs @@ -2,6 +2,7 @@ import crypto from 'node:crypto' import { getDb, withTransaction } from '../db/database.mjs' import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' import { budgetAllows } from './seat-planner.mjs' +import { lockHaRequestContext } from '../core/ha-request-context.mjs' function invalid(field) { return Object.assign(new Error(`Invalid shared execution ${field}`), { code: 'invalid_execution_admission' }) @@ -68,6 +69,8 @@ export class SharedExecutionAdmissions { throw invalid('seatReservePct') if (input.seatHeadroom != null && !Number.isFinite(input.seatHeadroom)) throw invalid('seatHeadroom') const gate = await withTransaction(this.db, async () => { + // Match direct admission's parent -> device/account/VM lock order. + await lockHaRequestContext(this.db) await this.executions.assertRoutingRevision(input.routingRevision) await this.lockDevice(seatKey) const before = await this.readSeat(seatKey) diff --git a/src/lib/pool/shared-session-turns.mjs b/src/lib/pool/shared-session-turns.mjs index cf78196eb..67afe1519 100644 --- a/src/lib/pool/shared-session-turns.mjs +++ b/src/lib/pool/shared-session-turns.mjs @@ -1,5 +1,6 @@ import { randomUUID } from 'node:crypto' import { getDb, withTransaction } from '../db/database.mjs' +import { lockHaRequestContext } from '../core/ha-request-context.mjs' function cancelled() { return Object.assign(new Error('Request was cancelled'), { code: 'request_cancelled' }) @@ -44,24 +45,28 @@ export class SharedSessionTurns { async enqueue(sessionKey, requestId) { const id = randomUUID() return withTransaction(this.db, async () => { + const context = await lockHaRequestContext(this.db) // Allocate sequence under the same lock as activation, so an uncommitted // earlier insert cannot be invisible when a later waiter claims the head. await this.lock(sessionKey) await this.db - .prepare(`INSERT INTO session_turns (id, session_key, owner_id, request_id, state) - VALUES ($1,$2,$3,$4,'waiting')`) - .run(id, sessionKey, this.ownerId, requestId ?? null) + .prepare(`INSERT INTO session_turns (id, session_key, owner_id, request_id, state, ha_request_id) + VALUES ($1,$2,$3,$4,'waiting',$5)`) + .run(id, sessionKey, this.ownerId, context?.requestId ?? requestId ?? null, context?.requestId ?? null) return { id, sessionKey, ownerId: this.ownerId } }) } async tryActivate(handle) { return withTransaction(this.db, async () => { + const context = await lockHaRequestContext(this.db) await this.lock(handle.sessionKey) const head = await this.db .prepare('SELECT * FROM session_turns WHERE session_key = $1 ORDER BY sequence LIMIT 1 FOR UPDATE') .get(handle.sessionKey) if (head?.id !== handle.id || head.owner_id !== handle.ownerId || head.state !== 'waiting') return false + if ((head.ha_request_id ?? null) !== (context?.requestId ?? null)) + throw Object.assign(new Error('HA request context mismatch'), { code: 'ha_request_context_mismatch' }) await this.db .prepare("UPDATE session_turns SET state = 'active', activated_at = clock_timestamp() WHERE id = $1") .run(handle.id) diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 2793b8047..8f8800ef0 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -1,4 +1,6 @@ import { trackPendingWrite } from '../db/pending-writes.mjs' +import { currentHaRequestContext } from '../core/ha-request-context.mjs' +import { isKubernetesHaMode } from '../core/control-plane-mode.mjs' /** * /v1 protocol handler. URLs, auth, and response envelopes stay with the * server wiring; this factory owns convert → pool → Go/Rust hop → client. @@ -546,6 +548,7 @@ export function createHandleProtocol(deps) { async function handleProtocolRequest(req, res, protocol, pathName, admission) { const logCtx = requestLog.start(req, { protocol, pathName }) + if (isKubernetesHaMode()) logCtx.request_id = currentHaRequestContext().requestId res._kinRequestId = logCtx.request_id // Registers its own 'finish' listener first so status/headers are final when finish() runs. requestLog.tapResponse?.(logCtx, res) diff --git a/test/unit/backup-added-foreign-key.test.mjs b/test/unit/backup-added-foreign-key.test.mjs index cb27973e5..c6b65f116 100644 --- a/test/unit/backup-added-foreign-key.test.mjs +++ b/test/unit/backup-added-foreign-key.test.mjs @@ -26,5 +26,8 @@ test('an older snapshot restores after a later same-schema parent adds an FK to await settings.set('synthetic-restore-marker', 'after') assert.equal((await service.restoreBackup(backup.id)).ok, true) assert.equal(await settings.get('synthetic-restore-marker'), 'before') - assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(`${db.schema}.fixture_later_request_context`)).name, null) + assert.equal( + (await db.prepare('SELECT to_regclass($1) AS name').get(`${db.schema}.fixture_later_request_context`)).name, + null, + ) }) diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 381d49eb8..13aa5912a 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -79,7 +79,8 @@ test('restoring the application schema preserves later writes in a neighboring s test('restoring a PG 001 snapshot removes later tables before reapplying current migrations', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; + `ALTER TABLE request_attempts DROP COLUMN ha_request_id; + DROP TABLE ha_request_contexts, panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; @@ -105,6 +106,7 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'execution_circuits', 'execution_seats', 'execution_jobs', + 'ha_request_contexts', 'session_turns', 'vm_operations', 'background_task_runs', @@ -113,13 +115,14 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 18) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 19) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( - `DROP TABLE panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; + `ALTER TABLE request_attempts DROP COLUMN ha_request_id; + DROP TABLE ha_request_contexts, panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index 00fa9ff34..a3ac92221 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 42) + assert.equal(rows.length, 43) for (const name of [ 'settings', 'users', @@ -52,6 +52,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'execution_circuits', 'execution_seats', 'execution_jobs', + 'ha_request_contexts', 'session_turns', 'vm_operations', 'background_task_runs', @@ -117,7 +118,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 18) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 19) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -133,7 +134,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 18) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 19) fs.rmSync(dir, { recursive: true, force: true }) }) diff --git a/test/unit/ha-request-context.test.mjs b/test/unit/ha-request-context.test.mjs index 5c35aad18..fea0aaef8 100644 --- a/test/unit/ha-request-context.test.mjs +++ b/test/unit/ha-request-context.test.mjs @@ -32,9 +32,22 @@ async function fixture(callback) { } } -const input = (requestId = 'unrelated-local-id') => ({ accountId: 'fixture-account', vmId: 'vm-fixture', requestId, - accountLimit: 5, vmLimit: 5, fableLimit: 0, rpmLimit: 0 }) -const binding = { protocol: 'KSG2', nodeId: 'fixture-node', vmId: 'vm-fixture', kind: 'kernel', fence: syntheticFence() } +const input = (requestId = 'unrelated-local-id') => ({ + accountId: 'fixture-account', + vmId: 'vm-fixture', + requestId, + accountLimit: 5, + vmLimit: 5, + fableLimit: 0, + rpmLimit: 0, +}) +const binding = { + protocol: 'KSG2', + nodeId: 'fixture-node', + vmId: 'vm-fixture', + kind: 'kernel', + fence: syntheticFence(), +} test('two PG connections see exact activity, key, turn, execution, attempt and original node-job links before submission', async () => { await fixture(async ({ db, peer, activity, withHaRequestContext, withHaRequestAttempt }) => { @@ -46,15 +59,24 @@ test('two PG connections see exact activity, key, turn, execution, attempt and o const turn = await new SharedSessionTurns(db).enqueue('fixture-session', 'not-a-join-key') const lease = await new ExecutionAdmissionsRepo(db).tryReserve(input()) assert.equal(lease.ok, true) - await new RequestAttemptsRepo(db).begin({ requestId: 'not-a-join-key', attemptNo: 1, - vmId: 'vm-fixture', accountId: 'fixture-account', model: 'claude-synthetic' }) + await new RequestAttemptsRepo(db).begin({ + requestId: 'not-a-join-key', + attemptNo: 1, + vmId: 'vm-fixture', + accountId: 'fixture-account', + model: 'claude-synthetic', + }) await withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'node-command-one', nodeBinding: binding })) const request = await peer.prepare('SELECT * FROM ha_request_contexts WHERE id=$1').get('server-request-one') assert.equal(request.activity_id, activity.activityId) assert.equal(request.activity_owner_id, activity.ownerId) assert.equal(String(request.activity_epoch), String(activity.epoch)) assert.equal(request.accepting_children, true) - for (const [table, id] of [['api_key_admissions', keyLease.reservationId], ['session_turns', turn.id], ['execution_admissions', lease.admissionId]]) { + for (const [table, id] of [ + ['api_key_admissions', keyLease.reservationId], + ['session_turns', turn.id], + ['execution_admissions', lease.admissionId], + ]) { const row = await peer.prepare(`SELECT ha_request_id FROM ${table} WHERE id=$1`).get(id) assert.equal(row.ha_request_id, 'server-request-one') } @@ -69,8 +91,14 @@ test('two PG connections see exact activity, key, turn, execution, attempt and o const ended = await peer.prepare('SELECT * FROM ha_request_contexts WHERE id=$1').get('server-request-one') assert.equal(ended.accepting_children, false) assert.ok(ended.handler_returned_at) - assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_admissions WHERE released_at IS NULL').get()).count, 1) - assert.equal((await peer.prepare('SELECT count(*) AS count FROM api_key_admissions WHERE released_at IS NULL').get()).count, 1) + assert.equal( + (await peer.prepare('SELECT count(*) AS count FROM execution_admissions WHERE released_at IS NULL').get()).count, + 1, + ) + assert.equal( + (await peer.prepare('SELECT count(*) AS count FROM api_key_admissions WHERE released_at IS NULL').get()).count, + 1, + ) }) }) @@ -78,8 +106,12 @@ test('a late child waits for the parent SQL lock and is rejected after another c await fixture(async ({ db, peer, activity, withHaRequestContext }) => { await withHaRequestContext({ db, activity, requestId: 'sealed-request' }, async () => { let resume, locked - const waiting = new Promise((resolve) => { resume = resolve }) - const ready = new Promise((resolve) => { locked = resolve }) + const waiting = new Promise((resolve) => { + resume = resolve + }) + const ready = new Promise((resolve) => { + locked = resolve + }) const close = withTransaction(peer, async () => { await peer.prepare('SELECT id FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get('sealed-request') locked() @@ -103,11 +135,23 @@ test('a node job cannot bind an admission from a different HA request or an abse let lease await withHaRequestContext({ db, activity, requestId: 'request-a' }, async () => { lease = await new ExecutionAdmissionsRepo(db).tryReserve(input()) - await assert.rejects(withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'missing-attempt', nodeBinding: binding })), { code: 'ha_request_attempt_missing' }) + await assert.rejects( + withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'missing-attempt', nodeBinding: binding })), + { code: 'ha_request_attempt_missing' }, + ) }) await withHaRequestContext({ db, activity, requestId: 'request-b' }, async () => { - await new RequestAttemptsRepo(db).begin({ requestId: 'request-b', attemptNo: 1, vmId: 'vm-fixture', accountId: 'fixture-account', model: 'claude-synthetic' }) - await assert.rejects(withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'cross-request', nodeBinding: binding })), { code: 'ha_request_context_mismatch' }) + await new RequestAttemptsRepo(db).begin({ + requestId: 'request-b', + attemptNo: 1, + vmId: 'vm-fixture', + accountId: 'fixture-account', + model: 'claude-synthetic', + }) + await assert.rejects( + withHaRequestAttempt(1, () => lease.bindExecution({ requestId: 'cross-request', nodeBinding: binding })), + { code: 'ha_request_context_mismatch' }, + ) }) assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_jobs').get()).count, 0) }) @@ -115,8 +159,85 @@ test('a node job cannot bind an admission from a different HA request or an abse test('HA requires its original live activity and context before reserving execution', async () => { await fixture(async ({ db, peer, activity, withHaRequestContext }) => { - await assert.rejects(withHaRequestContext({ db, activity: { ...activity, activityId: 'other-activity' }, requestId: 'forged' }, async () => {}), { code: 'ha_request_activity_mismatch' }) + await assert.rejects( + withHaRequestContext( + { db, activity: { ...activity, activityId: 'other-activity' }, requestId: 'forged' }, + async () => {}, + ), + { code: 'ha_request_activity_mismatch' }, + ) await assert.rejects(new ExecutionAdmissionsRepo(db).tryReserve(input()), { code: 'ha_request_context_required' }) assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_admissions').get()).count, 0) }) }) + +test('sealing the SQL parent rejects late key reservations, turns and attempts without new child rows', async () => { + await fixture(async ({ db, peer, activity, withHaRequestContext }) => { + const keys = new ApiKeyStore({ db }) + const key = await keys.create({ name: 'synthetic-sealed-key' }) + await withHaRequestContext({ db, activity, requestId: 'sealed-all-children' }, async () => { + await peer + .prepare('UPDATE ha_request_contexts SET accepting_children=false WHERE id=$1') + .run('sealed-all-children') + await assert.rejects(keys.reserve(key), { code: 'ha_request_context_closed' }) + await assert.rejects(new SharedSessionTurns(db).enqueue('late-session', 'late-request'), { + code: 'ha_request_context_closed', + }) + await assert.rejects(new RequestAttemptsRepo(db).begin({ requestId: 'late-request', attemptNo: 1 }), { + code: 'ha_request_context_closed', + }) + for (const table of ['api_key_admissions', 'session_turns', 'request_attempts']) + assert.equal((await peer.prepare(`SELECT count(*) AS count FROM ${table}`).get()).count, 0) + }) + }) +}) + +test('a waiting session turn cannot activate after another connection seals its request', async () => { + await fixture(async ({ db, peer, activity, withHaRequestContext }) => { + await withHaRequestContext({ db, activity, requestId: 'sealed-turn' }, async () => { + const turns = new SharedSessionTurns(db) + const handle = await turns.enqueue('sealed-session', 'untrusted-correlation') + await peer.prepare('UPDATE ha_request_contexts SET accepting_children=false WHERE id=$1').run('sealed-turn') + await assert.rejects(turns.tryActivate(handle), { code: 'ha_request_context_closed' }) + assert.equal((await peer.prepare('SELECT state FROM session_turns WHERE id=$1').get(handle.id)).state, 'waiting') + }) + }) +}) + +test('a callback retained past its request wrapper cannot create another execution', async () => { + await fixture(async ({ db, peer, activity, withHaRequestContext }) => { + let resume + const blocked = new Promise((resolve) => { + resume = resolve + }) + let pending + await withHaRequestContext({ db, activity, requestId: 'returned-request' }, async () => { + pending = blocked.then(() => new ExecutionAdmissionsRepo(db).tryReserve(input())) + pending.catch(() => {}) + }) + resume() + await assert.rejects(pending, { code: 'ha_request_context_closed' }) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM execution_admissions').get()).count, 0) + }) +}) + +for (const mode of ['standalone', 'kubernetes']) { + test(`${mode} keeps nullable request links and does not require an HA wrapper`, async () => { + await fixture(async ({ db, peer, withHaRequestContext }) => { + process.env.VM2API_CONTROL_PLANE_MODE = mode + try { + await withHaRequestContext({ db, requestId: 'unused-legacy', activity: null }, async () => { + const lease = await new ExecutionAdmissionsRepo(db).tryReserve(input()) + await lease.bindExecution({ requestId: 'legacy-job' }) + await new SharedSessionTurns(db).enqueue('legacy-session', 'legacy-request') + await new RequestAttemptsRepo(db).begin({ requestId: 'legacy-request', attemptNo: 1 }) + assert.equal((await peer.prepare('SELECT count(*) AS count FROM ha_request_contexts').get()).count, 0) + for (const table of ['execution_admissions', 'execution_jobs', 'session_turns', 'request_attempts']) + assert.equal((await peer.prepare(`SELECT ha_request_id FROM ${table}`).get()).ha_request_id, null) + }) + } finally { + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + } + }) + }) +} From a62382f3ffed2170769d3c0a179694969d429757 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:54:22 +0800 Subject: [PATCH 376/473] test: reject unfenced protocol backends in explicit HA mode --- test/fixtures/ha-protocol-boundaries.mjs | 89 +++++++++++++++++++++++ test/unit/ha-protocol-boundaries.test.mjs | 15 ++++ 2 files changed, 104 insertions(+) create mode 100644 test/fixtures/ha-protocol-boundaries.mjs create mode 100644 test/unit/ha-protocol-boundaries.test.mjs diff --git a/test/fixtures/ha-protocol-boundaries.mjs b/test/fixtures/ha-protocol-boundaries.mjs new file mode 100644 index 000000000..427aa43d1 --- /dev/null +++ b/test/fixtures/ha-protocol-boundaries.mjs @@ -0,0 +1,89 @@ +import { test, mock } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Readable } from 'node:stream' +import { EventEmitter } from 'node:events' +import { readBody } from '../../src/lib/http/respond.mjs' + +const kind = process.env.VM2API_HA_PROTOCOL_CASE +const events = [] +const supervisor = await import('../../src/lib/transport/rust-kernel-supervisor.mjs') +const telemetry = await import('../../src/lib/vm/worker-telemetry.mjs') +const identity = await import('../../src/lib/identity/vm-identity.mjs') +const context = await import('../../src/lib/core/ha-request-context.mjs') +mock.module(new URL('../../src/lib/core/ha-request-context.mjs', import.meta.url), { + namedExports: { ...context, currentHaRequestContext: () => ({ requestId: 'server-owned-ha-request' }) }, +}) +mock.module(new URL('../../src/lib/transport/rust-kernel-supervisor.mjs', import.meta.url), { + namedExports: { ...supervisor, syncClaudeKernelConfigsFromFile() { events.push('sync'); if (kind !== 'touch') throw new Error('Pod publication forbidden'); return {} } }, +}) +mock.module(new URL('../../src/lib/vm/worker-telemetry.mjs', import.meta.url), { + namedExports: { ...telemetry, touchTelemetrySession() { events.push('touch'); throw new Error('Pod telemetry publication forbidden') } }, +}) +mock.module(new URL('../../src/lib/identity/vm-identity.mjs', import.meta.url), { + namedExports: { ...identity, loadVmIdentity: () => ({ accountUuid: 'ha-account', fingerprint: { device_id: 'synthetic-ha-device' } }) }, +}) +mock.module(new URL('../../src/lib/protocol/handle-codex.mjs', import.meta.url), { + namedExports: { handleCodexProtocol: () => { events.push('codex'); return {} }, handleCodexSearch: () => { events.push('search'); return {} }, usageFromSseLine: () => null }, +}) +if (kind === 'rewrite-gpt') mock.module(new URL('../../src/lib/protocol/codex-model-mapping.mjs', import.meta.url), { + namedExports: { applyCodexInboundModelRewrite: (body) => ({ ok: true, mapped: true, body: { ...body, model: 'gpt-5.4' } }) }, +}) +const { createHandleProtocol } = await import('../../src/lib/protocol/handle-protocol.mjs') + +test(`HA protocol boundary ${kind}`, async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-648-ha-protocol-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const routing = { compatibility: { claude_code_only: false } } + const routingConfigPath = path.join(root, 'routing.json') + fs.writeFileSync(routingConfigPath, JSON.stringify(routing)) + let prepared + const handler = createHandleProtocol({ + cfg: { limits: { max_body_bytes: 1 << 20 }, rewrite: { enabled: false }, intercept: { rules: [] }, distill: { enabled: false }, paths: { project: root, data: root } }, + json(res, status, body) { res.statusCode = status; res.body = body }, readBody, + requireAuth(req) { + req.apiKeyKind = kind === 'api' ? 'managed' : 'master' + if (kind === 'api') req.apiKeyRecord = { id: 'fixture-key', category: 'api', group_type: 'all' } + return true + }, + requestLog: { start: () => ({ request_id: 'untrusted-client-id', t0: Date.now() }) }, + stats: { errors: 0, requests: 0, by_route: {} }, groupsRepo: { rateMultiplier: () => 1 }, + apiKeyStore: { reserve: async () => { events.push('reserve'); throw new Error('unsupported backend reservation forbidden') } }, + stickyRouter: { resolve: () => null }, accountQuota: { ingestHeaders() {} }, + routingConfig: routing, routingConfigPath, + failoverRunner: { async run(args) { + events.push('runner') + prepared = await args.applyAttempt(structuredClone(args.canonicalBody), { + vmId: 'ha-vm', accountId: 'ha-account', vm: { id: 'ha-vm', claude: { has_access: true } }, exec: { vmId: 'ha-vm', vm: {} }, + }, { attemptNo: 1, attemptStartedAt: Date.now() }) + return { ok: true, status: 200, finalState: 'verified', vmId: 'ha-vm', body: { id: 'ha-message', type: 'message', role: 'assistant', model: 'claude-sonnet-4-6', content: [{ type: 'text', text: 'synthetic' }], stop_reason: 'end_turn', usage: { input_tokens: 1, output_tokens: 1 } } } + } }, + }) + let body = { model: kind === 'gpt' ? 'gpt-5.4' : 'claude-sonnet-4-6', max_tokens: 20, messages: [{ role: 'user', content: 'synthetic static prompt' }] } + let protocol = 'anthropic.messages' + if (kind === 'chat') protocol = 'openai.chat' + if (kind === 'responses') { protocol = 'openai.responses'; body = { model: 'claude-sonnet-4-6', input: 'synthetic static prompt' } } + if (kind === 'completions') { protocol = 'openai.completions'; body = { model: 'claude-sonnet-4-6', prompt: 'synthetic static prompt', max_tokens: 20 } } + const req = Readable.from([Buffer.from(JSON.stringify(body))]) + req.headers = { 'user-agent': 'synthetic-ha-client', 'x-request-id': 'untrusted-client-id' } + const res = new EventEmitter() + res.setHeader = () => {} + if (kind === 'search') await handler.handleSearch(req, res, '/v1/alpha/search') + else await handler.handleProtocol(req, res, protocol, '/synthetic-messages') + if (kind === 'responses') { + assert.equal(res.statusCode, 400) + assert.equal(res.body.error.code, 'protocol_not_allowed') + assert.deepEqual(events, []) + } else if (['gpt', 'api', 'search', 'rewrite-gpt'].includes(kind)) { + assert.equal(res.statusCode, 503) + assert.equal(res.body.error.code, 'ha_operation_unsupported') + assert.deepEqual(events, []) + } else { + assert.equal(res.statusCode, 200, JSON.stringify(res.body)) + assert.equal(res._kinRequestId, 'server-owned-ha-request') + assert.ok(prepared?.body) + assert.deepEqual(events, ['runner']) + } +}) diff --git a/test/unit/ha-protocol-boundaries.test.mjs b/test/unit/ha-protocol-boundaries.test.mjs new file mode 100644 index 000000000..ae68d8c8b --- /dev/null +++ b/test/unit/ha-protocol-boundaries.test.mjs @@ -0,0 +1,15 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const fixture = fileURLToPath(new URL('../fixtures/ha-protocol-boundaries.mjs', import.meta.url)) +for (const kind of ['claude', 'touch', 'chat', 'responses', 'completions', 'gpt', 'api', 'search', 'rewrite-gpt']) { + test(`HA refuses legacy effects and unsupported backend: ${kind}`, () => { + const env = { ...process.env, VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', VM2API_HA_PROTOCOL_CASE: kind } + delete env.NODE_TEST_CONTEXT + const result = spawnSync(process.execPath, ['--experimental-test-module-mocks', '--test', fixture], { env, encoding: 'utf8', timeout: 15000 }) + assert.equal(result.status, 0, result.stdout + result.stderr) + assert.match(result.stdout, /# tests 1\b/) + }) +} From ff02b88d670996b79072b6a2ecceda4fd8424110 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:55:16 +0800 Subject: [PATCH 377/473] fix: fence HA protocol backends and skip Pod publication --- src/lib/protocol/handle-protocol.mjs | 32 +++++++++++++++++++++-- test/unit/ha-protocol-boundaries.test.mjs | 6 ++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 8f8800ef0..416ea0608 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -181,6 +181,17 @@ export function createHandleProtocol(deps) { const accountQuota = deps.accountQuota const apiKeyStore = deps.apiKeyStore const apiScheduler = deps.apiScheduler + const rejectHaOperation = (res) => + json( + res, + 503, + makeError({ + type: ErrorType.INVALID_REQUEST, + code: 'ha_operation_unsupported', + message: '该操作暂不支持多副本部署', + status: 503, + }).body, + ) const apiEndpointStore = deps.apiEndpointStore const stats = deps.stats const routingConfigPath = deps.routingConfigPath @@ -638,6 +649,14 @@ export function createHandleProtocol(deps) { logBag.session_id = sessionIdForLog(extractCallerSession({ inbound, headers: req.headers })) logBag.reasoning_effort = reasoningEffortOf(inbound) + if ( + isKubernetesHaMode() && + (!['anthropic.messages', 'openai.chat', 'openai.completions', 'openai.responses'].includes(protocol) || + resolveInferenceBackend(req) === 'api' || + detectInboundPlatform(inbound?.model).platform === 'openai') + ) + return rejectHaOperation(res) + const fp = fingerprintRequest(req, inbound) // Before the health probe: its synthesized or fail-closed replies would otherwise answer // probe-shaped third-party traffic, and with the switch on no probe is answered at all. @@ -752,6 +771,9 @@ export function createHandleProtocol(deps) { logBag.upstream_model = modelRewrite.upstreamModel || null } const platform = detectInboundPlatform(ctx.body?.model) + // A model mapping must not redirect an otherwise supported Claude request + // into the unfenced Codex backend after the initial capability check. + if (isKubernetesHaMode() && platform.ok && platform.platform !== 'anthropic') return rejectHaOperation(res) if (!platform.ok) { stats.errors++ const errorResult = makeError({ @@ -1083,7 +1105,8 @@ export function createHandleProtocol(deps) { const cacheBreakpoints = cacheBreakpointsFromRoutingFile(routingConfigPath) const openaiCompat = String(protocol || '').startsWith('openai.') try { - await syncClaudeKernelConfigsFromFile(cfg.paths?.project, routingConfigPath)?.publication + if (!isKubernetesHaMode()) + await syncClaudeKernelConfigsFromFile(cfg.paths?.project, routingConfigPath)?.publication } catch (error) { if ( error?.publicationFinished === true && @@ -1363,7 +1386,8 @@ export function createHandleProtocol(deps) { } } try { - await touchTelemetrySession(cfg.paths.project, selected.vmId, selected.exec?.vm)?.publication + if (!isKubernetesHaMode()) + await touchTelemetrySession(cfg.paths.project, selected.vmId, selected.exec?.vm)?.publication } catch (error) { if ( error?.publicationFinished === true && @@ -1857,6 +1881,10 @@ export function createHandleProtocol(deps) { /** POST /v1/alpha/search (Codex web search): same logging, auth and body handling as handleProtocol. */ async function handleSearch(req, res, pathName) { + if (isKubernetesHaMode()) { + if (!(await requireAuth(req, res))) return + return rejectHaOperation(res) + } const protocol = 'openai.search' const logCtx = requestLog.start(req, { protocol, pathName }) res._kinRequestId = logCtx.request_id diff --git a/test/unit/ha-protocol-boundaries.test.mjs b/test/unit/ha-protocol-boundaries.test.mjs index ae68d8c8b..f73e3ab56 100644 --- a/test/unit/ha-protocol-boundaries.test.mjs +++ b/test/unit/ha-protocol-boundaries.test.mjs @@ -8,7 +8,11 @@ for (const kind of ['claude', 'touch', 'chat', 'responses', 'completions', 'gpt' test(`HA refuses legacy effects and unsupported backend: ${kind}`, () => { const env = { ...process.env, VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', VM2API_HA_PROTOCOL_CASE: kind } delete env.NODE_TEST_CONTEXT - const result = spawnSync(process.execPath, ['--experimental-test-module-mocks', '--test', fixture], { env, encoding: 'utf8', timeout: 15000 }) + const result = spawnSync(process.execPath, ['--experimental-test-module-mocks', '--test', fixture], { + env, + encoding: 'utf8', + timeout: 15000, + }) assert.equal(result.status, 0, result.stdout + result.stderr) assert.match(result.stdout, /# tests 1\b/) }) From d12f5ea8bd6b778ae37ac3d161f9abcff49c0012 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:57:25 +0800 Subject: [PATCH 378/473] test: require captured node execution in HA scheduling --- test/unit/ha-pool-capture.test.mjs | 60 ++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 test/unit/ha-pool-capture.test.mjs diff --git a/test/unit/ha-pool-capture.test.mjs b/test/unit/ha-pool-capture.test.mjs new file mode 100644 index 000000000..ad4c914fd --- /dev/null +++ b/test/unit/ha-pool-capture.test.mjs @@ -0,0 +1,60 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { PoolScheduler } from '../../src/lib/pool/pool-scheduler.mjs' +import { HaVmCatalog, bindHaVmCatalog } from '../../src/lib/vm/ha-vm-catalog.mjs' + +for (const available of [true, false]) { + test(`HA scheduling ${available ? 'reuses one captured execution' : 'refuses missing node proof'} without Pod credential or worker files`, async (t) => { + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + t.after(() => { + bindHaVmCatalog(null) + if (previous === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + }) + const db = getDb() + const repo = new VmsRepo(db) + await repo.upsertFromVmJson({ + id: 'ha-pool-vm', node_id: 'ha-node', status: 'running', schedulable: true, + claude: { account_uuid: 'ha-pool-account', has_access: true, has_refresh: true, expires_at: Date.now() + 3600_000 }, + }) + const catalog = new HaVmCatalog(db) + bindHaVmCatalog(catalog) + let captures = 0 + let legacyHealth = 0 + const capability = Object.freeze({ + slotgate: Object.freeze({ trusted: true, pgBindingReady: true, vmId: 'ha-pool-vm', nodeId: 'ha-node', + fence: { generation: '1', incarnation: 'captured-incarnation' }, manifest: { revision: '1', digest: 'a'.repeat(64) } }), + nodeManifest: Object.freeze({ revision: '1', digest: 'a'.repeat(64) }), + workerStatus: Object.freeze({ ok: true, healthy: true, ready_slots: 1, + credential: { has_access: true, has_refresh: true, account_uuid: 'ha-pool-account', generation: 1 } }), + }) + const scheduler = new PoolScheduler({ + projectRoot: '/nonexistent-ha-pod-project', + captureNodeExecution: async ({ vmId, accountId, nodeId }) => { + captures++ + assert.deepEqual([vmId, accountId, nodeId], ['ha-pool-vm', 'ha-pool-account', 'ha-node']) + if (!available) throw Object.assign(new Error('synthetic node unavailable'), { code: 'node_capability_unavailable' }) + return capability + }, + workerHealth: async () => { legacyHealth++; throw new Error('legacy worker health must not run') }, + executionAdmissions: { + inspectCircuit: async () => ({ open: false }), + getCounts: async () => ({ accountInflight: 0, vmInflight: 0, fableInflight: 0, rpmCount: 0 }), + }, + }) + const candidates = await scheduler.eligibleCandidates({ model: 'claude-haiku-4-5' }) + assert.equal(captures, 1) + assert.equal(legacyHealth, 0) + assert.equal(candidates.length, available ? 1 : 0) + if (available) { + assert.equal(candidates[0].exec.slotgate, capability.slotgate) + assert.equal(candidates[0].exec.nodeManifest, capability.nodeManifest) + assert.equal(candidates[0].workerStatus, capability.workerStatus) + assert.equal(candidates[0].exec.homeDir, undefined) + assert.equal(candidates[0].exec.vmPath, undefined) + } + }) +} From ca8be11603559d981b5a53cac14a2d73ff9ce5a5 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:59:33 +0800 Subject: [PATCH 379/473] fix: restore older snapshots with later same-schema foreign keys --- src/lib/admin/backup-service.mjs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/lib/admin/backup-service.mjs b/src/lib/admin/backup-service.mjs index a17fa8bb8..99e404b3c 100644 --- a/src/lib/admin/backup-service.mjs +++ b/src/lib/admin/backup-service.mjs @@ -447,6 +447,7 @@ export class BackupService { ).map(({ tablename }) => tablename) const extraTables = tables.filter((name) => !archive.tables.has(name)) const tableName = (name) => `${quoteIdentifier(db.schema)}.${quoteIdentifier(name)}` + const sqlLiteral = (value) => "'" + value.replaceAll("'", "''") + "'" const cleanup = path.join(archive.tmp, 'cleanup.sql') const restore = path.join(archive.tmp, 'restore.sql') fs.writeFileSync( @@ -465,6 +466,29 @@ export class BackupService { RAISE EXCEPTION 'restore refuses cross-schema inheritance or partitions'; END IF; END $$;`, + // A later migration may add a FK from an archived table to a new + // parent. pg_restore will replace the archived table, but the parent + // must disappear first. Remove only that same-schema dependency; + // external dependencies still fail RESTRICT and roll back all changes. + extraTables.length && archive.tables.size + ? `DO $vm2api_restore_fk$ DECLARE dependency record; BEGIN + FOR dependency IN + SELECT source.relname AS source_name, constraint_row.conname + FROM pg_catalog.pg_constraint constraint_row + JOIN pg_catalog.pg_class source ON source.oid=constraint_row.conrelid + JOIN pg_catalog.pg_namespace source_schema ON source_schema.oid=source.relnamespace + JOIN pg_catalog.pg_class target ON target.oid=constraint_row.confrelid + JOIN pg_catalog.pg_namespace target_schema ON target_schema.oid=target.relnamespace + WHERE constraint_row.contype='f' + AND source_schema.nspname=${sqlLiteral(db.schema)} + AND target_schema.nspname=${sqlLiteral(db.schema)} + AND source.relname IN (${[...archive.tables].map(sqlLiteral).join(',')}) + AND target.relname IN (${extraTables.map(sqlLiteral).join(',')}) + LOOP + EXECUTE format('ALTER TABLE %I.%I DROP CONSTRAINT %I', ${sqlLiteral(db.schema)}, dependency.source_name, dependency.conname); + END LOOP; + END $vm2api_restore_fk$;` + : '', extraTables.length ? `DROP TABLE ${extraTables.map(tableName).join(', ')};` : '', ].join('\n'), { mode: 0o600 }, From ea21db4adc0cfa847dbf1a5b4259a9cb5b224431 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:59:33 +0800 Subject: [PATCH 380/473] feat: schedule HA requests using captured node capabilities --- src/lib/pool/pool-scheduler.mjs | 77 +++++++++++++++++++++++++----- test/unit/ha-pool-capture.test.mjs | 38 +++++++++++---- 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 8e51cbd6b..d133f063b 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -51,7 +51,8 @@ import { SeatPlanner, rankSeatCandidates } from './seat-planner.mjs' import { GATE_DRAINING } from './quota-gate.mjs' import { activeHoursOf, evaluateActiveHours } from './active-hours.mjs' import { vmNodeReady } from '../cluster/placement.mjs' -import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { isKubernetesMode, isKubernetesHaMode } from '../core/control-plane-mode.mjs' +import { haVmCatalog } from '../vm/ha-vm-catalog.mjs' import { getDb } from '../db/database.mjs' import { SharedExecutionAdmissions } from './shared-execution-admissions.mjs' @@ -223,7 +224,7 @@ function selectionSnapshot(candidates = [], available = [], extras = {}) { } export function accountIdOf(vm, projectRoot = null) { - if (projectRoot && vm?.id) { + if (!isKubernetesHaMode() && projectRoot && vm?.id) { const slot = readSlotCredentialIdentity(vmCliHomePath(projectRoot, vm.id)) if (slot?.account_uuid) return slot.account_uuid } @@ -332,6 +333,7 @@ export class PoolScheduler extends EventEmitter { runtimeRepo = null, workerHealth = null, executionAdmissions = null, + captureNodeExecution = null, routingState = null, config = {}, } = {}) { @@ -344,6 +346,8 @@ export class PoolScheduler extends EventEmitter { this.runtimeRepo = runtimeRepo this.workerHealth = workerHealth this.shared = isKubernetesMode() + this.ha = isKubernetesHaMode() + this.captureNodeExecution = captureNodeExecution this.routingState = routingState this.executionAdmissions = this.shared ? executionAdmissions || @@ -1034,6 +1038,7 @@ export class PoolScheduler extends EventEmitter { keyScope = null, quotaRequest = null, } = {}) { + if (this.ha) await haVmCatalog().refresh() const routingRevision = this.routingState?.currentRevision const now = Date.now() this.runtimeRepo?.clearExpired?.(now) @@ -1057,6 +1062,17 @@ export class PoolScheduler extends EventEmitter { if (!keyAllowsVm(keyScope, vm)) continue const accountId = accountIdOf(vm, this.projectRoot) if (!accountId || excluded.has(accountId) || excluded.has(vm.id)) continue + let captured = null + if (this.ha) { + if (!vm.node_id || typeof this.captureNodeExecution !== 'function') continue + try { + captured = await this.captureNodeExecution({ vmId: vm.id, accountId, nodeId: vm.node_id, signal }) + } catch { + continue + } + if (!captured?.slotgate || captured.workerStatus?.ok !== true) continue + } + const exec = this.executionContext(vm, accountId, captured) const state = this.runtimeRepo?.get?.(accountId) || null const eligibility = await this.checkEligibility({ vm, @@ -1067,6 +1083,8 @@ export class PoolScheduler extends EventEmitter { signal, pinned: !!pin, quotaRequest, + exec, + capturedHealth: captured?.workerStatus, }) if (!eligibility.ok) { const kind = eligibility.reason @@ -1115,12 +1133,24 @@ export class PoolScheduler extends EventEmitter { cooldownReason: state?.cooldown_reason || null, quotaRequest, sharedCounts: eligibility.sharedCounts, - exec: this.executionContext(vm, accountId), + exec, }) } return candidates } - async checkEligibility({ vm, accountId, state, model, now, signal, pinned = false, quotaRequest = null }) { + async checkEligibility({ + vm, + accountId, + state, + model, + now, + signal, + pinned = false, + quotaRequest = null, + exec = null, + capturedHealth = null, + }) { + if (this.ha && !exec?.slotgate) return { ok: false, reason: 'node_capability_unavailable' } // sub2api IsSchedulable: rate_limit_reset_at / overload_until gate before any // passive Extra reading or health hop. Pins are diagnostics and still reach the slot. const hardBlock = pinned ? null : hardBlockOf(state, now) @@ -1134,14 +1164,16 @@ export class PoolScheduler extends EventEmitter { if (circuit?.reason === 'circuit_probe') circuitProbeUntil = circuit.until } const gate = evaluateSlotGate(vm) - if (!gate.ok && gate.reason !== 'no_credential') { + if (!gate.ok && gate.reason !== 'no_credential' && !(this.ha && gate.reason === 'proxy_required')) { // Master pin may test a slot taken out of the pool, but SOCKS is still mandatory. if (!(pinned && gate.reason === 'vm_unschedulable')) return gate } - if (!slotHasBoundProxy(vm)) return { ok: false, reason: 'proxy_required' } - const workerProxyEndpoint = this.projectRoot ? readWorkerProxyEndpoint(this.projectRoot, vm.id) : undefined - const egressMode = this.projectRoot ? readWorkerEgressMode(this.projectRoot, vm.id) : '' - const proxySync = evaluateProxySync({ vm, workerProxyEndpoint, egressMode }) + if (!this.ha && !slotHasBoundProxy(vm)) return { ok: false, reason: 'proxy_required' } + const workerProxyEndpoint = + !this.ha && this.projectRoot ? readWorkerProxyEndpoint(this.projectRoot, vm.id) : undefined + const egressMode = !this.ha && this.projectRoot ? readWorkerEgressMode(this.projectRoot, vm.id) : '' + // HA egress is fixed by the node's independently checked launch policy. + const proxySync = this.ha ? { ok: true } : evaluateProxySync({ vm, workerProxyEndpoint, egressMode }) if (!proxySync.ok) { if ( this.projectRoot && @@ -1207,6 +1239,7 @@ export class PoolScheduler extends EventEmitter { }) if (!ev.accept) { if ( + !this.ha && this.projectRoot && vm.schedulable !== false && ev.reason === 'quota_refresh_failed' && @@ -1229,7 +1262,8 @@ export class PoolScheduler extends EventEmitter { const rest = evaluateActiveHours(activeHoursOf(vm), now, vm.timezone) if (!rest.active) return { ok: false, reason: 'rest_period', until: rest.until } } - const workerStatus = await this.getWorkerHealth(this.executionContext(vm, accountId), { signal }) + const workerStatus = + capturedHealth || (await this.getWorkerHealth(exec || this.executionContext(vm, accountId), { signal })) const cred = evaluateCredentialEligibility({ vm, workerStatus, now }) if (!cred.ok) return cred state = this.runtimeRepo?.get?.(accountId) || state @@ -1321,7 +1355,7 @@ export class PoolScheduler extends EventEmitter { return { ok: false, reason: 'kernel_faulted' } } // Node offline or asleep: the relay has no link behind it, a pick would only fail. Back on reconnect. - if (!vmNodeReady(vm)) return { ok: false, reason: 'node_offline' } + if (!this.ha && !vmNodeReady(vm)) return { ok: false, reason: 'node_offline' } if (rustKernelBusy(workerStatus)) { markWait('slot_busy') } else if (workerStatus && rustKernelProcessUp(workerStatus) && !rustKernelReachable(workerStatus)) { @@ -1330,7 +1364,23 @@ export class PoolScheduler extends EventEmitter { return { ok: true, account, workerStatus, busy, availableAt, waitReason, headroom, gateZone, sharedCounts } } - executionContext(vm, accountId) { + executionContext(vm, accountId, captured = null) { + if (this.ha) { + if (!captured?.slotgate) + throw Object.assign(new Error('HA node capability unavailable'), { code: 'node_capability_unavailable' }) + return { + vmId: vm.id, + accountId, + vm, + slotgate: captured.slotgate, + nodeManifest: captured.nodeManifest, + nodeIdentity: captured.workerStatus, + seedPolicy: vm.seed_policy, + timezone: vm.timezone || 'UTC', + locale: vm.locale || 'en_US.UTF-8', + kernel: vm.kernel || null, + } + } const homeDir = vmCliHomePath(this.projectRoot, vm.id) const slot = readSlotCredentialIdentity(homeDir) return { @@ -2072,6 +2122,9 @@ export class PoolScheduler extends EventEmitter { * schedule_manual is restored to on + restriction. */ syncQuotaSchedule(vm, account = null) { + // Eligibility still reads live PG quota/runtime state. No Pod file can + // change the node-owned manifest or become the VM metadata authority. + if (this.ha) return { action: 'keep', reason: 'node_owned_manifest' } if (!this.projectRoot || !vm?.id) return { action: 'keep', reason: null } const accountId = account?.account_id || vm.claude?.account_uuid || vm.id account = account || this.accountQuota?.repo?.get?.(accountId) || null diff --git a/test/unit/ha-pool-capture.test.mjs b/test/unit/ha-pool-capture.test.mjs index ad4c914fd..e0d636cfe 100644 --- a/test/unit/ha-pool-capture.test.mjs +++ b/test/unit/ha-pool-capture.test.mjs @@ -17,29 +17,51 @@ for (const available of [true, false]) { const db = getDb() const repo = new VmsRepo(db) await repo.upsertFromVmJson({ - id: 'ha-pool-vm', node_id: 'ha-node', status: 'running', schedulable: true, - claude: { account_uuid: 'ha-pool-account', has_access: true, has_refresh: true, expires_at: Date.now() + 3600_000 }, + id: 'ha-pool-vm', + node_id: 'ha-node', + status: 'running', + schedulable: true, + claude: { + account_uuid: 'ha-pool-account', + has_access: true, + has_refresh: true, + expires_at: Date.now() + 3600_000, + }, }) const catalog = new HaVmCatalog(db) bindHaVmCatalog(catalog) let captures = 0 let legacyHealth = 0 const capability = Object.freeze({ - slotgate: Object.freeze({ trusted: true, pgBindingReady: true, vmId: 'ha-pool-vm', nodeId: 'ha-node', - fence: { generation: '1', incarnation: 'captured-incarnation' }, manifest: { revision: '1', digest: 'a'.repeat(64) } }), + slotgate: Object.freeze({ + trusted: true, + pgBindingReady: true, + vmId: 'ha-pool-vm', + nodeId: 'ha-node', + fence: { generation: '1', incarnation: 'captured-incarnation' }, + manifest: { revision: '1', digest: 'a'.repeat(64) }, + }), nodeManifest: Object.freeze({ revision: '1', digest: 'a'.repeat(64) }), - workerStatus: Object.freeze({ ok: true, healthy: true, ready_slots: 1, - credential: { has_access: true, has_refresh: true, account_uuid: 'ha-pool-account', generation: 1 } }), + workerStatus: Object.freeze({ + ok: true, + healthy: true, + ready_slots: 1, + credential: { has_access: true, has_refresh: true, account_uuid: 'ha-pool-account', generation: 1 }, + }), }) const scheduler = new PoolScheduler({ projectRoot: '/nonexistent-ha-pod-project', captureNodeExecution: async ({ vmId, accountId, nodeId }) => { captures++ assert.deepEqual([vmId, accountId, nodeId], ['ha-pool-vm', 'ha-pool-account', 'ha-node']) - if (!available) throw Object.assign(new Error('synthetic node unavailable'), { code: 'node_capability_unavailable' }) + if (!available) + throw Object.assign(new Error('synthetic node unavailable'), { code: 'node_capability_unavailable' }) return capability }, - workerHealth: async () => { legacyHealth++; throw new Error('legacy worker health must not run') }, + workerHealth: async () => { + legacyHealth++ + throw new Error('legacy worker health must not run') + }, executionAdmissions: { inspectCircuit: async () => ({ open: false }), getCounts: async () => ({ accountInflight: 0, vmInflight: 0, fableInflight: 0, rpmCount: 0 }), From 495769d5605b7f66a890344a6e4a7ae6c462895f Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:51:06 +0800 Subject: [PATCH 381/473] test: specify typed node management transport and receipt persistence --- test/fixtures/real-allocation-gate.mjs | 5 +- test/unit/node-management-client.test.mjs | 292 ++++++++++++++++++++++ 2 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 test/unit/node-management-client.test.mjs diff --git a/test/fixtures/real-allocation-gate.mjs b/test/fixtures/real-allocation-gate.mjs index cfaa8b348..ff08e48e2 100644 --- a/test/fixtures/real-allocation-gate.mjs +++ b/test/fixtures/real-allocation-gate.mjs @@ -5,7 +5,7 @@ import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' -export async function realAllocationGate(t, management) { +export async function realAllocationGate(t, management, { launch, dockerSocket } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(),'ag2-')) const worker = path.join(root,'worker') fs.cpSync(fileURLToPath(new URL('../../worker',import.meta.url)),worker,{recursive:true}) @@ -36,7 +36,8 @@ export async function realAllocationGate(t, management) { fs.mkdirSync(runtime,{mode:0o700}) const configFile = path.join(root,'config.json') fs.writeFileSync(configFile,JSON.stringify({listen:'127.0.0.1:0',root:runtime,token:'synthetic-node-allocation-token-000000',tls_dir:path.join(root,'tls'), - fenced:true,authority_dir:path.join(root,'authority'),docker_socket:path.join(root,'never-opened.sock'),management}),{mode:0o600}) + fenced:true,authority_dir:path.join(root,'authority'),docker_socket:dockerSocket || path.join(root,'never-opened.sock'),management, + ...(launch ? {launch} : {})}),{mode:0o600}) child = spawn(binary,[configFile],{stdio:['ignore','pipe','pipe']}) exited = once(child,'exit') const link = await new Promise((resolve,reject) => { diff --git a/test/unit/node-management-client.test.mjs b/test/unit/node-management-client.test.mjs new file mode 100644 index 000000000..bfdbc08cd --- /dev/null +++ b/test/unit/node-management-client.test.mjs @@ -0,0 +1,292 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import tls from 'node:tls' +import pg from 'pg' +import * as management from '../../src/lib/transport/slotgate-management-client.mjs' +import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' + +const nodeId = 'command-node' +const vmId = 'command-vm' +const allocationId = 'command-allocation' +const payload = Buffer.from( + JSON.stringify({ + credential: { + type: 'oauth', + access_token: 'synthetic-command-access', + refresh_token: 'synthetic-command-refresh', + account_uuid: 'synthetic-account', + org_uuid: 'synthetic-org', + email: 'synthetic@example.invalid', + }, + }), +) + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const coordinator = new GlobalMaintenanceCoordinator(db) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'nmc-')) + let node + let readerRole + let engine + let closed = false + const service = new NodeManagementGrants(db, { resolveNodeLink: async () => node.link }) + const close = async () => { + if (closed) return + closed = true + await node?.close() + if (engine) { + engine.closeAllConnections() + await new Promise((resolve) => engine.close(resolve)) + } + await coordinator.stop() + await peer.close() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + if (readerRole) + await cleanup.query(`DROP OWNED BY ${quoteIdentifier(readerRole)}; DROP ROLE ${quoteIdentifier(readerRole)}`) + await cleanup.query( + `DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE; DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`, + ) + } finally { + await cleanup.end() + fs.rmSync(root, { recursive: true, force: true }) + } + } + t.after(close) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + await service.initialize() + await service.initializeCommands() + readerRole = 'command_wire_' + crypto.randomBytes(8).toString('hex') + const password = crypto.randomBytes(24).toString('hex') + await db.pool.query( + `CREATE ROLE ${quoteIdentifier(readerRole)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, + ) + await db.pool.query(service.readerRoleSql({ role: readerRole, nodeId })) + await db.pool.query(service.commandReaderRoleSql({ role: readerRole })) + const url = new URL(db.connectionString) + url.username = readerRole + url.password = password + const policy = { + image_id: 'sha256:' + 'a'.repeat(64), + network_id: 'none', + uid: process.getuid() || 1000, + gid: process.getgid() || 1000, + } + const policyDigest = crypto + .createHash('sha256') + .update( + [ + 'slotgate-policy-v1', + 'claude-docker-v1', + policy.image_id, + policy.network_id, + String(policy.uid), + String(policy.gid), + ].join('\0'), + ) + .digest('hex') + const calls = [] + let spec + const containerId = 'b'.repeat(64) + engine = http.createServer(async (req, res) => { + calls.push(`${req.method} ${req.url}`) + const parsed = new URL(req.url, 'http://synthetic-engine') + const send = (body, status = 200) => { + res.writeHead(status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(body)) + } + if (parsed.pathname === '/info') return send({ ID: 'synthetic-daemon' }) + if (parsed.pathname.startsWith('/images/')) + return send({ Id: policy.image_id, Config: { Labels: { 'kin.slot.full': '1' } } }) + if (parsed.pathname === '/containers/create') { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + spec = JSON.parse(Buffer.concat(chunks).toString()) + return send({ Id: containerId }, 201) + } + if (parsed.pathname === `/containers/${containerId}/start`) return res.writeHead(204).end() + if (parsed.pathname === `/containers/${containerId}/json`) + return send({ + Id: containerId, + Image: policy.image_id, + Config: { Labels: spec.Labels }, + State: { Running: true, Restarting: false, StartedAt: '2026-10-22T00:00:00.000000001Z' }, + }) + res.writeHead(404).end() + }) + const socket = path.join(root, 'engine.sock') + await new Promise((resolve, reject) => { + engine.once('error', reject) + engine.listen(socket, resolve) + }) + node = await realAllocationGate( + t, + { connection_string: url.toString(), authority_schema: service.schema, node_id: nodeId }, + { launch: policy, dockerSocket: socket }, + ) + const run = (kind, callback) => new SharedVmOperations(db).run(vmId, callback, { kind }) + const allocated = await run('node_allocate', (operation) => + service.allocate({ operation, nodeId, vmId, allocationId }), + ) + const input = (operation, patch = {}) => ({ + operation, + nodeId, + vmId, + allocationId, + commandId: 'wire-publish', + capturedFence: allocated.fence, + presetId: 'claude-docker-v1', + policyDigest, + payload, + ...patch, + }) + return { db, peer, service, run, input, node, calls, root, close, allocated, q: quoteIdentifier(service.schema) } +} + +function nodeCase(name, callback) { + test(name, async (t) => { + const fx = await fixture(t) + try { + await callback(fx, t) + } finally { + await fx.close() + } + }) +} + +test('management transport validates the bounded payload before opening any connection', async (t) => { + let connections = 0 + t.mock.method(tls, 'connect', () => { + connections++ + throw new Error('unexpected network') + }) + for (const value of [Buffer.alloc(0), Buffer.alloc(65537)]) { + await assert.rejects(management.publishNodeSlotInitial({ link: {}, grant: {}, payload: value }), { + code: 'node_command_invalid', + vmOperationNotStarted: true, + }) + } + assert.equal(connections, 0) +}) + +nodeCase( + 'real PostgreSQL grants drive pinned TLS initial publication and launch without holding a transaction over node I/O', + async (fx, t) => { + const connect = tls.connect + let sends = 0 + t.mock.method(tls, 'connect', (...args) => { + assert.equal(fx.db.isTransaction, false) + sends++ + return connect.apply(tls, args) + }) + const published = await fx.run('node_publish_initial', (operation) => + fx.service.publishInitial(fx.input(operation)), + ) + assert.equal(published.manifest_revision, '1') + assert.equal(fx.calls.length, 0, 'publishing never calls Docker') + const launched = await fx.run('node_launch', (operation) => + fx.service.launch( + fx.input(operation, { + commandId: 'wire-launch', + manifestDigest: published.manifest_digest, + }), + ), + ) + assert.equal(sends, 2, 'one management connection per issued action') + assert.equal(launched.fence.executor.kind, 'container') + assert.equal(launched.fence.executor.container_id, 'b'.repeat(64)) + assert.equal(launched.fence.incarnation, fx.allocated.fence.incarnation) + assert.deepEqual(await new NodeManagementGrants(fx.peer).readLaunchReceipt({ vmId }), launched) + const rows = (await fx.peer.query(`SELECT receipt,revoked_at FROM ${fx.q}.command_grants`)).rows + assert.ok(rows.every((row) => row.revoked_at && row.receipt)) + assert.equal(JSON.stringify(rows).includes('synthetic-command-access'), false) + assert.equal(JSON.stringify(rows).includes('synthetic-command-refresh'), false) + assert.equal(fx.calls.filter((call) => call.startsWith('POST /containers/create')).length, 1) + }, +) + +nodeCase( + 'a lost publication acknowledgement retains the original VM operation and does not issue another command', + async (fx, t) => { + const connect = tls.connect + let sends = 0 + t.mock.method(tls, 'connect', (...args) => { + sends++ + const socket = connect.apply(tls, args) + const emit = socket.emit + socket.emit = function (event, ...values) { + if (event === 'data') { + socket.destroy() + return true + } + return emit.call(this, event, ...values) + } + return socket + }) + await assert.rejects( + fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))), + { code: 'node_publication_unconfirmed', vmOperationUncertain: true }, + ) + assert.equal(sends, 1) + assert.equal(fx.calls.length, 0) + const row = (await fx.peer.query(`SELECT receipt,revoked_at FROM ${fx.q}.command_grants`)).rows[0] + assert.equal(row.receipt, null) + assert.ok(row.revoked_at) + const operation = ( + await fx.peer.query("SELECT released_at,uncertain_at FROM vm_operations WHERE kind='node_publish_initial'") + ).rows[0] + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + assert.equal((await new SharedVmOperations(fx.peer).tryAcquire({ vmId })).ok, false) + const remoteConfig = path.join( + fx.node.root, + 'runtime', + 'vms', + vmId, + 'incarnations', + fx.allocated.fence.incarnation, + 'run', + 'kernel.json', + ) + assert.equal(fs.existsSync(remoteConfig), true, 'the real node committed files before its acknowledgement was lost') + }, +) + +nodeCase( + 'failure to commit the received node receipt retains ownership and never repeats the mutation', + async (fx, t) => { + await fx.db.query(`CREATE FUNCTION reject_command_receipt() RETURNS trigger LANGUAGE plpgsql AS $$BEGIN RAISE EXCEPTION 'synthetic receipt failure' USING ERRCODE='P648C'; END$$; + CREATE TRIGGER reject_command_receipt BEFORE UPDATE OF receipt ON ${fx.q}.command_grants FOR EACH ROW EXECUTE FUNCTION reject_command_receipt()`) + const connect = tls.connect + let sends = 0 + t.mock.method(tls, 'connect', (...args) => { + sends++ + return connect.apply(tls, args) + }) + await assert.rejects( + fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))), + { code: 'P648C' }, + ) + assert.equal(sends, 1) + const row = (await fx.peer.query(`SELECT receipt,revoked_at FROM ${fx.q}.command_grants`)).rows[0] + assert.equal(row.receipt, null) + assert.ok(row.revoked_at) + const operation = ( + await fx.peer.query("SELECT released_at,uncertain_at FROM vm_operations WHERE kind='node_publish_initial'") + ).rows[0] + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + }, +) From ffea6c148ca2ee042779d3c5dcf8bd34c5702d79 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:56:59 +0800 Subject: [PATCH 382/473] test: distinguish unstarted node publication refusal from lost acknowledgements --- test/unit/node-management-client.test.mjs | 176 +++++++++++++++++++++- 1 file changed, 174 insertions(+), 2 deletions(-) diff --git a/test/unit/node-management-client.test.mjs b/test/unit/node-management-client.test.mjs index bfdbc08cd..93a494ad1 100644 --- a/test/unit/node-management-client.test.mjs +++ b/test/unit/node-management-client.test.mjs @@ -152,7 +152,20 @@ async function fixture(t) { payload, ...patch, }) - return { db, peer, service, run, input, node, calls, root, close, allocated, q: quoteIdentifier(service.schema) } + return { + db, + peer, + coordinator, + service, + run, + input, + node, + calls, + root, + close, + allocated, + q: quoteIdentifier(service.schema), + } } function nodeCase(name, callback) { @@ -257,7 +270,7 @@ nodeCase( vmId, 'incarnations', fx.allocated.fence.incarnation, - 'run', + 'config', 'kernel.json', ) assert.equal(fs.existsSync(remoteConfig), true, 'the real node committed files before its acknowledgement was lost') @@ -290,3 +303,162 @@ nodeCase( assert.ok(operation.uncertain_at) }, ) + +nodeCase('a payload digest mismatch is rejected before TLS or any publication bytes', async (fx, t) => { + let connections = 0 + t.mock.method(tls, 'connect', () => { + connections++ + throw new Error('unexpected network') + }) + await fx.run('node_publish_initial', async (operation) => { + const grant = await fx.service.issueInitialPublish( + fx.input(operation, { + expectedManifestRevision: '0', + revision: '1', + payloadDigest: crypto.createHash('sha256').update(payload).digest('hex'), + }), + ) + await assert.rejects( + management.publishNodeSlotInitial({ link: fx.node.link, grant, payload: Buffer.from('tampered input') }), + { + code: 'node_command_invalid', + vmOperationNotStarted: true, + }, + ) + }) + assert.equal(connections, 0) + assert.equal(fx.calls.length, 0) +}) + +nodeCase( + 'DRAINING refusal before issuing a publication sends no bytes and releases only that unstarted operation', + async (fx, t) => { + const maintenance = await fx.coordinator.beginMaintenance({ kind: 'backup' }) + let connections = 0 + t.mock.method(tls, 'connect', () => { + connections++ + throw new Error('unexpected network') + }) + try { + await assert.rejects( + fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))), + { + code: 'node_command_epoch_stale', + vmOperationNotStarted: true, + }, + ) + assert.equal(connections, 0) + assert.equal((await fx.peer.query(`SELECT count(*) AS n FROM ${fx.q}.command_grants`)).rows[0].n, 0) + const operation = ( + await fx.peer.query("SELECT released_at,uncertain_at FROM vm_operations WHERE kind='node_publish_initial'") + ).rows[0] + assert.ok(operation.released_at) + assert.equal(operation.uncertain_at, null) + } finally { + try { + const pending = (await fx.peer.query('SELECT count(*) AS n FROM vm_operations WHERE released_at IS NULL')) + .rows[0].n + if (!pending) await maintenance.resume() + } finally { + await maintenance.close() + } + } + }, +) + +nodeCase('a certificate pin mismatch sends no application bytes and retains no active VM operation', async (fx, t) => { + const connect = tls.connect + let writes = 0 + t.mock.method(tls, 'connect', (...args) => { + const socket = connect.apply(tls, args) + const write = socket.write + socket.write = function (...values) { + writes++ + return write.apply(this, values) + } + return socket + }) + fx.service.resolveNodeLink = async () => ({ ...fx.node.link, certSha256: 'f'.repeat(64) }) + await assert.rejects( + fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))), + { + code: 'node_publication_unconfirmed', + vmOperationNotStarted: true, + }, + ) + assert.equal(writes, 0) + const operation = ( + await fx.peer.query("SELECT released_at,uncertain_at FROM vm_operations WHERE kind='node_publish_initial'") + ).rows[0] + assert.ok(operation.released_at) + assert.equal(operation.uncertain_at, null) +}) + +nodeCase( + 'an extra secret field in a completed node acknowledgement is rejected and never persisted or exposed', + async (fx, t) => { + const connect = tls.connect + t.mock.method(tls, 'connect', (...args) => { + const socket = connect.apply(tls, args) + const emit = socket.emit + let bytes = Buffer.alloc(0) + socket.emit = function (event, ...values) { + if (event !== 'data') return emit.call(this, event, ...values) + bytes = Buffer.concat([bytes, values[0]]) + const newline = bytes.indexOf(10) + if (newline < 0) return true + const reply = JSON.parse(bytes.subarray(0, newline)) + reply.publication.access_token = 'synthetic-response-secret' + return emit.call(this, 'data', Buffer.from(JSON.stringify(reply) + '\n')) + } + return socket + }) + await assert.rejects( + fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))), + (error) => { + assert.equal(error.code, 'node_publication_unconfirmed') + assert.equal(error.vmOperationUncertain, true) + assert.equal(error.message.includes('synthetic-response-secret'), false) + return true + }, + ) + assert.equal((await fx.peer.query(`SELECT receipt FROM ${fx.q}.command_grants`)).rows[0].receipt, null) + }, +) + +nodeCase('a lost launch acknowledgement preserves the exact launch operation after Docker start', async (fx, t) => { + const published = await fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))) + const connect = tls.connect + let sends = 0 + t.mock.method(tls, 'connect', (...args) => { + sends++ + const socket = connect.apply(tls, args) + const emit = socket.emit + socket.emit = function (event, ...values) { + if (event === 'data') { + socket.destroy() + return true + } + return emit.call(this, event, ...values) + } + return socket + }) + await assert.rejects( + fx.run('node_launch', (operation) => + fx.service.launch( + fx.input(operation, { + commandId: 'wire-launch', + manifestDigest: published.manifest_digest, + }), + ), + ), + { code: 'node_launch_unconfirmed', vmOperationUncertain: true }, + ) + assert.equal(sends, 1) + assert.equal(fx.calls.filter((call) => call.endsWith('/start')).length, 1) + const operation = (await fx.peer.query("SELECT released_at,uncertain_at FROM vm_operations WHERE kind='node_launch'")) + .rows[0] + assert.equal(operation.released_at, null) + assert.ok(operation.uncertain_at) + assert.equal(await new NodeManagementGrants(fx.peer).readLaunchReceipt({ vmId }), null) +}) From 13360459f373328f0253265292d33dad8fd4f1ff Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:58:27 +0800 Subject: [PATCH 383/473] feat: publish and launch node slots through pinned typed management RPC --- .../2026-10-22-node-command-authority.md | 4 + src/lib/cluster/node-management-grants.mjs | 192 ++++++------------ src/lib/core/node-management-command.mjs | 141 +++++++++++++ .../transport/slotgate-management-client.mjs | 115 +++++++++++ 4 files changed, 320 insertions(+), 132 deletions(-) create mode 100644 src/lib/core/node-management-command.mjs diff --git a/docs/plans/2026-10-22-node-command-authority.md b/docs/plans/2026-10-22-node-command-authority.md index a430a8e59..0d76e2652 100644 --- a/docs/plans/2026-10-22-node-command-authority.md +++ b/docs/plans/2026-10-22-node-command-authority.md @@ -14,6 +14,10 @@ - `recordInitialPublishReceipt(grant, receipt)` / `recordLaunchReceipt(grant, receipt)`：在原精确操作仍有效时重新检查授权、epoch 和完整来源链，原子保存固定字段回执。相同回执可重复确认；不同回执拒绝覆盖。 - `readLaunchReceipt({vmId, nodeId?})`：从 PG 确定节点并验证完整链，返回原 launch 回执。正常撤销授权不会删除已确认回执。它证明历史启动已被确认，不证明容器当前仍在运行，也不获取新 fence 替换原值。 +`publishInitial(input)` 和 `launch(input)` 将上述接口组合为发行授权、事务外单次 TLS 请求、严格校验回执、原子保存。首次发布要求 Buffer，复制后绑定其完整 SHA-256，限制为 1..65536 字节。客户端只接受固定 IP、TLS 1.3 和证书指纹；节点 token 用于运输认证，不能代替 PG 动作授权。客户端不重试、不重新发行授权，不接受 shell、路径、Docker 参数或任意配置 RPC。 + +仅在 TLS 应用数据尚未发送时，证书或连接拒绝可带明确的 `vmOperationNotStarted`。发行前的维护阶段/特性拒绝同样使用这一证明。发送后的断连、无效回执和 PG 回执提交失败继续保留原 VM 操作。回执上限为 8192 字节，错误不包含上传内容或节点返回的原始文本。 + 每个 allocation 的每种新动作只允许一条命令。command ID、动作、操作、owner、epoch、fence、payload、preset 和 policy 不能通过重新发行修改。未知操作继续占用 VM；本增量不提供超时释放或自动恢复。 ## 固定协议 diff --git a/src/lib/cluster/node-management-grants.mjs b/src/lib/cluster/node-management-grants.mjs index b73b8e272..9a2223997 100644 --- a/src/lib/cluster/node-management-grants.mjs +++ b/src/lib/cluster/node-management-grants.mjs @@ -4,9 +4,20 @@ import { coordinationSchemaFor } from '../db/global-maintenance.mjs' import { currentVmOperationIds } from '../vm/shared-vm-operations.mjs' import { physicalContainerName } from '../vm/vm-physical-target.mjs' import { ClusterNodesRepo } from '../db/repos/cluster-nodes-repo.mjs' -import { allocateNodeSlot, normalizeAllocationReceipt } from '../transport/slotgate-management-client.mjs' +import { + allocateNodeSlot, + normalizeAllocationReceipt, + publishNodeSlotInitial, + launchNodeSlot, +} from '../transport/slotgate-management-client.mjs' import { isDeepStrictEqual } from 'node:util' -import { normalizeNodeFence } from '../core/node-execution-proof.mjs' +import { + normalizeVacantNodeFence as vacantFence, + normalizeNodeManagementIntent as commandIntent, + nodeManagementIntentDigest as commandDigest, + normalizeNodeManagementGrant as commandGrant, + normalizeNodeManagementReceipt as commandReceipt, +} from '../core/node-management-command.mjs' import { initializeNodeAuthority, nodeAuthorityState, @@ -30,109 +41,9 @@ function literal(value) { return "'" + value.replaceAll("'", "''") + "'" } -const COMMAND_KEYS = [ - 'action', - 'node_id', - 'vm_id', - 'allocation_id', - 'operation_id', - 'owner_id', - 'command_id', - 'epoch', - 'fence', - 'expected_manifest_revision', - 'manifest_revision', - 'payload_digest', - 'preset_id', - 'policy_digest', - 'manifest_digest', -] -const digestPattern = /^[a-f0-9]{64}$/ -function isDigest(value) { - return typeof value === 'string' && digestPattern.test(value) -} function commandFailure(code = 'node_command_invalid') { return nodeAuthorityError(code) } -function exactShape(value, keys) { - if ( - !value || - typeof value !== 'object' || - Array.isArray(value) || - ![Object.prototype, null].includes(Object.getPrototypeOf(value)) || - Object.keys(value).length !== keys.length || - keys.some((key) => !Object.hasOwn(value, key)) - ) - throw commandFailure() -} -function vacantFence(value) { - exactShape(value, ['generation', 'incarnation', 'executor']) - exactShape(value.executor, ['kind', 'node_boot_id']) - if ( - (value.generation !== '1' && value.generation !== 1) || - typeof value.incarnation !== 'string' || - !/^[a-f0-9]{32}$/.test(value.incarnation) || - value.executor.kind !== '' || - value.executor.node_boot_id !== '' - ) - throw commandFailure() - // Initial allocation generation 1 is exactly representable in the existing Go JSON number wire format. - return { generation: 1, incarnation: value.incarnation, executor: { kind: '', node_boot_id: '' } } -} -function commandIntent(value) { - exactShape(value, COMMAND_KEYS) - for (const key of ['node_id', 'vm_id', 'allocation_id', 'operation_id', 'owner_id', 'command_id']) - identifier(value[key]) - if ( - typeof value.epoch !== 'string' || - !/^[1-9][0-9]{0,18}$/.test(value.epoch) || - BigInt(value.epoch) > 9223372036854775807n || - value.manifest_revision !== '1' || - value.preset_id !== 'claude-docker-v1' || - !isDigest(value.policy_digest) - ) - throw commandFailure() - if (value.action === 'publish-initial') { - if (value.expected_manifest_revision !== '0' || !isDigest(value.payload_digest) || value.manifest_digest !== '') - throw commandFailure() - } else if (value.action === 'launch') { - if (value.expected_manifest_revision !== '1' || value.payload_digest !== '' || !isDigest(value.manifest_digest)) - throw commandFailure() - } else throw commandFailure() - return { ...value, fence: vacantFence(value.fence) } -} -function commandDigest(intent) { - return crypto - .createHash('sha256') - .update( - [ - 'slotgate-command-v1', - intent.action, - intent.node_id, - intent.vm_id, - intent.allocation_id, - intent.operation_id, - intent.owner_id, - intent.command_id, - intent.epoch, - String(intent.fence.generation), - intent.fence.incarnation, - intent.expected_manifest_revision, - intent.manifest_revision, - intent.payload_digest, - intent.preset_id, - intent.policy_digest, - intent.manifest_digest, - ].join('\0'), - ) - .digest('hex') -} -function commandGrant(value) { - exactShape(value, ['intent', 'intentDigest']) - const intent = commandIntent(value.intent) - if (value.intentDigest !== commandDigest(intent)) throw commandFailure() - return { intent, intentDigest: value.intentDigest } -} function storedGrant(row) { if (!row) throw commandFailure('node_command_conflict') const grant = commandGrant({ intent: row.intent, intentDigest: row.intent_digest }) @@ -150,36 +61,6 @@ function storedGrant(row) { throw commandFailure('node_command_conflict') return grant } -function commandReceipt(grant, value) { - try { - const launch = grant.intent.action === 'launch' - exactShape(value, ['intent', 'intent_digest', 'manifest_revision', 'manifest_digest', ...(launch ? ['fence'] : [])]) - const intent = commandIntent(value.intent) - if ( - !isDeepStrictEqual(intent, grant.intent) || - value.intent_digest !== grant.intentDigest || - value.manifest_revision !== '1' || - !isDigest(value.manifest_digest) || - (launch && value.manifest_digest !== intent.manifest_digest) - ) - throw commandFailure() - const result = { - intent, - intent_digest: value.intent_digest, - manifest_revision: '1', - manifest_digest: value.manifest_digest, - } - if (launch) { - result.fence = normalizeNodeFence(value.fence) - if (result.fence.generation !== '1' || result.fence.incarnation !== intent.fence.incarnation) - throw commandFailure() - } - return result - } catch { - throw commandFailure('node_command_receipt_invalid') - } -} - export class NodeManagementGrants { constructor(db, { resolveNodeLink } = {}) { this.db = db @@ -343,6 +224,53 @@ export class NodeManagementGrants { return this._issueCommand('launch', input) } + async _issueCommandBeforeSend(action, input) { + try { + return await this._issueCommand(action, input) + } catch (error) { + // These authority refusals precede grant insertion and node transmission. + // Receipt persistence errors occur later and must not inherit this proof. + if ( + ['node_command_epoch_stale', 'node_command_authority_unavailable'].includes(error.code) && + error.vmOperationUncertain !== true && + error.executionUncertain !== true + ) + error.vmOperationNotStarted = true + throw error + } + } + + async publishInitial(input) { + if (!Buffer.isBuffer(input?.payload) || input.payload.length < 1 || input.payload.length > 65536) { + throw Object.assign(commandFailure(), { vmOperationNotStarted: true }) + } + const payload = Buffer.from(input.payload) + const link = await this.resolveNodeLink(input.nodeId) + if (!link) throw Object.assign(nodeAuthorityError('node_link_unavailable'), { vmOperationNotStarted: true }) + const grant = await this._issueCommandBeforeSend('publish-initial', { + ...input, + expectedManifestRevision: '0', + revision: '1', + payloadDigest: crypto.createHash('sha256').update(payload).digest('hex'), + manifestDigest: '', + }) + const receipt = await publishNodeSlotInitial({ link, grant, payload }) + return this.recordInitialPublishReceipt(grant, receipt) + } + + async launch(input) { + const link = await this.resolveNodeLink(input?.nodeId) + if (!link) throw Object.assign(nodeAuthorityError('node_link_unavailable'), { vmOperationNotStarted: true }) + const grant = await this._issueCommandBeforeSend('launch', { + ...input, + expectedManifestRevision: '1', + revision: '1', + payloadDigest: '', + }) + const receipt = await launchNodeSlot({ link, grant }) + return this.recordLaunchReceipt(grant, receipt) + } + async _recordCommand(grantValue, receiptValue, action) { const grant = commandGrant(grantValue) if (grant.intent.action !== action) throw commandFailure('node_command_receipt_invalid') diff --git a/src/lib/core/node-management-command.mjs b/src/lib/core/node-management-command.mjs new file mode 100644 index 000000000..d53314ce9 --- /dev/null +++ b/src/lib/core/node-management-command.mjs @@ -0,0 +1,141 @@ +import crypto from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' +import { normalizeNodeFence } from './node-execution-proof.mjs' + +function identifier(value) { + if (typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(value)) throw commandFailure() +} + +const COMMAND_KEYS = [ + 'action', + 'node_id', + 'vm_id', + 'allocation_id', + 'operation_id', + 'owner_id', + 'command_id', + 'epoch', + 'fence', + 'expected_manifest_revision', + 'manifest_revision', + 'payload_digest', + 'preset_id', + 'policy_digest', + 'manifest_digest', +] +const digestPattern = /^[a-f0-9]{64}$/ +function isDigest(value) { + return typeof value === 'string' && digestPattern.test(value) +} +function commandFailure(code = 'node_command_invalid') { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} +function exactShape(value, keys) { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + ![Object.prototype, null].includes(Object.getPrototypeOf(value)) || + Object.keys(value).length !== keys.length || + keys.some((key) => !Object.hasOwn(value, key)) + ) + throw commandFailure() +} +export function normalizeVacantNodeFence(value) { + exactShape(value, ['generation', 'incarnation', 'executor']) + exactShape(value.executor, ['kind', 'node_boot_id']) + if ( + (value.generation !== '1' && value.generation !== 1) || + typeof value.incarnation !== 'string' || + !/^[a-f0-9]{32}$/.test(value.incarnation) || + value.executor.kind !== '' || + value.executor.node_boot_id !== '' + ) + throw commandFailure() + // Initial allocation generation 1 is exactly representable in the existing Go JSON number wire format. + return { generation: 1, incarnation: value.incarnation, executor: { kind: '', node_boot_id: '' } } +} +export function normalizeNodeManagementIntent(value) { + exactShape(value, COMMAND_KEYS) + for (const key of ['node_id', 'vm_id', 'allocation_id', 'operation_id', 'owner_id', 'command_id']) + identifier(value[key]) + if ( + typeof value.epoch !== 'string' || + !/^[1-9][0-9]{0,18}$/.test(value.epoch) || + BigInt(value.epoch) > 9223372036854775807n || + value.manifest_revision !== '1' || + value.preset_id !== 'claude-docker-v1' || + !isDigest(value.policy_digest) + ) + throw commandFailure() + if (value.action === 'publish-initial') { + if (value.expected_manifest_revision !== '0' || !isDigest(value.payload_digest) || value.manifest_digest !== '') + throw commandFailure() + } else if (value.action === 'launch') { + if (value.expected_manifest_revision !== '1' || value.payload_digest !== '' || !isDigest(value.manifest_digest)) + throw commandFailure() + } else throw commandFailure() + return { ...value, fence: normalizeVacantNodeFence(value.fence) } +} +export function nodeManagementIntentDigest(intent) { + return crypto + .createHash('sha256') + .update( + [ + 'slotgate-command-v1', + intent.action, + intent.node_id, + intent.vm_id, + intent.allocation_id, + intent.operation_id, + intent.owner_id, + intent.command_id, + intent.epoch, + String(intent.fence.generation), + intent.fence.incarnation, + intent.expected_manifest_revision, + intent.manifest_revision, + intent.payload_digest, + intent.preset_id, + intent.policy_digest, + intent.manifest_digest, + ].join('\0'), + ) + .digest('hex') +} +export function normalizeNodeManagementGrant(value) { + exactShape(value, ['intent', 'intentDigest']) + const intent = normalizeNodeManagementIntent(value.intent) + if (value.intentDigest !== nodeManagementIntentDigest(intent)) throw commandFailure() + return { intent, intentDigest: value.intentDigest } +} +export function normalizeNodeManagementReceipt(grant, value) { + grant = normalizeNodeManagementGrant(grant) + try { + const launch = grant.intent.action === 'launch' + exactShape(value, ['intent', 'intent_digest', 'manifest_revision', 'manifest_digest', ...(launch ? ['fence'] : [])]) + const intent = normalizeNodeManagementIntent(value.intent) + if ( + !isDeepStrictEqual(intent, grant.intent) || + value.intent_digest !== grant.intentDigest || + value.manifest_revision !== '1' || + !isDigest(value.manifest_digest) || + (launch && value.manifest_digest !== intent.manifest_digest) + ) + throw commandFailure() + const result = { + intent, + intent_digest: value.intent_digest, + manifest_revision: '1', + manifest_digest: value.manifest_digest, + } + if (launch) { + result.fence = normalizeNodeFence(value.fence) + if (result.fence.generation !== '1' || result.fence.incarnation !== intent.fence.incarnation) + throw commandFailure() + } + return result + } catch { + throw commandFailure('node_command_receipt_invalid') + } +} diff --git a/src/lib/transport/slotgate-management-client.mjs b/src/lib/transport/slotgate-management-client.mjs index 4be95131f..96a383425 100644 --- a/src/lib/transport/slotgate-management-client.mjs +++ b/src/lib/transport/slotgate-management-client.mjs @@ -1,6 +1,8 @@ import tls from 'node:tls' import net from 'node:net' import { isDeepStrictEqual } from 'node:util' +import crypto from 'node:crypto' +import { normalizeNodeManagementGrant, normalizeNodeManagementReceipt } from '../core/node-management-command.mjs' function denied() { return Object.assign(new Error('node allocation acknowledgement unavailable'), { @@ -108,3 +110,116 @@ export function allocateNodeSlot({ link, grant }) { }) }) } + +function managementInvalid() { + return Object.assign(new Error('invalid node management command'), { + code: 'node_command_invalid', + vmOperationNotStarted: true, + }) +} + +function managementFailure(action, sent) { + return Object.assign(new Error('node management acknowledgement unavailable'), { + code: action === 'publish-initial' ? 'node_publication_unconfirmed' : 'node_launch_unconfirmed', + ...(sent ? { vmOperationUncertain: true } : { vmOperationNotStarted: true }), + }) +} + +function managementLink(link) { + if ( + !link || + !net.isIP(link.address) || + !Number.isInteger(link.port) || + link.port < 1 || + link.port > 65535 || + typeof link.token !== 'string' || + !/^[\x21-\x7e]{32,256}$/.test(link.token) || + typeof link.certSha256 !== 'string' || + !/^[a-f0-9]{64}$/.test(link.certSha256) + ) + throw managementInvalid() + return { address: link.address, port: link.port, token: link.token, certSha256: link.certSha256 } +} + +/** Exactly one typed action. No retry or shell/Docker/path parameter is accepted. */ +async function commandNodeSlot({ link: linkInput, grant: grantInput, payload: payloadInput }, action) { + let grant + let link + let payload + try { + if (action === 'publish-initial') { + if (!Buffer.isBuffer(payloadInput) || payloadInput.length < 1 || payloadInput.length > 65536) + throw managementInvalid() + payload = Buffer.from(payloadInput) + } + grant = normalizeNodeManagementGrant(grantInput) + if ( + grant.intent.action !== action || + (payload && crypto.createHash('sha256').update(payload).digest('hex') !== grant.intent.payload_digest) + ) + throw managementInvalid() + link = managementLink(linkInput) + } catch { + throw managementInvalid() + } + const request = { + op: action, + vm_id: grant.intent.vm_id, + management: grant.intent, + ...(payload ? { content_length: payload.length } : {}), + } + return new Promise((resolve, reject) => { + let settled = false + let sent = false + let buffer = Buffer.alloc(0) + const socket = tls.connect({ + host: link.address, + port: link.port, + rejectUnauthorized: false, + minVersion: 'TLSv1.3', + }) + const finish = (error, value) => { + if (settled) return + settled = true + clearTimeout(timer) + socket.destroy() + if (error) reject(error) + else resolve(value) + } + const failed = () => finish(managementFailure(action, sent)) + const timer = setTimeout(failed, 8000) + socket.once('error', failed) + socket.once('close', failed) + socket.once('secureConnect', () => { + const pin = socket.getPeerCertificate()?.fingerprint256?.replaceAll(':', '').toLowerCase() + if (pin !== link.certSha256) return failed() + sent = true + socket.write(`KSG2 ${link.token}\n${JSON.stringify(request)}\n`) + if (payload) socket.write(payload) + }) + socket.on('data', (chunk) => { + buffer = Buffer.concat([buffer, chunk]) + if (buffer.length > 8192) return failed() + const newline = buffer.indexOf(10) + if (newline < 0) return + try { + const reply = JSON.parse(buffer.subarray(0, newline).toString(), (key, value, context) => + key === 'generation' && typeof value === 'number' ? context.source : value, + ) + const field = action === 'publish-initial' ? 'publication' : 'launch' + exact(reply, ['ok', field]) + if (reply.ok !== true) return failed() + finish(null, normalizeNodeManagementReceipt(grant, reply[field])) + } catch { + failed() + } + }) + }) +} + +export function publishNodeSlotInitial(input) { + return commandNodeSlot(input, 'publish-initial') +} +export function launchNodeSlot(input) { + return commandNodeSlot(input, 'launch') +} From 1533d029b714d9b6790ea1b7192e3ca69fd9df33 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:58:50 +0800 Subject: [PATCH 384/473] test: require file-free captured node identity in HA mode --- test/unit/ha-vm-identity.test.mjs | 64 +++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 test/unit/ha-vm-identity.test.mjs diff --git a/test/unit/ha-vm-identity.test.mjs b/test/unit/ha-vm-identity.test.mjs new file mode 100644 index 000000000..0c48ad982 --- /dev/null +++ b/test/unit/ha-vm-identity.test.mjs @@ -0,0 +1,64 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import { loadVmIdentity, OFFICIAL_CLI_VERSION } from '../../src/lib/identity/vm-identity.mjs' + +function exec(patch = {}) { + return { vmId: 'vm-ha-identity', accountId: 'node-account', homeDir: '/forbidden/pod/cli-home', + slotgate: { trusted: true, pgBindingReady: true, vmId: 'vm-ha-identity' }, + nodeIdentity: { ok: true, credential: { account_uuid: 'node-account', org_uuid: 'node-org', email: 'node@example.invalid', has_access: true }, + identity: { account_uuid: 'node-account', org_uuid: 'node-org', email: 'node@example.invalid' } }, + oauth: { account_uuid: 'stale-pod-account', org_uuid: 'stale-pod-org', email: 'stale@example.invalid' }, + seedPolicy: { theme: 'light', telemetry_disabled: false }, + vm: { id: 'vm-ha-identity', fingerprint: { device_id: 'a'.repeat(64), session_id: 'pg-session', locale: 'en_US.UTF-8', timezone: 'Asia/Taipei' } }, + ...patch } +} + +function withoutFiles(callback) { + const beforeMode = process.env.VM2API_CONTROL_PLANE_MODE + const read = fs.readFileSync + const calls = [] + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + fs.readFileSync = (...args) => { calls.push(String(args[0])); throw new Error('Pod files forbidden') } + try { callback(calls) } finally { + fs.readFileSync = read + if (beforeMode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = beforeMode + } +} + +test('HA identity uses only captured node identity and the PG fingerprint without reading any Pod file', () => { + withoutFiles((calls) => { + const first = loadVmIdentity(exec()) + assert.equal(first.accountUuid, 'node-account') + assert.equal(first.orgUuid, 'node-org') + assert.equal(first.email, 'node@example.invalid') + assert.equal(first.deviceId, 'a'.repeat(64)) + assert.equal(first.sessionId, 'pg-session') + assert.equal(first.timezone, 'Asia/Taipei') + assert.equal(first.settings.theme, 'light') + assert.equal(first.cliVersion, OFFICIAL_CLI_VERSION) + assert.equal(first.userId, null) + assert.equal(first.displayName, null) + assert.deepEqual(loadVmIdentity(exec()), first, 'two Pods produce the same identity snapshot') + assert.equal(JSON.stringify(first).includes('stale-pod'), false) + assert.deepEqual(calls, []) + }) +}) + +for (const [name, patch] of [ + ['missing node proof', { nodeIdentity: undefined }], + ['failed node health', { nodeIdentity: { ok: false, credential: { account_uuid: 'node-account' } } }], + ['different credential account', { nodeIdentity: { ok: true, credential: { account_uuid: 'other-account' } } }], + ['identity account conflict', { nodeIdentity: { ok: true, credential: { account_uuid: 'node-account' }, identity: { account_uuid: 'other-account' } } }], + ['missing PG fingerprint', { vm: { id: 'vm-ha-identity' } }], + ['different PG VM', { vm: { id: 'another-vm', fingerprint: { device_id: 'a'.repeat(64), session_id: 'pg-session' } } }], + ['different captured VM', { slotgate: { trusted: true, pgBindingReady: true, vmId: 'another-vm' } }], +]) { + test(`HA identity refuses ${name} before local IO`, () => { + withoutFiles((calls) => { + assert.throws(() => loadVmIdentity(exec(patch)), { code: 'ha_identity_unavailable' }) + assert.deepEqual(calls, []) + }) + }) +} From d1a4ed2e282fa7ae2708f046382de2c5bda6b4e8 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:59:59 +0800 Subject: [PATCH 385/473] fix: construct HA identity from node proof and PostgreSQL metadata --- src/lib/identity/vm-identity.mjs | 46 ++++++++++++++++++++++++-- test/unit/ha-vm-identity.test.mjs | 55 ++++++++++++++++++++++++++----- 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/src/lib/identity/vm-identity.mjs b/src/lib/identity/vm-identity.mjs index 5ba83a8df..367641976 100644 --- a/src/lib/identity/vm-identity.mjs +++ b/src/lib/identity/vm-identity.mjs @@ -10,6 +10,7 @@ import { readSlotCredentialIdentity } from '../oauth/oauth-credentials.mjs' import { buildSlotSettingsEnv } from '../protocol/seed-policy.mjs' import { buildKinSeedJson } from './workstation-profile.mjs' import { isHexDeviceId } from './workstation-fingerprint.mjs' +import { isKubernetesHaMode } from '../core/control-plane-mode.mjs' export function readJsonSafe(p, fallback = null) { try { @@ -62,14 +63,56 @@ export function outboundDeviceId(raw, fallback = 'kin-vm') { } export function loadVmIdentity(exec = {}) { + if (isKubernetesHaMode()) return loadHaVmIdentity(exec) const home = exec.homeDir || '' const claudeDir = path.join(home, '.claude') const settings = readJsonSafe(path.join(claudeDir, 'settings.json'), {}) || {} const homeClaude = readJsonSafe(path.join(home, '.claude.json'), {}) || {} const dirClaude = readJsonSafe(path.join(claudeDir, '.claude.json'), {}) || {} const claudeJson = { ...dirClaude, ...homeClaude } + return buildVmIdentity(exec, settings, claudeJson, readSlotCredentialIdentity(home), exec.oauth || {}) +} + +/** Pure construction from the captured public node status and PG VM record. */ +export function loadHaVmIdentity(exec = {}) { + const node = exec.nodeIdentity + const credential = node?.credential + const fp = exec.vm?.fingerprint + const valid = (value) => + typeof value === 'string' && + !!value && + value.trim() === value && + value.length <= 512 && + !/[\x00-\x1f\x7f]/.test(value) + if ( + exec.slotgate?.trusted !== true || + exec.slotgate.pgBindingReady !== true || + exec.slotgate.vmId !== exec.vmId || + exec.vm?.id !== exec.vmId || + node?.ok !== true || + !valid(exec.accountId) || + credential?.account_uuid !== exec.accountId || + (node.identity?.account_uuid !== undefined && node.identity.account_uuid !== exec.accountId) || + !valid(fp?.device_id) || + !valid(fp?.session_id) || + (credential.org_uuid && node.identity?.org_uuid && credential.org_uuid !== node.identity.org_uuid) + ) + throw Object.assign(new Error('Captured HA identity is unavailable'), { code: 'ha_identity_unavailable' }) + return buildVmIdentity( + { ...exec, seedPolicy: exec.seedPolicy || exec.vm.seed_policy || {} }, + {}, + {}, + { + account_uuid: credential.account_uuid, + org_uuid: credential.org_uuid ?? node.identity?.org_uuid ?? '', + email: credential.email ?? node.identity?.email ?? null, + }, + {}, + ) +} + +function buildVmIdentity(exec, settings, claudeJson, slotCred, oauth) { const fp = exec.vm?.fingerprint || {} - const oauth = exec.oauth || {} const seed = exec.seedPolicy || {} const timezone = fp.timezone || exec.timezone || 'UTC' @@ -78,7 +121,6 @@ export function loadVmIdentity(exec = {}) { // machineID is never outbound — guest collect also never rotates device_id. const deviceId = outboundDeviceId(fp.device_id, exec.vmId || 'kin-vm') const sessionId = fp.session_id || crypto.randomUUID() - const slotCred = readSlotCredentialIdentity(home) // Slot worker credentials.json is the only credential location. // Do not take account identity from leftover .claude.json / client inbound. const accountUuid = slotCred?.account_uuid || oauth.account_uuid || '' diff --git a/test/unit/ha-vm-identity.test.mjs b/test/unit/ha-vm-identity.test.mjs index 0c48ad982..389280a61 100644 --- a/test/unit/ha-vm-identity.test.mjs +++ b/test/unit/ha-vm-identity.test.mjs @@ -4,14 +4,34 @@ import fs from 'node:fs' import { loadVmIdentity, OFFICIAL_CLI_VERSION } from '../../src/lib/identity/vm-identity.mjs' function exec(patch = {}) { - return { vmId: 'vm-ha-identity', accountId: 'node-account', homeDir: '/forbidden/pod/cli-home', + return { + vmId: 'vm-ha-identity', + accountId: 'node-account', + homeDir: '/forbidden/pod/cli-home', slotgate: { trusted: true, pgBindingReady: true, vmId: 'vm-ha-identity' }, - nodeIdentity: { ok: true, credential: { account_uuid: 'node-account', org_uuid: 'node-org', email: 'node@example.invalid', has_access: true }, - identity: { account_uuid: 'node-account', org_uuid: 'node-org', email: 'node@example.invalid' } }, + nodeIdentity: { + ok: true, + credential: { + account_uuid: 'node-account', + org_uuid: 'node-org', + email: 'node@example.invalid', + has_access: true, + }, + identity: { account_uuid: 'node-account', org_uuid: 'node-org', email: 'node@example.invalid' }, + }, oauth: { account_uuid: 'stale-pod-account', org_uuid: 'stale-pod-org', email: 'stale@example.invalid' }, seedPolicy: { theme: 'light', telemetry_disabled: false }, - vm: { id: 'vm-ha-identity', fingerprint: { device_id: 'a'.repeat(64), session_id: 'pg-session', locale: 'en_US.UTF-8', timezone: 'Asia/Taipei' } }, - ...patch } + vm: { + id: 'vm-ha-identity', + fingerprint: { + device_id: 'a'.repeat(64), + session_id: 'pg-session', + locale: 'en_US.UTF-8', + timezone: 'Asia/Taipei', + }, + }, + ...patch, + } } function withoutFiles(callback) { @@ -19,8 +39,13 @@ function withoutFiles(callback) { const read = fs.readFileSync const calls = [] process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' - fs.readFileSync = (...args) => { calls.push(String(args[0])); throw new Error('Pod files forbidden') } - try { callback(calls) } finally { + fs.readFileSync = (...args) => { + calls.push(String(args[0])) + throw new Error('Pod files forbidden') + } + try { + callback(calls) + } finally { fs.readFileSync = read if (beforeMode == null) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = beforeMode @@ -50,9 +75,21 @@ for (const [name, patch] of [ ['missing node proof', { nodeIdentity: undefined }], ['failed node health', { nodeIdentity: { ok: false, credential: { account_uuid: 'node-account' } } }], ['different credential account', { nodeIdentity: { ok: true, credential: { account_uuid: 'other-account' } } }], - ['identity account conflict', { nodeIdentity: { ok: true, credential: { account_uuid: 'node-account' }, identity: { account_uuid: 'other-account' } } }], + [ + 'identity account conflict', + { + nodeIdentity: { + ok: true, + credential: { account_uuid: 'node-account' }, + identity: { account_uuid: 'other-account' }, + }, + }, + ], ['missing PG fingerprint', { vm: { id: 'vm-ha-identity' } }], - ['different PG VM', { vm: { id: 'another-vm', fingerprint: { device_id: 'a'.repeat(64), session_id: 'pg-session' } } }], + [ + 'different PG VM', + { vm: { id: 'another-vm', fingerprint: { device_id: 'a'.repeat(64), session_id: 'pg-session' } } }, + ], ['different captured VM', { slotgate: { trusted: true, pgBindingReady: true, vmId: 'another-vm' } }], ]) { test(`HA identity refuses ${name} before local IO`, () => { From 4e6d1a1d7906c27b97966421a2af0f19e9b4e45c Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:57:18 +0800 Subject: [PATCH 386/473] test: retain node response consumption after controller disconnect --- .../slotgate/detached_completion_test.go | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 worker/internal/slotgate/detached_completion_test.go diff --git a/worker/internal/slotgate/detached_completion_test.go b/worker/internal/slotgate/detached_completion_test.go new file mode 100644 index 000000000..4fc6fb8bf --- /dev/null +++ b/worker/internal/slotgate/detached_completion_test.go @@ -0,0 +1,155 @@ +package slotgate + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func resetControllerConnection(t *testing.T, conn net.Conn) { + t.Helper() + secure, ok := conn.(*tls.Conn) + if !ok { + t.Fatal("expected TLS controller") + } + tcp, ok := secure.NetConn().(*net.TCPConn) + if !ok { + t.Fatal("expected TCP controller") + } + if err := tcp.SetLinger(0); err != nil { + t.Fatal(err) + } + if err := tcp.Close(); err != nil { + t.Fatal(err) + } +} + +func TestNodeDrainsOriginalKernelAfterControllerDisconnect(t *testing.T) { + for _, mode := range []string{"before-ok", "json", "sse"} { + t.Run(mode, func(t *testing.T) { + addr, root, server, state, _ := fencedGate(t) + resume := make(chan struct{}) + var once sync.Once + release := func() { once.Do(func() { close(resume) }) } + defer release() + finished := make(chan struct{}) + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(finished) + if mode == "before-ok" { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, normalClaudeJSON) + return + } + if mode == "json" { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"id":"msg_detached","type":"message","role":"assistant","content":[{"type":"text","text":"`+strings.Repeat("x", 8192)) + } else { + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, strings.Split(normalClaudeSSE, "event: message_stop")[0]+":"+strings.Repeat("x", 8192)+"\n\n") + } + w.(http.Flusher).Flush() + <-resume + if mode == "json" { + fmt.Fprint(w, `done"}],"stop_reason":"end_turn"}`) + } else { + fmt.Fprint(w, "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") + } + })) + request := forwardRequest(t, addr, state, "disconnected-"+mode) + if mode == "before-ok" { + func() { + server.authority.mu.Lock() + defer server.authority.mu.Unlock() + connection, _ := gate2Connect(t, addr, testToken, request, claudeWire(request.CommandID)) + resetControllerConnection(t, connection) + }() + } else { + connection, reader := gate2Connect(t, addr, testToken, request, claudeWire(request.CommandID)) + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + if _, err = io.ReadFull(response.Body, make([]byte, 1)); err != nil { + t.Fatal(err) + } + resetControllerConnection(t, connection) + release() + } + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Fatal("kernel fixture did not finish") + } + waitCommandStatus(t, server.authority, request.CommandID, CommandCompleted) + }) + } +} + +func TestNodeDetachedDrainKeepsIncompleteOrErrorTerminalUnknown(t *testing.T) { + for _, mode := range []string{"truncated", "trailing-error", "bad-trailer"} { + t.Run(mode, func(t *testing.T) { + addr, root, server, state, _ := fencedGate(t) + resume := make(chan struct{}) + var once sync.Once + release := func() { once.Do(func() { close(resume) }) } + defer release() + finished := make(chan struct{}) + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(finished) + w.Header().Set("Content-Type", "text/event-stream") + if mode == "bad-trailer" { + w.Header().Set("Trailer", "X-Terminal-State") + } + fmt.Fprint(w, strings.Split(normalClaudeSSE, "event: message_stop")[0]+":"+strings.Repeat("x", 8192)+"\n\n") + w.(http.Flusher).Flush() + <-resume + if mode != "truncated" { + fmt.Fprint(w, "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") + } + if mode == "trailing-error" { + fmt.Fprint(w, "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"provider_error\"}}\n\n") + } + if mode == "bad-trailer" { + w.Header().Set("X-Terminal-State", "unknown") + } + })) + request := forwardRequest(t, addr, state, "detached-unknown-"+mode) + connection, reader := gate2Connect(t, addr, testToken, request, claudeWire(request.CommandID)) + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + if _, err = io.ReadFull(response.Body, make([]byte, 1)); err != nil { + t.Fatal(err) + } + resetControllerConnection(t, connection) + release() + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Fatal("kernel fixture did not finish") + } + until := time.Now().Add(time.Second) + for time.Now().Before(until) { + stored, _ := server.authority.State("v") + if stored.Commands[request.CommandID].Status == CommandCompleted { + t.Fatal("disconnected error/truncation was called completed") + } + time.Sleep(time.Millisecond) + } + }) + } +} From 5960ecfdd0873c38cd9058215994438592b9d067 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 00:58:46 +0800 Subject: [PATCH 387/473] fix: drain the original node execution after controller disconnect --- docs/plans/2026-10-22-node-health.md | 2 +- docs/plans/2026-10-22-node-recovery.md | 19 +++++++++++ .../slotgate/detached_completion_test.go | 13 ++++++++ worker/internal/slotgate/detached_writer.go | 32 +++++++++++++++++++ worker/internal/slotgate/ksg2.go | 7 ++-- 5 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 docs/plans/2026-10-22-node-recovery.md create mode 100644 worker/internal/slotgate/detached_writer.go diff --git a/docs/plans/2026-10-22-node-health.md b/docs/plans/2026-10-22-node-health.md index 113a4c531..4965d6e7c 100644 --- a/docs/plans/2026-10-22-node-health.md +++ b/docs/plans/2026-10-22-node-health.md @@ -26,4 +26,4 @@ generation 保留原始数字文本或受限 opaque string，避免 JavaScript 这是原捕获身份的一次节点观察，不是某个推理 job 的终止证明。ready_slots、正常 health、HTTP 取消或连接关闭不能释放 unknown execution。controller 仍须在提交前用 PG 准入，并等待原 command 的持久完成。 -尚未提供节点 credential refresh、运行期 config 修改、controller 断连后独立 drain、无 ticket recovery receipt 或完整 HA 故障接管。这些不通过 health 隐式执行。旧 SSH/SFTP/Docker 权限迁移仍未完成，因此不能据此开启生产多副本 HA。 +controller 断连后的独立读取见同目录 `2026-10-22-node-recovery.md`。尚未提供节点 credential refresh、运行期 config 修改、无 ticket recovery receipt 或完整 HA 故障接管。这些不通过 health 隐式执行。旧 SSH/SFTP/Docker 权限迁移仍未完成，因此不能据此开启生产多副本 HA。 diff --git a/docs/plans/2026-10-22-node-recovery.md b/docs/plans/2026-10-22-node-recovery.md new file mode 100644 index 000000000..4fe99adfd --- /dev/null +++ b/docs/plans/2026-10-22-node-recovery.md @@ -0,0 +1,19 @@ +# 节点独立消费响应与恢复观察 + +## Controller 断连后的读取 + +失败规格 `53bbcd65` 使用本地 TCP RST 分别复现 OK 前缀前、JSON 中途和 SSE 中途断连；原 command 都停留 pending。原因是转发写失败立即返回，defer 关闭了原 kernel body/socket。 + +本批 KSG2 在提交原请求后由节点 goroutine 继续拥有同一 kernel 响应。下游写有固定短 deadline；controller 断连或持续阻塞时，停止向该连接交付并丢弃后续输出。HTTP 序列化仍持续读取原 body，通过现有正常 JSON/SSE 终态解析器消费到结束。没有重试、第二个 job、重新查询最新 Fence 或取消后猜测结束。 + +不能在 `http.Response.Write` 返回写错误之后才尝试补读 body，因为该路径可能已经关闭 body。节点 writer 在客户端交付失败时切换 discard，使原 body 保持可读；HTTP/body 读取错误仍会传播。 + +正常终态和完整响应允许沿用 Authority 的 journal fsync 标记 completed。断连本身不提供完成证明；截断、尾随 error、错误 trailer、parser 限制或 journal 失败继续保留 pending/unknown。节点生命周期取消仍关闭本地连接；不能据此释放远端未知执行。此路径不会用 timeout 或 TTL 释放容量。 + +测试只用 fake kernel、临时 Unix/TLS 监听和 controller RST，并验证错误终态不能 completed。没有真实 Docker、SSH、节点或 provider 操作。 + +## 后续恢复状态和计费 receipt + +当前本批尚未提供无 ticket 的历史 recovery-status、seal-not-started 或持久 usage receipt。后续只读查询必须使用 PG 保存的原 VM/Fence/command ID，不能从当前 state 重新授权。notfound 不等于未开始；未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 + +usage 缺失不得补零结算；节点只提供原始公开 model/usage/terminal 信息，PG 根据原 command 对应的 parent/attempt 和价格快照处理。node health、ready_slots 和取消 ACK 均不是恢复终止证明。以上节点能力不表示生产 HA 已完成。 diff --git a/worker/internal/slotgate/detached_completion_test.go b/worker/internal/slotgate/detached_completion_test.go index 4fc6fb8bf..970b8b37a 100644 --- a/worker/internal/slotgate/detached_completion_test.go +++ b/worker/internal/slotgate/detached_completion_test.go @@ -153,3 +153,16 @@ func TestNodeDetachedDrainKeepsIncompleteOrErrorTerminalUnknown(t *testing.T) { }) } } + +func TestNodeDetachedDeliveryBoundsStalledControllerWithoutEndingKernelRead(t *testing.T) { + writer, reader := net.Pipe() + defer writer.Close() + defer reader.Close() + sink := &detachedResponseWriter{connection: writer, writeTimeout: 5 * time.Millisecond} + if n, err := sink.Write([]byte("first")); err != nil || n != 5 || !sink.detached { + t.Fatalf("stalled delivery: %d %v %t", n, err, sink.detached) + } + if n, err := sink.Write([]byte("remaining kernel bytes")); err != nil || n != 22 { + t.Fatalf("discarded delivery: %d %v", n, err) + } +} diff --git a/worker/internal/slotgate/detached_writer.go b/worker/internal/slotgate/detached_writer.go new file mode 100644 index 000000000..86052c1f0 --- /dev/null +++ b/worker/internal/slotgate/detached_writer.go @@ -0,0 +1,32 @@ +package slotgate + +import ( + "net" + "time" +) + +// A submitted job belongs to the node until its original kernel response ends. +// Stop delivering after a dead/stalled controller, while Response.Write keeps +// consuming the same body through the terminal observer. Returning the delivery +// error would make net/http close that body before the node can observe its end. +type detachedResponseWriter struct { + connection net.Conn + writeTimeout time.Duration + detached bool +} + +func (w *detachedResponseWriter) Write(data []byte) (int, error) { + if !w.detached { + err := w.connection.SetWriteDeadline(time.Now().Add(w.writeTimeout)) + if err == nil { + var written int + written, err = w.connection.Write(data) + if err == nil && written == len(data) { + return written, nil + } + } + w.detached = true + w.connection.Close() + } + return len(data), nil +} diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index 2dfe30d75..82c34463c 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -367,9 +367,8 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio } stop := context.AfterFunc(ctx, func() { target.Close() }) defer stop() - if _, err = io.WriteString(conn, "OK\n"); err != nil { - return - } + delivery := &detachedResponseWriter{connection: conn, writeTimeout: prefaceTimeout} + io.WriteString(delivery, "OK\n") response, err := http.ReadResponse(bufio.NewReader(target), req) if err != nil { return @@ -381,7 +380,7 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio response.Close = true terminal := observeClaudeTerminal(response, req, body, request.CommandID, request.Kind) response.Body = terminal - if err = response.Write(conn); err != nil { + if err = response.Write(delivery); err != nil { return } if terminal.completed(response) { From 422088f7de0755687fce5d772bfafdec42d43563 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:03:39 +0800 Subject: [PATCH 388/473] test: require host-free multi-process HA bootstrap --- test/e2e/ha-bootstrap.e2e.test.mjs | 51 +++++++++++++++++++++++++++++ test/fixtures/ha-no-host-loader.mjs | 22 +++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 test/e2e/ha-bootstrap.e2e.test.mjs create mode 100644 test/fixtures/ha-no-host-loader.mjs diff --git a/test/e2e/ha-bootstrap.e2e.test.mjs b/test/e2e/ha-bootstrap.e2e.test.mjs new file mode 100644 index 000000000..b4fc7c2ef --- /dev/null +++ b/test/e2e/ha-bootstrap.e2e.test.mjs @@ -0,0 +1,51 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { startGateway } from '../harness.mjs' + +test('two HA processes start with PG-only metadata and never enter legacy host bootstrap', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-648-ha-bootstrap-')) + const gateways = [] + t.after(async () => { + for (const gateway of gateways.reverse()) await gateway.stop() + fs.rmSync(root, { recursive: true, force: true }) + }) + const trust = path.join(root, 'nodes.json') + fs.writeFileSync(trust, JSON.stringify({ nodes: { 'fixture-node': { + address: '127.0.0.1', port: 19999, token: 'synthetic-bootstrap-token', + certSha256: 'a'.repeat(64), policyDigest: 'b'.repeat(64), presetId: 'claude-docker-v1', + } } }), { mode: 0o600 }) + const barrier = path.join(root, 'legacy-effects.log') + const env = { + VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', VM2API_NODE_CONFIG_FILE: trust, + VM2API_TEST_HOST_BARRIER_FILE: barrier, + NODE_OPTIONS: `--import=${new URL('../fixtures/ha-no-host-loader.mjs', import.meta.url).href}`, + KIN_BACKUP_DISABLED: '1', + } + for (let index = 0; index < 2; index++) { + const project = path.join(root, `pod-${index}`) + fs.mkdirSync(path.join(project, 'vms'), { recursive: true }) + fs.writeFileSync(path.join(project, 'vms', 'active.json'), '{}') + const gateway = await startGateway({ project, env: { ...env, ...(gateways[0] ? { KIN_DATABASE_SCHEMA: gateways[0].schema } : {}) } }) + gateways.push(gateway) + assert.equal((await fetch(`${gateway.baseUrl}/readyz`)).status, 200) + } + const login = await fetch(`${gateways[0].baseUrl}/api/panel/login`, { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'admin', password: 'testpass' }), + }) + assert.equal(login.status, 200) + const cookie = login.headers.get('set-cookie').split(';')[0] + assert.equal((await fetch(`${gateways[1].baseUrl}/api/panel/me`, { headers: { cookie } })).status, 200) + const blocked = await fetch(`${gateways[1].baseUrl}/api/panel/vms/create`, { + method: 'POST', headers: { cookie, 'content-type': 'application/json' }, body: '{}', + }) + assert.equal(blocked.status, 503) + assert.equal((await blocked.json()).error.code, 'ha_operation_unsupported') + assert.equal(fs.existsSync(barrier) ? fs.readFileSync(barrier, 'utf8') : '', '') + for (const gateway of gateways) { + assert.equal(fs.existsSync(gateway.runtimeCallsFile) ? fs.readFileSync(gateway.runtimeCallsFile, 'utf8') : '', '') + } +}) diff --git a/test/fixtures/ha-no-host-loader.mjs b/test/fixtures/ha-no-host-loader.mjs new file mode 100644 index 000000000..9c4b967b7 --- /dev/null +++ b/test/fixtures/ha-no-host-loader.mjs @@ -0,0 +1,22 @@ +import { registerHooks } from 'node:module' + +const effects = new Map([ + ['../../src/lib/vm/egress.mjs', ['ensureProxyEgress', 'ensureLocalProxyEgress', 'startProxyEgress', 'stopProxyEgress']], + ['../../src/lib/vm/vm-db-sync.mjs', ['initVmDbSync']], + ['../../src/lib/vm/vm-runtime.mjs', ['refreshSlotGuestBinDir']], + ['../../src/lib/cluster/node-egress-socks.mjs', ['startNodeEgressSocks']], + ['../../src/lib/cluster/placement.mjs', ['nodeSession']], +].map(([file, names]) => [new URL(file, import.meta.url).href, names])) + +registerHooks({ + resolve(specifier, context, nextResolve) { + const resolved = nextResolve(specifier, context) + const names = effects.get(resolved.url) + if (!names) return resolved + const original = JSON.stringify(resolved.url + '?ha-original') + const source = `import fs from 'node:fs'; export * from ${original}; + function deny(name) { fs.appendFileSync(process.env.VM2API_TEST_HOST_BARRIER_FILE, name+'\\n'); throw new Error('HA attempted legacy host effect: '+name) } + ${names.map((name) => `export function ${name}() { return deny(${JSON.stringify(name)}) }`).join('\n')}` + return { url: 'data:text/javascript,' + encodeURIComponent(source), shortCircuit: true } + }, +}) From f4ef24ab1424f4010cb56b9b97bf5fc21d14f652 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:08:39 +0800 Subject: [PATCH 389/473] feat: bootstrap HA controllers without legacy host permissions --- src/lib/admin/routing-runtime.mjs | 5 +- src/lib/core/config.mjs | 2 + src/lib/vm/vm-registry.mjs | 1 + src/server.mjs | 318 +++++++++++++++++------------ test/e2e/ha-bootstrap.e2e.test.mjs | 35 +++- 5 files changed, 218 insertions(+), 143 deletions(-) diff --git a/src/lib/admin/routing-runtime.mjs b/src/lib/admin/routing-runtime.mjs index e2be3c479..e99f810a8 100644 --- a/src/lib/admin/routing-runtime.mjs +++ b/src/lib/admin/routing-runtime.mjs @@ -54,7 +54,8 @@ import { normalizeVmConfig } from '../vm/machine-spec.mjs' import { syncWorkerTelemetry } from '../vm/vm-runtime.mjs' import { slotHost } from '../vm/slot-host.mjs' import { telemetryUploadInterval } from '../vm/worker-telemetry.mjs' -import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { isKubernetesMode, isKubernetesHaMode } from '../core/control-plane-mode.mjs' +import { assertHaRoutingPatch } from '../core/ha-control-plane-policy.mjs' import { SharedRoutingConfig } from './shared-routing-config.mjs' export function createRoutingRuntime(ctx) { @@ -567,6 +568,7 @@ export function createRoutingRuntime(ctx) { } async function persistRoutingPatch(body = {}, options = {}) { + if (isKubernetesHaMode()) assertHaRoutingPatch(body) if (routingState) return routingState.patch(body, options) const hasCodexQuota = body?.codex && Object.prototype.hasOwnProperty.call(body.codex, 'quota') if (hasCodexQuota) validateOpenAIQuotaPatch(body.codex.quota) @@ -847,6 +849,7 @@ export function createRoutingRuntime(ctx) { runtimeRepo, routingState, workerHealth: rustKernelHealth, + captureNodeExecution: ctx.captureNodeExecution, config: poolSchedulerConfig(), }) setPool(poolScheduler) diff --git a/src/lib/core/config.mjs b/src/lib/core/config.mjs index dff6516af..628a72557 100644 --- a/src/lib/core/config.mjs +++ b/src/lib/core/config.mjs @@ -5,6 +5,7 @@ import { hashKey } from './security.mjs' import { atomicWriteJson } from '../vm/vm-file.mjs' import { loadDistillRules } from './distill-detect.mjs' import { readRoutingDocument, routingDocumentRevision } from './routing-document.mjs' +import { isKubernetesHaMode } from './control-plane-mode.mjs' const __dirname = path.dirname(fileURLToPath(import.meta.url)) // this file lives at src/lib/core/ — the gateway root is src/ @@ -149,6 +150,7 @@ export function resolvePublicBaseUrl(env = process.env, port = Number(env.PORT | } function readActiveVmRecord(projectRoot) { + if (isKubernetesHaMode()) return null try { const active = JSON.parse(fs.readFileSync(path.join(projectRoot, 'vms', 'active.json'), 'utf8')) const id = active?.active_vm diff --git a/src/lib/vm/vm-registry.mjs b/src/lib/vm/vm-registry.mjs index de4b9c9a0..177df482d 100644 --- a/src/lib/vm/vm-registry.mjs +++ b/src/lib/vm/vm-registry.mjs @@ -183,6 +183,7 @@ export function summarizeVm(vm, projectRoot = null, routing = null) { } export function getActiveVmId(projectRoot) { + if (isKubernetesHaMode()) return null try { const a = readJsonCached(path.join(projectRoot, 'vms', 'active.json')) return a?.active_vm ?? null diff --git a/src/server.mjs b/src/server.mjs index 41d4790e2..8a7e1e048 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -10,6 +10,7 @@ import http from 'node:http' import fs from 'node:fs' import path from 'node:path' +import crypto from 'node:crypto' import { loadConfig, reloadActiveVm, routingConfigFile } from './lib/core/config.mjs' import { extractApiKey, @@ -104,7 +105,11 @@ import { createHandleProtocol } from './lib/protocol/handle-protocol.mjs' import { createResponsesWsHandler } from './lib/protocol/responses-ws.mjs' import { handleUserCountTokens, handleUserUsage } from './lib/protocol/user-count-tokens.mjs' import { createPanelHandler } from './lib/admin/panel-routes.mjs' -import { isKubernetesMode, shutdownTimeoutMs } from './lib/core/control-plane-mode.mjs' +import { isKubernetesMode, isKubernetesHaMode, shutdownTimeoutMs } from './lib/core/control-plane-mode.mjs' +import { loadHaNodeTrust, haRouteAllowed, haUnsupported } from './lib/core/ha-control-plane-policy.mjs' +import { HaVmCatalog, bindHaVmCatalog } from './lib/vm/ha-vm-catalog.mjs' +import { captureNodeExecution } from './lib/cluster/node-execution-capabilities.mjs' +import { withHaRequestContext } from './lib/core/ha-request-context.mjs' import { ControlPlaneLifecycle, withTimeout } from './lib/core/control-plane-lifecycle.mjs' import { getSharedBackgroundTasks } from './lib/core/shared-background-tasks.mjs' import { ControlPlaneLock } from './lib/db/control-plane-lock.mjs' @@ -140,6 +145,8 @@ if (!getPanelAdmin().password) { throw new Error('VM2API_ADMIN_PASSWORD must be set; insecure default panel credentials are disabled') } const kubernetesMode = isKubernetesMode() +const haMode = isKubernetesHaMode() +const trustedNodes = haMode ? loadHaNodeTrust(process.env.VM2API_NODE_CONFIG_FILE) : null const shutdownGraceMs = shutdownTimeoutMs() const allowRate = createRateLimiter({ @@ -154,7 +161,14 @@ const dataDir = cfg.paths.data || path.join(cfg.paths.root, 'data') const db = await openDatabase({ dataDir }) const activityCoordinator = getActivityCoordinator(db) db.bootstrapActivity?.enter() +const haCatalog = haMode ? new HaVmCatalog(db) : null +if (haCatalog) { + bindHaVmCatalog(haCatalog) + await haCatalog.refresh() +} let controlPlaneLock = null +const controlPlaneHealthy = () => + !kubernetesMode || (activityCoordinator?.healthy === true && (haMode || controlPlaneLock?.healthy === true)) let startupAborting = false const lifecycle = new ControlPlaneLifecycle({ probe: async () => { @@ -164,7 +178,7 @@ const lifecycle = new ControlPlaneLifecycle({ } return db.prepare('SELECT 1').get() }, - lockHealthy: () => !kubernetesMode || (controlPlaneLock?.healthy === true && activityCoordinator?.healthy === true), + lockHealthy: controlPlaneHealthy, isMaintaining: () => backupService.isMaintaining, }) if (activityCoordinator) { @@ -192,6 +206,8 @@ async function failStartup(error) { if (kubernetesMode) { process.once('uncaughtException', failStartup) process.once('unhandledRejection', failStartup) +} +if (kubernetesMode && !haMode) { controlPlaneLock = new ControlPlaneLock(db, { onLost(error) { lifecycle.beginDrain() @@ -215,7 +231,7 @@ if (kubernetesMode) { } } fs.mkdirSync(cfg.paths.captures, { recursive: true }) -const legacyImport = await runLegacyImport({ dataDir, projectRoot: cfg.paths.project }) +const legacyImport = haMode ? null : await runLegacyImport({ dataDir, projectRoot: cfg.paths.project }) if (legacyImport?.imported) { console.log('[db] legacy JSON import done:', JSON.stringify(legacyImport.counts || {})) } @@ -223,7 +239,7 @@ await loadSettingsState() await loadModelPolicy() await loadGptModelPolicy() // VM/credential mirror: write-through hook + startup reconcile + fs.watch -const vmSync = await initVmDbSync(cfg.paths.project) +const vmSync = haMode ? { upserted: 0, rebuilt: 0, reclaimed_vm_ids: [] } : await initVmDbSync(cfg.paths.project) if (vmSync.upserted || vmSync.rebuilt) { console.log(`[db] vm mirror reconciled: upserted=${vmSync.upserted} rebuilt=${vmSync.rebuilt}`) } @@ -258,6 +274,7 @@ let failoverRunner let groupsRepo const routingRt = createRoutingRuntime({ + captureNodeExecution: haMode ? (input) => captureNodeExecution({ db, trustedNodes, ...input }) : null, get routingConfig() { return routingConfig }, @@ -444,25 +461,27 @@ proxyPool = new ProxyPool({ }) await proxyPool.init() // An upgrade replaces bin/ files; slots mounting data/run/slot-guest-bin must see the new ones. -try { - refreshSlotGuestBinDir(cfg.paths.project) -} catch (err) { - console.warn('[bootstrap] slot-guest-bin refresh failed:', err?.message || err) -} -proxyPool.startScheduler() +if (!haMode) + try { + refreshSlotGuestBinDir(cfg.paths.project) + } catch (err) { + console.warn('[bootstrap] slot-guest-bin refresh failed:', err?.message || err) + } +if (!haMode) proxyPool.startScheduler() // 首次安装的出口池是空的，槽没有可绑出口就起不来。本机出口永远成立，先补上。 -try { - if (!proxyPool.snapshot().proxies?.length) { - const seeded = await proxyPool.ensureLocal() - if (seeded.created) { - void (await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(seeded.proxy.id))) - console.log('[bootstrap] seeded local egress px-local') +if (!haMode) + try { + if (!proxyPool.snapshot().proxies?.length) { + const seeded = await proxyPool.ensureLocal() + if (seeded.created) { + void (await ensureProxyEgress(cfg.paths.project, proxyPool.getProxyByIdWithAuth(seeded.proxy.id))) + console.log('[bootstrap] seeded local egress px-local') + } } + } catch (e) { + console.warn('[bootstrap] local egress seed failed', e?.message || e) } -} catch (e) { - console.warn('[bootstrap] local egress seed failed', e?.message || e) -} await initPoolRuntime() routingRt.routingState?.start() @@ -679,7 +698,8 @@ backupService.onRestored(async (db) => { notifyMonitor?.setConfig(routingConfig.notify) await initPoolRuntime() - await clusterManager.rebind(db) + if (haCatalog) await haCatalog.refresh() + await clusterManager?.rebind(db) reloadActiveVm(cfg) await vmTestSchedule.reload() }) @@ -708,15 +728,15 @@ backupService.onMaintenance(async (enabled, _reason, context = {}) => { } else { if (_shuttingDown) return routingRt.routingState?.start() - resumeVmDbSync() - resumePriceCatalog() + if (!haMode) resumeVmDbSync() + if (!haMode) resumePriceCatalog() for (const monitor of maintenanceMonitors()) await monitor?.start?.({ immediate: false }) - proxyPool.startScheduler() + if (!haMode) proxyPool.startScheduler() requestLog.startCleanupScheduler() - await vmTestSchedule.start() + if (!haMode) await vmTestSchedule.start() } }) -await backupService.startScheduler() +if (!haMode) await backupService.startScheduler() const stats = { requests: 0, @@ -1020,7 +1040,7 @@ const protocolHandlers = createHandleProtocol({ }) async function dispatchProtocol(name, req, res, ...args) { - if (lifecycle.draining || (kubernetesMode && controlPlaneLock?.healthy !== true)) { + if (lifecycle.draining || !controlPlaneHealthy()) { return json(res, 503, { error: { type: 'overloaded_error', code: 'control_plane_draining', message: 'Control plane is draining' }, }) @@ -1074,58 +1094,73 @@ async function syncNodeLocalEgress() { } } -const clusterManager = new ClusterManager({ - repo: new ClusterNodesRepo(), - dataDir, - gateBin: process.env.KIN_CLUSTER_GATE_BIN || path.join(cfg.paths.project, 'bin', 'kin-slot-gate'), - listen: { host: cfg.host, port: cfg.port }, - vmsOnNode: (nodeId) => listVms(cfg.paths.project).filter((vm) => vm.node_id === nodeId), - assertNodeEditable: (nodeId) => { - const busy = listVms(cfg.paths.project).some( - (vm) => - vm.node_id === nodeId && - (Number(poolScheduler?.vmInflight?.get(vm.id) || 0) > 0 || - vmLockBusy(path.join(cfg.paths.project, 'vms', `${vm.id}.json`))), - ) - if (busy || nodeImageBuildRunning(nodeId)) { - throw new ClusterError(409, 'node_in_use', '节点仍有进行中的请求、虚拟机操作或镜像构建，请完成后重试') - } - }, - onNodesChanged: () => { - if (!backupService.isMaintaining) return syncNodeLocalEgress() - }, - // A node's local row follows its link: re-probe as soon as the link settles. - onLinkState: (nodeId, state) => { - if (!backupService.isMaintaining && (state === 'ready' || state === 'error' || state === 'backoff')) { - return proxyPool.probeLocalNode(nodeId) - } - }, -}) +const clusterManager = haMode + ? null + : new ClusterManager({ + repo: new ClusterNodesRepo(), + dataDir, + gateBin: process.env.KIN_CLUSTER_GATE_BIN || path.join(cfg.paths.project, 'bin', 'kin-slot-gate'), + listen: { host: cfg.host, port: cfg.port }, + vmsOnNode: (nodeId) => listVms(cfg.paths.project).filter((vm) => vm.node_id === nodeId), + assertNodeEditable: (nodeId) => { + const busy = listVms(cfg.paths.project).some( + (vm) => + vm.node_id === nodeId && + (Number(poolScheduler?.vmInflight?.get(vm.id) || 0) > 0 || + vmLockBusy(path.join(cfg.paths.project, 'vms', `${vm.id}.json`))), + ) + if (busy || nodeImageBuildRunning(nodeId)) { + throw new ClusterError(409, 'node_in_use', '节点仍有进行中的请求、虚拟机操作或镜像构建，请完成后重试') + } + }, + onNodesChanged: () => { + if (!backupService.isMaintaining) return syncNodeLocalEgress() + }, + // A node's local row follows its link: re-probe as soon as the link settles. + onLinkState: (nodeId, state) => { + if (!backupService.isMaintaining && (state === 'ready' || state === 'error' || state === 'backoff')) { + return proxyPool.probeLocalNode(nodeId) + } + }, + }) localEgressCluster = clusterManager -bindPlacement({ manager: clusterManager, projectRoot: cfg.paths.project }) -await startNodeEgressSocks({ clientFor: (nodeId) => clusterManager.client(nodeId) }) -await clusterManager.start() -await clusterManager.restoreSlotRelays() -await syncNodeLocalEgress() -if (proxyPool.snapshot().config.ipv6_enabled !== true) { - const exits = await syncIpv6ProxyEgress(cfg.paths.project, proxyPool) - for (const exit of exits.filter((item) => !item.ok)) { - console.warn('[ipv6-policy] exit not blocked', exit.vm_id || exit.proxy_id, exit.error) +if (!haMode) { + bindPlacement({ manager: clusterManager, projectRoot: cfg.paths.project }) + await startNodeEgressSocks({ clientFor: (nodeId) => clusterManager.client(nodeId) }) + await clusterManager.start() + await clusterManager.restoreSlotRelays() + await syncNodeLocalEgress() + if (proxyPool.snapshot().config.ipv6_enabled !== true) { + const exits = await syncIpv6ProxyEgress(cfg.paths.project, proxyPool) + for (const exit of exits.filter((item) => !item.ok)) { + console.warn('[ipv6-policy] exit not blocked', exit.vm_id || exit.proxy_id, exit.error) + } } } -const clusterRoutes = createClusterRoutes({ - manager: clusterManager, - json, - readBody, - ok: panel.ok, - onExecutionState: noteRequestExecution, -}) -const slotShell = createSlotShell({ - projectRoot: cfg.paths.project, - importOauth: commitImportedOauth, - onExecutionState: noteRequestExecution, -}) -for (const vm of listVms(cfg.paths.project)) { +const noHostRoutes = { + handle: () => false, + handleUpgrade: () => false, + beginDrain() {}, + terminate() {}, + async drain() {}, +} +const clusterRoutes = haMode + ? noHostRoutes + : createClusterRoutes({ + manager: clusterManager, + json, + readBody, + ok: panel.ok, + onExecutionState: noteRequestExecution, + }) +const slotShell = haMode + ? noHostRoutes + : createSlotShell({ + projectRoot: cfg.paths.project, + importOauth: commitImportedOauth, + onExecutionState: noteRequestExecution, + }) +for (const vm of haMode ? [] : listVms(cfg.paths.project)) { if (isCodexVm(vm)) continue try { ensureSlotSubscriptionType(path.join(cfg.paths.project, 'vms', vm.id, 'cli-home'), vm.account_tier) @@ -1133,10 +1168,12 @@ for (const vm of listVms(cfg.paths.project)) { console.warn('[credential-subscription]', vm.id, error?.message || error) } } -cliNodeGuard = createCliNodeGuard({ - listTargets: () => listVms(cfg.paths.project), - liveTokens: () => slotShell.liveShellTokens?.() || [], -}) +cliNodeGuard = haMode + ? null + : createCliNodeGuard({ + listTargets: () => listVms(cfg.paths.project), + liveTokens: () => slotShell.liveShellTokens?.() || [], + }) const handlePanel = createPanelHandler({ onPersistenceFailure: async () => { @@ -1241,7 +1278,8 @@ function noteRequestExecution(req, execution) { async function withRequestActivity(req, res, callback, kind = 'http') { if (!activityCoordinator || requestActivities.has(req)) return callback() - const gate = await activityCoordinator.admit({ kind }) + const requestId = haMode ? crypto.randomUUID() : null + const gate = await activityCoordinator.admit({ kind, requestId }) const state = { gate, executions: new Map(), unknown: false } requestActivities.set(req, state) let responseDone @@ -1255,9 +1293,13 @@ async function withRequestActivity(req, res, callback, kind = 'http') { try { return await gate.run(async () => { try { - const value = await callback(state) - await responseDone - await flushRuntimeWrites() + const value = await withHaRequestContext({ db, requestId, activity: gate }, async () => { + if (haCatalog) await haCatalog.refresh() + const result = await callback(state) + await responseDone + await flushRuntimeWrites() + return result + }) return value } catch (error) { state.unknown = true @@ -1287,6 +1329,10 @@ async function handleHttp(req, res) { return json(res, 400, { error: { message: 'Invalid URL', type: 'invalid_request' } }) } const p = url.pathname + if (haMode && !haRouteAllowed(req.method, p)) { + const error = haUnsupported() + return json(res, error.status, { error: { code: error.code, message: error.message } }) + } if ((req.method === 'GET' || req.method === 'HEAD') && p === '/readyz') { const ready = await lifecycle.isReady() @@ -1297,7 +1343,7 @@ async function handleHttp(req, res) { return json(res, ready ? 200 : 503, { status: ready ? 'ready' : 'not_ready' }) } if ( - (lifecycle.draining || (kubernetesMode && (!lifecycle.initialized || controlPlaneLock?.healthy !== true))) && + (lifecycle.draining || (kubernetesMode && (!lifecycle.initialized || !controlPlaneHealthy()))) && p !== '/healthz' && p !== '/health' ) { @@ -1493,13 +1539,17 @@ server.on('connection', (socket) => { server.on('upgrade', async (req, socket, head) => { if ( lifecycle.draining || - (kubernetesMode && (!lifecycle.initialized || controlPlaneLock?.healthy !== true)) || + (kubernetesMode && (!lifecycle.initialized || !controlPlaneHealthy())) || backupService.isMaintaining ) { socket.end('HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n') return } try { + if (haMode && !haRouteAllowed(req.method, parseRequestUrl(req)?.pathname || '', { upgrade: true })) { + socket.end('HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\n') + return + } await withRequestActivity( req, null, @@ -1565,7 +1615,8 @@ const pub = { }, } // Never persist the raw API key. Public snapshot only. -fs.writeFileSync(path.join(cfg.paths.root, 'config', 'gateway-v2.public.json'), JSON.stringify(pub, null, 2)) +if (!haMode) + fs.writeFileSync(path.join(cfg.paths.root, 'config', 'gateway-v2.public.json'), JSON.stringify(pub, null, 2)) process.on('uncaughtException', (e) => console.error('[uncaught]', e)) process.on('unhandledRejection', (e) => console.error('[unhandled]', e)) @@ -1670,7 +1721,7 @@ async function shutdown(signal, { failed = false } = {}) { if (kubernetesMode) await getSharedBackgroundTasks(db).flush() await flushRuntimeWrites() stopAllRustKernels() - await clusterManager.stop() + await clusterManager?.stop() await shutdownActivity?.complete({ ended: true, reason: 'shutdown_persistence_completed' }) await activityCoordinator?.stop() await controlPlaneLock?.stop() @@ -1694,12 +1745,9 @@ process.on('SIGINT', async () => await shutdown('SIGINT')) function maintenanceMonitors() { return [ - healthMonitor, - credentialRefreshMonitor, - kernelWatchdog, - cliNodeGuard, - usageProbeMonitor, - catalogMonitor, + ...(haMode + ? [] + : [healthMonitor, credentialRefreshMonitor, kernelWatchdog, cliNodeGuard, usageProbeMonitor, catalogMonitor]), notifyMonitor, ] } @@ -1715,7 +1763,7 @@ async function flushRuntimeWrites() { await flushVmDbSync() } -await bootApiKernel(cfg) +if (!haMode) await bootApiKernel(cfg) if (activityCoordinator) { await flushRuntimeWrites() await activityCoordinator.acknowledgeEpoch(db.bootstrapActivity.epoch) @@ -1723,48 +1771,50 @@ if (activityCoordinator) { } server.listen(cfg.port, cfg.host, async () => { - try { - await vmTestSchedule.start() - } catch (e) { - console.warn('[vm-test-schedule] start failed', e?.message || e) - } - try { - healthMonitor?.start?.({ immediate: true }) - } catch (e) { - console.warn('[health-probe] start failed', e?.message || e) - } - try { - credentialRefreshMonitor?.start?.({ immediate: true }) - } catch (e) { - console.warn('[credential-refresh] start failed', e?.message || e) - } - try { - kernelWatchdog?.start?.({ immediate: true }) - } catch (e) { - console.warn('[kernel-watchdog] start failed', e?.message || e) - } - // Boot fills what Docker's restart policy missed (removed or exited slot containers); off with the watchdog. - if (process.env.KIN_CRS_MOCK !== '1' && normalizeKernelWatchdogConfig(routingConfig.kernel_watchdog).enabled) { - reviveDownSlots({ vms: listVms(cfg.paths.project), powerOn: bootSlot, watchdog: kernelWatchdog }).catch((e) => - console.warn('[slot-revive] boot pass failed', e?.message || e), - ) - } - try { - cliNodeGuard?.start?.({ immediate: true }) - } catch (e) { - console.warn('[cli-node-guard] start failed', e?.message || e) - } - try { - usageProbeMonitor?.start?.({ immediate: true }) - } catch (e) { - console.warn('[usage-probe] start failed', e?.message || e) - } - // The KIN_CRS_MOCK harness promises no outbound network; catalogs stay on builtin data there. - if (process.env.KIN_CRS_MOCK !== '1') { + if (!haMode) { + try { + await vmTestSchedule.start() + } catch (e) { + console.warn('[vm-test-schedule] start failed', e?.message || e) + } try { - catalogMonitor?.start?.() + healthMonitor?.start?.({ immediate: true }) } catch (e) { - console.warn('[catalog-sync] start failed', e?.message || e) + console.warn('[health-probe] start failed', e?.message || e) + } + try { + credentialRefreshMonitor?.start?.({ immediate: true }) + } catch (e) { + console.warn('[credential-refresh] start failed', e?.message || e) + } + try { + kernelWatchdog?.start?.({ immediate: true }) + } catch (e) { + console.warn('[kernel-watchdog] start failed', e?.message || e) + } + // Boot fills what Docker's restart policy missed (removed or exited slot containers); off with the watchdog. + if (process.env.KIN_CRS_MOCK !== '1' && normalizeKernelWatchdogConfig(routingConfig.kernel_watchdog).enabled) { + reviveDownSlots({ vms: listVms(cfg.paths.project), powerOn: bootSlot, watchdog: kernelWatchdog }).catch((e) => + console.warn('[slot-revive] boot pass failed', e?.message || e), + ) + } + try { + cliNodeGuard?.start?.({ immediate: true }) + } catch (e) { + console.warn('[cli-node-guard] start failed', e?.message || e) + } + try { + usageProbeMonitor?.start?.({ immediate: true }) + } catch (e) { + console.warn('[usage-probe] start failed', e?.message || e) + } + // The KIN_CRS_MOCK harness promises no outbound network; catalogs stay on builtin data there. + if (process.env.KIN_CRS_MOCK !== '1') { + try { + catalogMonitor?.start?.() + } catch (e) { + console.warn('[catalog-sync] start failed', e?.message || e) + } } } try { diff --git a/test/e2e/ha-bootstrap.e2e.test.mjs b/test/e2e/ha-bootstrap.e2e.test.mjs index b4fc7c2ef..2ab6e7667 100644 --- a/test/e2e/ha-bootstrap.e2e.test.mjs +++ b/test/e2e/ha-bootstrap.e2e.test.mjs @@ -13,13 +13,26 @@ test('two HA processes start with PG-only metadata and never enter legacy host b fs.rmSync(root, { recursive: true, force: true }) }) const trust = path.join(root, 'nodes.json') - fs.writeFileSync(trust, JSON.stringify({ nodes: { 'fixture-node': { - address: '127.0.0.1', port: 19999, token: 'synthetic-bootstrap-token', - certSha256: 'a'.repeat(64), policyDigest: 'b'.repeat(64), presetId: 'claude-docker-v1', - } } }), { mode: 0o600 }) + fs.writeFileSync( + trust, + JSON.stringify({ + nodes: { + 'fixture-node': { + address: '127.0.0.1', + port: 19999, + token: 'synthetic-bootstrap-token', + certSha256: 'a'.repeat(64), + policyDigest: 'b'.repeat(64), + presetId: 'claude-docker-v1', + }, + }, + }), + { mode: 0o600 }, + ) const barrier = path.join(root, 'legacy-effects.log') const env = { - VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', VM2API_NODE_CONFIG_FILE: trust, + VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', + VM2API_NODE_CONFIG_FILE: trust, VM2API_TEST_HOST_BARRIER_FILE: barrier, NODE_OPTIONS: `--import=${new URL('../fixtures/ha-no-host-loader.mjs', import.meta.url).href}`, KIN_BACKUP_DISABLED: '1', @@ -28,19 +41,25 @@ test('two HA processes start with PG-only metadata and never enter legacy host b const project = path.join(root, `pod-${index}`) fs.mkdirSync(path.join(project, 'vms'), { recursive: true }) fs.writeFileSync(path.join(project, 'vms', 'active.json'), '{}') - const gateway = await startGateway({ project, env: { ...env, ...(gateways[0] ? { KIN_DATABASE_SCHEMA: gateways[0].schema } : {}) } }) + const gateway = await startGateway({ + project, + env: { ...env, ...(gateways[0] ? { KIN_DATABASE_SCHEMA: gateways[0].schema } : {}) }, + }) gateways.push(gateway) assert.equal((await fetch(`${gateway.baseUrl}/readyz`)).status, 200) } const login = await fetch(`${gateways[0].baseUrl}/api/panel/login`, { - method: 'POST', headers: { 'content-type': 'application/json' }, + method: 'POST', + headers: { 'content-type': 'application/json' }, body: JSON.stringify({ username: 'admin', password: 'testpass' }), }) assert.equal(login.status, 200) const cookie = login.headers.get('set-cookie').split(';')[0] assert.equal((await fetch(`${gateways[1].baseUrl}/api/panel/me`, { headers: { cookie } })).status, 200) const blocked = await fetch(`${gateways[1].baseUrl}/api/panel/vms/create`, { - method: 'POST', headers: { cookie, 'content-type': 'application/json' }, body: '{}', + method: 'POST', + headers: { cookie, 'content-type': 'application/json' }, + body: '{}', }) assert.equal(blocked.status, 503) assert.equal((await blocked.json()).error.code, 'ha_operation_unsupported') From a5e031882e6e91c53f2174b4238778bf22050560 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:09:01 +0800 Subject: [PATCH 390/473] test: exclude mixed HA and singleton controllers with one mode guard --- test/unit/control-plane-lock.test.mjs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/unit/control-plane-lock.test.mjs b/test/unit/control-plane-lock.test.mjs index 6e7c0da3f..0487868b5 100644 --- a/test/unit/control-plane-lock.test.mjs +++ b/test/unit/control-plane-lock.test.mjs @@ -3,6 +3,29 @@ import assert from 'node:assert/strict' import { getDb, createDatabase } from '../../src/lib/db/database.mjs' import { ControlPlaneLock } from '../../src/lib/db/control-plane-lock.mjs' +test('HA controllers share the mode guard and exclude a singleton controller in either startup order', async () => { + const db = getDb() + const first = new ControlPlaneLock(db, { shared: true }) + const second = new ControlPlaneLock(db, { shared: true }) + const singleton = new ControlPlaneLock(db) + try { + await first.start() + await second.start() + assert.equal(first.healthy && second.healthy, true) + await assert.rejects(singleton.start(), { code: 'control_plane_already_active' }) + await first.stop() + await assert.rejects(singleton.start(), { code: 'control_plane_already_active' }) + await second.stop() + await singleton.start() + await assert.rejects(first.start(), { code: 'control_plane_already_active' }) + await singleton.stop() + await first.start() + assert.equal(first.healthy, true) + } finally { + await Promise.all([first.stop(), second.stop(), singleton.stop()]) + } +}) + test('a one-connection pool rejects control-plane startup without consuming the business connection', async () => { const db = getDb() const single = await createDatabase({ connectionString: db.connectionString, schema: db.schema + '_single', max: 1 }) From d39559d04bfc74cd5dc79d8f6125449d2d9dd57a Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:11:33 +0800 Subject: [PATCH 391/473] fix: exclude singleton controllers while sharing the HA mode guard --- src/lib/db/control-plane-lock.mjs | 11 +++++++---- src/server.mjs | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/db/control-plane-lock.mjs b/src/lib/db/control-plane-lock.mjs index 47a37e0a1..7a9c0b109 100644 --- a/src/lib/db/control-plane-lock.mjs +++ b/src/lib/db/control-plane-lock.mjs @@ -1,16 +1,19 @@ -/** Single active controller guard. This does not fence remote execution or provide failover. */ +/** Excludes singleton controllers from shared HA mode; it does not fence remote execution. */ const LOCK_SQL = 'SELECT pg_try_advisory_lock(hashtext(current_database()), hashtext($1)) AS acquired' const UNLOCK_SQL = 'SELECT pg_advisory_unlock(hashtext(current_database()), hashtext($1))' +const SHARED_LOCK_SQL = 'SELECT pg_try_advisory_lock_shared(hashtext(current_database()), hashtext($1)) AS acquired' +const SHARED_UNLOCK_SQL = 'SELECT pg_advisory_unlock_shared(hashtext(current_database()), hashtext($1))' function lockError(code, message) { return Object.assign(new Error(message), { code }) } export class ControlPlaneLock { - constructor(db, { onLost = null } = {}) { + constructor(db, { onLost = null, shared = false } = {}) { this.db = db this.onLost = onLost + this.shared = shared === true this.resource = `vm2api:control-plane:${db.schema}` this._healthy = false this._client = null @@ -57,7 +60,7 @@ export class ControlPlaneLock { client.on('error', this._onError) client.on('end', this._onEnd) try { - const result = await client.query(LOCK_SQL, [this.resource]) + const result = await client.query(this.shared ? SHARED_LOCK_SQL : LOCK_SQL, [this.resource]) if (!result.rows[0]?.acquired) { throw lockError('control_plane_already_active', 'Another control plane is active for this database and schema') } @@ -109,7 +112,7 @@ export class ControlPlaneLock { this._detach(client) let failed = this._lost try { - if (!this._lost) await client.query(UNLOCK_SQL, [this.resource]) + if (!this._lost) await client.query(this.shared ? SHARED_UNLOCK_SQL : UNLOCK_SQL, [this.resource]) } catch (error) { failed = true throw error diff --git a/src/server.mjs b/src/server.mjs index 8a7e1e048..27db07841 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -168,7 +168,7 @@ if (haCatalog) { } let controlPlaneLock = null const controlPlaneHealthy = () => - !kubernetesMode || (activityCoordinator?.healthy === true && (haMode || controlPlaneLock?.healthy === true)) + !kubernetesMode || (activityCoordinator?.healthy === true && controlPlaneLock?.healthy === true) let startupAborting = false const lifecycle = new ControlPlaneLifecycle({ probe: async () => { @@ -207,8 +207,9 @@ if (kubernetesMode) { process.once('uncaughtException', failStartup) process.once('unhandledRejection', failStartup) } -if (kubernetesMode && !haMode) { +if (kubernetesMode) { controlPlaneLock = new ControlPlaneLock(db, { + shared: haMode, onLost(error) { lifecycle.beginDrain() console.error('[control-plane] exclusive lock lost') From 13ecd76dab669653ac5d79580d2dd60b22e4cb10 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:00:58 +0800 Subject: [PATCH 392/473] test: recover original node command status without controller tickets --- .../internal/slotgate/recovery_status_test.go | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 worker/internal/slotgate/recovery_status_test.go diff --git a/worker/internal/slotgate/recovery_status_test.go b/worker/internal/slotgate/recovery_status_test.go new file mode 100644 index 000000000..abe26619e --- /dev/null +++ b/worker/internal/slotgate/recovery_status_test.go @@ -0,0 +1,135 @@ +package slotgate + +import ( + "context" + "crypto/tls" + "encoding/json" + "net" + "path/filepath" + "testing" +) + +type recoveryStatusReply struct { + OK bool `json:"ok"` + Error string `json:"error"` + Found *bool `json:"found"` + Unknown bool `json:"unknown"` + Command *struct { + ID string `json:"id"` + Fence Fence `json:"fence"` + Status CommandStatus `json:"status"` + Dispatched bool `json:"dispatched"` + } `json:"command"` + State json.RawMessage `json:"state"` +} + +func readRecoveryStatus(t *testing.T, addr string, request gate2Request) recoveryStatusReply { + t.Helper() + c, reader := gate2Connect(t, addr, testToken, request, "") + defer c.Close() + var reply recoveryStatusReply + if err := json.NewDecoder(reader).Decode(&reply); err != nil { + t.Fatal(err) + } + return reply +} + +func historicalRecoveryGate(t *testing.T, completed, advance bool) (string, *Server, Fence) { + t.Helper() + socket, _, executor := engineForTest(t) + root := shortRoot(t) + a := authorityForTest(t, filepath.Join(root, "authority"), terminatedExecutor) + state := launchForTest(t, a, "v", executor) + handle, err := a.BeginCommand("v", state.Fence, "original-job", CommandForward) + if err != nil { + t.Fatal(err) + } + if err = a.DispatchCommand(handle, func(ExecutorIdentity) error { return nil }); err != nil { + t.Fatal(err) + } + if completed { + if err = a.CompleteCommand(handle); err != nil { + t.Fatal(err) + } + } + if advance { + if _, err = a.Revoke("v", state.Fence); err != nil { + t.Fatal(err) + } + if _, err = a.Advance(context.Background(), "v", state.Fence); err != nil { + t.Fatal(err) + } + } + if err = a.Close(); err != nil { + t.Fatal(err) + } + s, err := New(Config{Listen: "127.0.0.1:0", Root: root, Token: testToken, TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: a.root, DockerSocket: socket}) + if err != nil { + t.Fatal(err) + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.ServeListener(ctx, tls.NewListener(ln, s.TLSConfig())) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + return ln.Addr().String(), s, state.Fence +} + +func TestNodeRecoveryStatusReadsOriginalCompletedCommandAcrossNodeRestartAndGeneration(t *testing.T) { + addr, s, original := historicalRecoveryGate(t, true, true) + before, _ := s.authority.State("v") + if before.Fence == original { + t.Fatal("fixture did not advance the current generation") + } + reply := readRecoveryStatus(t, addr, gate2Request{Op: "recovery-status", VMID: "v", Fence: original, CommandID: "original-job"}) + if !reply.OK || reply.Found == nil || !*reply.Found || reply.Command == nil || reply.Command.ID != "original-job" || reply.Command.Fence != original || reply.Command.Status != CommandCompleted || !reply.Command.Dispatched { + t.Fatalf("historical completion unavailable without a dead controller ticket: %+v", reply) + } + if len(reply.State) != 0 { + t.Fatal("historical query returned latest state as new authorization") + } + after, _ := s.authority.State("v") + if len(before.Commands) != len(after.Commands) || before.Fence != after.Fence { + t.Fatal("recovery observation mutated node state") + } +} + +func TestNodeRecoveryStatusRetainsRestartedUnknownAndNotFound(t *testing.T) { + addr, s, original := historicalRecoveryGate(t, false, false) + reply := readRecoveryStatus(t, addr, gate2Request{Op: "recovery-status", VMID: "v", Fence: original, CommandID: "original-job"}) + if !reply.OK || reply.Found == nil || !*reply.Found || reply.Command == nil || reply.Command.Status != CommandUnknown || !reply.Command.Dispatched { + t.Fatalf("restarted pending command was lost: %+v", reply) + } + reply = readRecoveryStatus(t, addr, gate2Request{Op: "recovery-status", VMID: "v", Fence: original, CommandID: "not-found"}) + if !reply.OK || reply.Found == nil || *reply.Found || !reply.Unknown || reply.Command != nil { + t.Fatalf("notfound must remain unknown: %+v", reply) + } + state, _ := s.authority.State("v") + if _, ok := state.Commands["not-found"]; ok { + t.Fatal("readonly lookup minted or sealed a command") + } +} + +func TestNodeRecoveryStatusRejectsRelabeledFenceAndMutationFields(t *testing.T) { + addr, _, original := historicalRecoveryGate(t, true, false) + wrong := original + wrong.Generation++ + for _, request := range []gate2Request{ + {Op: "recovery-status", VMID: "v", Fence: wrong, CommandID: "original-job"}, + {Op: "recovery-status", VMID: "v", Fence: original, CommandID: "original-job", Ticket: "not-required-nonce"}, + {Op: "recovery-status", VMID: "v", Fence: original, CommandID: "original-job", Kind: "kernel"}, + } { + reply := readRecoveryStatus(t, addr, request) + if reply.OK || reply.Command != nil { + t.Fatalf("relabeled/mutation request accepted: %+v", reply) + } + } +} From f640c07b6ebe8355fd7fea20c86fd3489dd9657e Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:04:20 +0800 Subject: [PATCH 393/473] feat: expose original node command recovery status without tickets --- docs/plans/2026-10-22-node-recovery.md | 12 +++++- worker/internal/slotgate/ksg2.go | 17 +++++++- worker/internal/slotgate/recovery_status.go | 43 +++++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 worker/internal/slotgate/recovery_status.go diff --git a/docs/plans/2026-10-22-node-recovery.md b/docs/plans/2026-10-22-node-recovery.md index 4fe99adfd..5cfa61f81 100644 --- a/docs/plans/2026-10-22-node-recovery.md +++ b/docs/plans/2026-10-22-node-recovery.md @@ -12,8 +12,16 @@ 测试只用 fake kernel、临时 Unix/TLS 监听和 controller RST，并验证错误终态不能 completed。没有真实 Docker、SSH、节点或 provider 操作。 -## 后续恢复状态和计费 receipt +## 历史恢复状态 -当前本批尚未提供无 ticket 的历史 recovery-status、seal-not-started 或持久 usage receipt。后续只读查询必须使用 PG 保存的原 VM/Fence/command ID，不能从当前 state 重新授权。notfound 不等于未开始；未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 +失败规格 `caab8b81` 证明原接口不能在 controller/node runtime 变化后读取历史完成记录。新增 authenticated KSG2 `recovery-status` 请求固定为 `{op,vm_id,fence,command_id}`，无需 ticket。fence 必须与该 command 的历史记录完全相等；不会拿当前 slot state 的 generation、executor 或新 nonce 替换旧身份。 + +找到记录时返回 `{ok:true,found:true,command:{id,fence,status,dispatched}}`。历史 completed 跨 generation 保留；重启后 pending 变 unknown 的记录继续返回 unknown。接口仅查询 inference command，不提供管理动作的执行权限，不发送 kernel/Engine 请求，不创建或修改 command。 + +找不到记录时仅返回 `{ok:true,found:false,unknown:true}`。它不证明未开始，也不产生 seal/tombstone；旧 owner 仍可能延迟 prepare/forward。调用方必须继续保留 PG 未知占位。输入错误、原 Fence 不匹配、节点 journal 不可用时拒绝，不退回当前 state 或其他传输。 + +## 后续 seal 和计费 receipt + +当前本批尚未提供 seal-not-started 或持久 usage receipt。notfound 不等于未开始；未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 usage 缺失不得补零结算；节点只提供原始公开 model/usage/terminal 信息，PG 根据原 command 对应的 parent/attempt 和价格快照处理。node health、ready_slots 和取消 ACK 均不是恢复终止证明。以上节点能力不表示生产 HA 已完成。 diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index 82c34463c..c7c3a5b6e 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -137,6 +137,21 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["manifest_revision"] = request.ManifestRevision reply["manifest_digest"] = request.ManifestDigest } + case "recovery-status": + if request.Kind != "" || request.Ticket != "" || request.AllocationID != "" || request.OperationID != "" || request.OwnerID != "" || request.Management != nil || request.ContentLength != 0 { + err = ErrInvalidAuthority + break + } + var command *recoveryCommandStatus + command, err = s.authority.recoveryStatus(request.VMID, request.Fence, request.CommandID) + if err == nil { + reply["found"] = command != nil + if command != nil { + reply["command"] = command + } else { + reply["unknown"] = true + } + } case "allocate": if s.allocator == nil || request.Fence != (Fence{}) || request.CommandID != "" || request.Kind != "" || request.Ticket != "" { err = ErrManagementDenied @@ -233,7 +248,7 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["error"] = gate2Code(err) } else { reply["ok"] = true - if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" && request.Op != "health" { + if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" && request.Op != "health" && request.Op != "recovery-status" { reply["state"] = publicGateState(state) } } diff --git a/worker/internal/slotgate/recovery_status.go b/worker/internal/slotgate/recovery_status.go new file mode 100644 index 000000000..1fda86652 --- /dev/null +++ b/worker/internal/slotgate/recovery_status.go @@ -0,0 +1,43 @@ +package slotgate + +import ( + "encoding/hex" + "strings" +) + +// Historical status is a read of an original execution identity. It does not +// authorize dispatch and deliberately does not require a dead controller's +// nonce/runtime or match the current generation of the slot. +type recoveryCommandStatus struct { + ID string `json:"id"` + Fence Fence `json:"fence"` + Status CommandStatus `json:"status"` + Dispatched bool `json:"dispatched"` +} + +func (a *Authority) recoveryStatus(vm string, fence Fence, id string) (*recoveryCommandStatus, error) { + a.mu.Lock() + defer a.mu.Unlock() + if err := a.available(); err != nil { + return nil, err + } + incarnation, err := hex.DecodeString(fence.Incarnation) + if !vmIDRe.MatchString(vm) || !validAuthorityID(id) || !publicHealthString(id) || fence.Generation == 0 || err != nil || len(incarnation) != 16 || strings.ToLower(fence.Incarnation) != fence.Incarnation || validateExecutor(fence.Executor) != nil { + return nil, ErrInvalidAuthority + } + state, ok := a.journal.Slots[vm] + if !ok { + return nil, nil + } + command, ok := state.Commands[id] + if !ok { + return nil, nil + } + if command.Fence != fence { + return nil, ErrFenceMismatch + } + if command.Kind != CommandForward { + return nil, ErrManagementDenied + } + return &recoveryCommandStatus{ID: command.ID, Fence: command.Fence, Status: command.Status, Dispatched: command.Dispatched}, nil +} From 80e073dc227dba93740f779f48c3532e7b101bbf Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:06:00 +0800 Subject: [PATCH 394/473] test: reject new node execution authorized by a previous boot --- worker/internal/slotgate/node_boot_test.go | 118 +++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 worker/internal/slotgate/node_boot_test.go diff --git a/worker/internal/slotgate/node_boot_test.go b/worker/internal/slotgate/node_boot_test.go new file mode 100644 index 000000000..0a72bc0e5 --- /dev/null +++ b/worker/internal/slotgate/node_boot_test.go @@ -0,0 +1,118 @@ +package slotgate + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "path/filepath" + "sync/atomic" + "testing" +) + +func oldBootGate(t *testing.T) (string, string, *Server, AuthorityState) { + t.Helper() + socket, _, executor := engineForTest(t) + executor.NodeBootID = "previous-node-boot" + root := shortRoot(t) + s, err := New(Config{Listen: "127.0.0.1:0", Root: root, Token: testToken, TLSDir: filepath.Join(root, "tls"), Fenced: true, AuthorityDir: filepath.Join(root, "authority"), DockerSocket: socket}) + if err != nil { + t.Fatal(err) + } + // A trusted test adapter represents an ACTIVE journal surviving a host boot. + // The actual server captures this test host's current boot independently. + state := launchForTest(t, s.authority, "v", executor) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.ServeListener(ctx, tls.NewListener(ln, s.TLSConfig())) }() + t.Cleanup(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + return ln.Addr().String(), root, s, state +} + +func TestNodeBootChangeRejectsPrepareWithoutCreatingCommand(t *testing.T) { + addr, _, s, state := oldBootGate(t) + before, _ := s.authority.State("v") + reply := gate2Control(t, addr, gate2Request{Op: "prepare", VMID: "v", Fence: state.Fence, CommandID: "delayed-old-owner", Kind: "kernel"}) + after, _ := s.authority.State("v") + if reply.OK || reply.Ticket != "" || len(after.Commands) != len(before.Commands) { + t.Fatalf("old boot authorized a new command: reply=%+v commands=%d->%d", reply, len(before.Commands), len(after.Commands)) + } + if after.Phase != PhaseActive || after.Fence != before.Fence { + t.Fatal("boot mismatch was treated as termination or automatic adoption") + } +} + +func TestNodeBootChangeRejectsCapturedForwardWithoutTouchingSocket(t *testing.T) { + addr, root, s, state := oldBootGate(t) + handle, err := s.authority.BeginCommand("v", state.Fence, "old-prepared-job", CommandForward) + if err != nil { + t.Fatal(err) + } + var calls atomic.Int64 + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + fmt.Fprint(w, `{}`) + })) + request := gate2Request{Op: "forward", VMID: "v", Fence: state.Fence, CommandID: handle.ID, Kind: "kernel"} + request.Ticket = gate2Ticket(handle.token, request) + c, reader := gate2Connect(t, addr, testToken, request, "GET /old-socket HTTP/1.1\r\nHost: kernel\r\nConnection: close\r\n\r\n") + defer c.Close() + line, err := reader.ReadString('\n') + if err != nil { + t.Fatal(err) + } + after, _ := s.authority.State("v") + if line == "OK\n" || calls.Load() != 0 || after.Commands[handle.ID].Dispatched { + t.Fatalf("old boot touched a replacement socket: line=%q calls=%d dispatched=%v", line, calls.Load(), after.Commands[handle.ID].Dispatched) + } +} + +func TestNodeBootChangeRejectsHealthWithoutCallingKernel(t *testing.T) { + f := publishingFixture(t) + f.publish(t) + f.l.boot = "previous-node-boot" + state, _ := f.a.State("v") + if _, err := f.l.LaunchManifest(context.Background(), launchIntentForTest(state, "old-boot-launch"), allowNodeCommandForTest); err != nil { + t.Fatal(err) + } + addr, s := publishingGate(t, f) + state, _ = s.authority.State("v") + var calls atomic.Int64 + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"ok":true,"healthy":true}`) + })) + reply := readNodeHealth(t, addr, capturedHealthRequest(state)) + if reply.OK || calls.Load() != 0 || reply.Health != nil { + t.Fatalf("old boot returned running health: reply=%+v calls=%d", reply, calls.Load()) + } +} + +func TestNodeBootChangePreservesOriginalCompletedRecovery(t *testing.T) { + addr, _, s, state := oldBootGate(t) + handle, err := s.authority.BeginCommand("v", state.Fence, "completed-on-old-boot", CommandForward) + if err != nil { + t.Fatal(err) + } + if err = s.authority.DispatchCommand(handle, func(ExecutorIdentity) error { return nil }); err != nil { + t.Fatal(err) + } + if err = s.authority.CompleteCommand(handle); err != nil { + t.Fatal(err) + } + reply := readRecoveryStatus(t, addr, gate2Request{Op: "recovery-status", VMID: "v", Fence: state.Fence, CommandID: handle.ID}) + if !reply.OK || reply.Command == nil || reply.Command.Status != CommandCompleted || reply.Command.Fence != state.Fence { + t.Fatalf("historical completion was replaced by current boot identity: %+v", reply) + } +} From 76ef7660447547e295e7a8e9296484282a426b46 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:06:48 +0800 Subject: [PATCH 395/473] fix: reject captured execution authority from a previous host boot --- docs/plans/2026-10-22-node-recovery.md | 6 ++++++ worker/internal/slotgate/ksg2.go | 25 ++++++++++++++++++++++++- worker/internal/slotgate/slotgate.go | 2 ++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/plans/2026-10-22-node-recovery.md b/docs/plans/2026-10-22-node-recovery.md index 5cfa61f81..71db58249 100644 --- a/docs/plans/2026-10-22-node-recovery.md +++ b/docs/plans/2026-10-22-node-recovery.md @@ -20,6 +20,12 @@ 找不到记录时仅返回 `{ok:true,found:false,unknown:true}`。它不证明未开始，也不产生 seal/tombstone；旧 owner 仍可能延迟 prepare/forward。调用方必须继续保留 PG 未知占位。输入错误、原 Fence 不匹配、节点 journal 不可用时拒绝，不退回当前 state 或其他传输。 +## 节点启动标识 + +节点启动时独立读取宿主 bootID。新 prepare、forward 和 health 必须匹配原 Fence 中 executor 的 bootID；forward 在 Dispatch 边界再次核对。旧 ACTIVE 日志和相同路径的 Unix socket 不能自动授权新启动环境中的执行。失败规格 `5824f8a9` 分别复现旧 boot 仍可 prepare、dispatch 和读取健康状态。 + +bootID 不匹配仅拒绝新授权，不是旧 executor 终止证明，不更新 generation，也不释放 PG 容量。已有旧 boot 的 ACTIVE 状态仍需撤销、保留未知并走可信恢复。历史 recovery-status 按原 command/Fence 读取已持久结果，不要求当前 boot 或当前 generation 相同。 + ## 后续 seal 和计费 receipt 当前本批尚未提供 seal-not-started 或持久 usage receipt。notfound 不等于未开始；未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index c7c3a5b6e..f288f1615 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -129,6 +129,9 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. var state AuthorityState switch request.Op { case "health": + if err = s.currentBootFence(request.Fence); err != nil { + break + } var health map[string]any health, err = s.readHealth(ctx, request) if err == nil { @@ -165,6 +168,9 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. case "state": state, err = s.authority.State(request.VMID) case "prepare": + if err = s.currentBootFence(request.Fence); err != nil { + break + } if managed, e := s.authority.State(request.VMID); e == nil && managed.Allocation != nil && (managed.InitialManifest == nil || request.Kind != "kernel") { err = ErrManagementDenied break @@ -296,6 +302,10 @@ func (s *Server) forwardHandle(request gate2Request) (CommandHandle, error) { } func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio.Reader, request gate2Request) { + if err := s.currentBootFence(request.Fence); err != nil { + fmt.Fprintf(conn, "ERR %s\n", gate2Code(err)) + return + } handle, err := s.forwardHandle(request) if err != nil { code := gate2Code(err) @@ -347,7 +357,10 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio } conn.SetDeadline(time.Time{}) var target net.Conn - err = s.authority.DispatchCommand(handle, func(ExecutorIdentity) error { + err = s.authority.DispatchCommand(handle, func(executor ExecutorIdentity) error { + if executor.NodeBootID != s.nodeBootID || s.nodeBootID == "" { + return ErrFenceMismatch + } if state.Allocation != nil { token, e := readPublishedInitial(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy, *state.InitialManifest, false) if e != nil { @@ -408,3 +421,13 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio } } } + +// A surviving journal/socket is not evidence that an executor from a previous +// host boot is still the captured executor. Only trusted recovery may replace +// that identity; this rejection does not prove termination or release a job. +func (s *Server) currentBootFence(fence Fence) error { + if s.nodeBootID == "" || fence.Executor.NodeBootID != s.nodeBootID { + return ErrFenceMismatch + } + return nil +} diff --git a/worker/internal/slotgate/slotgate.go b/worker/internal/slotgate/slotgate.go index f686e867b..1e0b5efef 100644 --- a/worker/internal/slotgate/slotgate.go +++ b/worker/internal/slotgate/slotgate.go @@ -65,6 +65,7 @@ type Server struct { publisher *initialPublisher authorizeCommand CommandAuthorizer managementNodeID string + nodeBootID string // Internal injection for trusted adapters/tests. Config alone never grants // unimplemented network mutations to a holder of the forwarding token. authorizeManagement func(context.Context, gate2Request, func() error) error @@ -105,6 +106,7 @@ func New(cfg Config) (*Server, error) { if err != nil { return nil, err } + srv.nodeBootID = boot socket := cfg.DockerSocket if socket == "" { socket = "/var/run/docker.sock" From 46fd5fae9e761aab4b02f7b4f35659cdd64c2b89 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:03:22 +0800 Subject: [PATCH 396/473] test: specify owner lifetime proof and exact HA request sealing --- test/unit/ha-request-recovery.test.mjs | 132 +++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 test/unit/ha-request-recovery.test.mjs diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs new file mode 100644 index 000000000..3c4da7b5f --- /dev/null +++ b/test/unit/ha-request-recovery.test.mjs @@ -0,0 +1,132 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb, quoteIdentifier, registerActivityCoordinator, withTransaction } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator, coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { withHaRequestContext } from '../../src/lib/core/ha-request-context.mjs' + +async function fixture(t) { + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const first = new GlobalMaintenanceCoordinator(db, { ownerId: 'original-owner' }) + const second = new GlobalMaintenanceCoordinator(peer, { ownerId: 'recovery-owner' }) + await first.start() + await second.start() + registerActivityCoordinator(db, first) + registerActivityCoordinator(peer, second) + await first.acknowledgeEpoch(1) + await second.acknowledgeEpoch(1) + const previousMode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + t.after(async () => { + if (previousMode == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previousMode + await Promise.allSettled([first.stop(), second.stop()]) + await peer.pool.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinationSchemaFor(db.schema))} CASCADE`) + await peer.close() + }) + const gate = await second.admit({ kind: 'orphan_recovery' }) + t.after(() => gate.complete({ ended: true }).catch(() => {})) + return { db, peer, first, second, gate } +} + +async function parent(first, db, requestId, recoveryKind = 'claude_messages') { + const activity = await first.admit({ kind: 'request', requestId }) + await activity.run(() => withHaRequestContext({ db, requestId, activity, recoveryKind }, async () => {})) + await activity.complete({ ended: false, reason: 'synthetic_owner_lost' }) + return activity +} + +test('a live owner keeps its lifetime lock regardless of old member timestamps', async (t) => { + const { peer, first, second, gate } = await fixture(t) + await peer.pool.query(`UPDATE ${first.qualified}.members SET started_at='2000-01-01' WHERE owner_id=$1`, [ + first.ownerId, + ]) + await gate.run(async () => { + const result = await withTransaction(peer, () => second.lockAbsentOwner(first.ownerId)) + assert.deepEqual(result, { absent: false, reason: 'owner_connected' }) + }) + const member = ( + await peer.pool.query(`SELECT owner_lock_version FROM ${first.qualified}.members WHERE owner_id=$1`, [ + first.ownerId, + ]) + ).rows[0] + assert.equal(member.owner_lock_version, 1) +}) + +test('disconnect releases the lifetime lock but never permits re-registering an old owner identity', async (t) => { + const { db, peer, first, second, gate } = await fixture(t) + await first._client.end() + await gate.run(async () => { + assert.deepEqual(await withTransaction(peer, () => second.lockAbsentOwner(first.ownerId)), { absent: true }) + }) + const duplicate = new GlobalMaintenanceCoordinator(db, { ownerId: first.ownerId }) + t.after(() => duplicate.stop()) + await assert.rejects(duplicate.start(), { code: '23505' }) +}) + +test('upgrading old private members leaves legacy owners unproven without a version marker', async (t) => { + const { peer, first, second, gate } = await fixture(t) + await first.stop() + await peer.pool.query(`ALTER TABLE ${first.qualified}.members DROP COLUMN owner_lock_version`) + const fresh = new GlobalMaintenanceCoordinator(peer, { ownerId: 'new-protocol-owner' }) + t.after(() => fresh.stop()) + await fresh.start() + const rows = ( + await peer.pool.query(`SELECT owner_id, owner_lock_version FROM ${first.qualified}.members ORDER BY owner_id`) + ).rows + assert.equal(rows.find((row) => row.owner_id === first.ownerId).owner_lock_version, null) + assert.equal(rows.find((row) => row.owner_id === fresh.ownerId).owner_lock_version, 1) + await gate.run(async () => { + assert.deepEqual(await withTransaction(peer, () => second.lockAbsentOwner(first.ownerId)), { + absent: false, + reason: 'owner_protocol_unproven', + }) + }) +}) + +test('recovery seals only an exact supported request after its registered owner disconnects', async (t) => { + const { db, peer, first, second, gate } = await fixture(t) + const original = await parent(first, db, 'recoverable') + await parent(first, db, 'unrelated') + const { HaRequestRecovery } = await import('../../src/lib/core/ha-request-recovery.mjs') + const recovery = new HaRequestRecovery(peer, { coordinator: second }) + await gate.run(async () => { + assert.deepEqual(await recovery.seal('recoverable'), { sealed: false, reason: 'owner_connected' }) + }) + await first.stop() + await gate.run(async () => { + const result = await recovery.seal('recoverable') + assert.equal(result.sealed, true) + const row = await peer.prepare('SELECT * FROM ha_request_contexts WHERE id=$1').get('recoverable') + assert.equal(row.activity_id, original.activityId) + assert.equal(row.accepting_children, false) + assert.equal(row.recovery_kind, 'claude_messages') + assert.equal(row.recovery_sealed_by, second.ownerId) + assert.ok(row.recovery_sealed_at) + assert.equal( + (await peer.prepare('SELECT recovery_sealed_at FROM ha_request_contexts WHERE id=$1').get('unrelated')) + .recovery_sealed_at, + null, + ) + }) +}) + +test('untyped legacy parents and stale epochs cannot be sealed from absent owner alone', async (t) => { + const { db, peer, first, second, gate } = await fixture(t) + await parent(first, db, 'legacy-parent', null) + await parent(first, db, 'stale-parent') + await peer.pool.query('UPDATE ha_request_contexts SET activity_epoch=2 WHERE id=$1', ['stale-parent']) + await first.stop() + const { HaRequestRecovery } = await import('../../src/lib/core/ha-request-recovery.mjs') + const recovery = new HaRequestRecovery(peer, { coordinator: second }) + await gate.run(async () => { + assert.deepEqual(await recovery.seal('legacy-parent'), { sealed: false, reason: 'request_kind_unproven' }) + assert.deepEqual(await recovery.seal('stale-parent'), { sealed: false, reason: 'request_activity_mismatch' }) + assert.equal( + (await peer.prepare('SELECT count(*) AS n FROM ha_request_contexts WHERE recovery_sealed_at IS NOT NULL').get()) + .n, + 0, + ) + }) +}) From 76888d63de8bc8a2d70e288333637ffb710cc3fc Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:05:33 +0800 Subject: [PATCH 397/473] feat: prove disconnected owners before sealing exact HA requests --- src/lib/core/ha-request-context.mjs | 9 +++-- src/lib/core/ha-request-recovery.mjs | 40 +++++++++++++++++++ src/lib/db/global-maintenance.mjs | 35 +++++++++++++++- .../020_ha_request_recovery.sql | 5 +++ test/unit/database.test.mjs | 4 +- test/unit/ha-request-recovery.test.mjs | 11 ++++- 6 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 src/lib/core/ha-request-recovery.mjs create mode 100644 src/lib/db/postgres-migrations/020_ha_request_recovery.sql diff --git a/src/lib/core/ha-request-context.mjs b/src/lib/core/ha-request-context.mjs index be065dcf8..46b36bf40 100644 --- a/src/lib/core/ha-request-context.mjs +++ b/src/lib/core/ha-request-context.mjs @@ -36,7 +36,7 @@ export async function lockHaRequestContext(db) { } /** Correlation only: closing this context never releases a lease or unknown work. */ -export async function withHaRequestContext({ db = getDb(), requestId, activity }, callback) { +export async function withHaRequestContext({ db = getDb(), requestId, activity, recoveryKind = null }, callback) { if (!isKubernetesHaMode()) return callback() if (requests.getStore()) throw failure('ha_request_context_nested') const coordinator = getActivityCoordinator(db) @@ -50,10 +50,11 @@ export async function withHaRequestContext({ db = getDb(), requestId, activity } throw failure('ha_request_activity_mismatch') if (typeof requestId !== 'string' || !requestId.trim() || requestId.length > 128) throw failure('ha_request_id_invalid') + if (recoveryKind !== null && recoveryKind !== 'claude_messages') throw failure('ha_request_recovery_kind_invalid') const inserted = await db - .prepare(`INSERT INTO ha_request_contexts(id,activity_id,activity_owner_id,activity_epoch) - VALUES($1,$2,$3,$4) ON CONFLICT(id) DO NOTHING RETURNING id`) - .get(requestId, scope.id, coordinator.ownerId, scope.epoch) + .prepare(`INSERT INTO ha_request_contexts(id,activity_id,activity_owner_id,activity_epoch,recovery_kind) + VALUES($1,$2,$3,$4,$5) ON CONFLICT(id) DO NOTHING RETURNING id`) + .get(requestId, scope.id, coordinator.ownerId, scope.epoch, recoveryKind) if (!inserted) throw failure('ha_request_context_exists') let active = true const context = Object.freeze({ diff --git a/src/lib/core/ha-request-recovery.mjs b/src/lib/core/ha-request-recovery.mjs new file mode 100644 index 000000000..0ff80872f --- /dev/null +++ b/src/lib/core/ha-request-recovery.mjs @@ -0,0 +1,40 @@ +import { getActivityCoordinator, getDb, withTransaction } from '../db/database.mjs' + +/** Explicit recovery only. Closing a parent proves no late child can be admitted, not that a node ended. */ +export class HaRequestRecovery { + constructor(db = getDb(), { coordinator = getActivityCoordinator(db) } = {}) { + if (!coordinator || coordinator.db !== db) throw new Error('HA recovery requires its activity coordinator') + this.db = db + this.coordinator = coordinator + } + + async seal(requestId) { + return withTransaction(this.db, async () => { + const request = await this.db.prepare('SELECT * FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get(requestId) + if (!request) return { sealed: false, reason: 'request_missing' } + if (request.recovery_kind !== 'claude_messages') return { sealed: false, reason: 'request_kind_unproven' } + const absent = await this.coordinator.lockAbsentOwner(request.activity_owner_id) + if (!absent.absent) return { sealed: false, reason: absent.reason } + const scope = this.coordinator.assertActivity() + const activity = ( + await this.db.query(`SELECT * FROM ${this.coordinator.qualified}.activities WHERE id=$1 FOR UPDATE`, [ + request.activity_id, + ]) + ).rows[0] + if ( + !activity || + activity.owner_id !== request.activity_owner_id || + String(activity.epoch) !== String(request.activity_epoch) || + String(request.activity_epoch) !== String(scope.epoch) + ) { + return { sealed: false, reason: 'request_activity_mismatch' } + } + await this.db + .prepare(`UPDATE ha_request_contexts SET accepting_children=false, + recovery_sealed_at=COALESCE(recovery_sealed_at,clock_timestamp()), + recovery_sealed_by=COALESCE(recovery_sealed_by,$2) WHERE id=$1`) + .run(requestId, this.coordinator.ownerId) + return { sealed: true } + }) + } +} diff --git a/src/lib/db/global-maintenance.mjs b/src/lib/db/global-maintenance.mjs index 107c96650..bc3006fba 100644 --- a/src/lib/db/global-maintenance.mjs +++ b/src/lib/db/global-maintenance.mjs @@ -162,10 +162,12 @@ export class GlobalMaintenanceCoordinator { ); CREATE TABLE IF NOT EXISTS ${q}.members ( owner_id text PRIMARY KEY, + owner_lock_version integer, bound_epoch bigint, started_at timestamptz NOT NULL DEFAULT clock_timestamp(), stopped_at timestamptz ); + ALTER TABLE ${q}.members ADD COLUMN IF NOT EXISTS owner_lock_version integer; CREATE TABLE IF NOT EXISTS ${q}.activities ( id text PRIMARY KEY, owner_id text NOT NULL, @@ -211,7 +213,11 @@ export class GlobalMaintenanceCoordinator { try { await client.connect() this._client = client - await this.db.pool.query(`INSERT INTO ${this.qualified}.members(owner_id) VALUES ($1)`, [this.ownerId]) + const lock = await client.query(TRY_EXCLUSIVE, [this._ownerResource(this.ownerId)]) + if (!lock.rows[0].acquired) throw failure('maintenance_owner_connected', 'Owner identity is already connected') + await client.query(`INSERT INTO ${this.qualified}.members(owner_id,owner_lock_version) VALUES ($1,1)`, [ + this.ownerId, + ]) if (this._lost) throw failure('maintenance_connection_lost', 'Activity connection ended during startup') this._healthy = true return this @@ -238,6 +244,33 @@ export class GlobalMaintenanceCoordinator { return stateOf(row) } + _ownerResource(ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim() || ownerId.length > 256) { + throw failure('maintenance_owner_invalid', 'Owner identity is invalid') + } + return `vm2api:owner-lifetime:${this.db.schema}:${ownerId}` + } + + /** Proof is valid only inside this short transaction. No timestamp establishes owner loss. */ + async lockAbsentOwner(ownerId) { + const scope = this.assertActivity() + if (!this.db.isTransaction) throw failure('maintenance_transaction_required', 'Owner proof requires a transaction') + const control = (await this.db.query(`SELECT phase,epoch FROM ${this.qualified}.control WHERE id=1 FOR SHARE`)) + .rows[0] + if (control?.phase !== 'open' || String(control.epoch) !== String(scope.epoch)) { + throw failure('database_maintenance', 'Owner proof requires the current open epoch') + } + const member = ( + await this.db.query(`SELECT owner_lock_version FROM ${this.qualified}.members WHERE owner_id=$1`, [ownerId]) + ).rows[0] + if (member?.owner_lock_version !== 1) return { absent: false, reason: 'owner_protocol_unproven' } + const lock = await this.db.query( + 'SELECT pg_try_advisory_xact_lock(hashtext(current_database()),hashtext($1)) AS acquired', + [this._ownerResource(ownerId)], + ) + return lock.rows[0].acquired ? { absent: true } : { absent: false, reason: 'owner_connected' } + } + async acknowledgeEpoch(epoch) { this._assertHealthy() return this._transaction(async (client) => { diff --git a/src/lib/db/postgres-migrations/020_ha_request_recovery.sql b/src/lib/db/postgres-migrations/020_ha_request_recovery.sql new file mode 100644 index 000000000..a79f317fa --- /dev/null +++ b/src/lib/db/postgres-migrations/020_ha_request_recovery.sql @@ -0,0 +1,5 @@ +ALTER TABLE ha_request_contexts ADD COLUMN recovery_kind text + CHECK (recovery_kind IS NULL OR recovery_kind = 'claude_messages'); +ALTER TABLE ha_request_contexts ADD COLUMN recovery_sealed_at timestamptz; +ALTER TABLE ha_request_contexts ADD COLUMN recovery_sealed_by text COLLATE "C"; +ALTER TABLE ha_request_contexts ADD COLUMN execution_recovered_at timestamptz; diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index a3ac92221..2f6bc153f 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -118,7 +118,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 19) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 20) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -134,7 +134,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 19) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 20) fs.rmSync(dir, { recursive: true, force: true }) }) diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs index 3c4da7b5f..2650f0788 100644 --- a/test/unit/ha-request-recovery.test.mjs +++ b/test/unit/ha-request-recovery.test.mjs @@ -1,5 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' +import pg from 'pg' import { getDb, quoteIdentifier, registerActivityCoordinator, withTransaction } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator, coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' import { createRepositoryTestDatabase } from '../repository-database.mjs' @@ -22,7 +23,13 @@ async function fixture(t) { if (previousMode == null) delete process.env.VM2API_CONTROL_PLANE_MODE else process.env.VM2API_CONTROL_PLANE_MODE = previousMode await Promise.allSettled([first.stop(), second.stop()]) - await peer.pool.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinationSchemaFor(db.schema))} CASCADE`) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinationSchemaFor(db.schema))} CASCADE`) + } finally { + await cleanup.end() + } await peer.close() }) const gate = await second.admit({ kind: 'orphan_recovery' }) @@ -70,7 +77,7 @@ test('upgrading old private members leaves legacy owners unproven without a vers await first.stop() await peer.pool.query(`ALTER TABLE ${first.qualified}.members DROP COLUMN owner_lock_version`) const fresh = new GlobalMaintenanceCoordinator(peer, { ownerId: 'new-protocol-owner' }) - t.after(() => fresh.stop()) + t.after(() => fresh.stop().catch(() => {})) await fresh.start() const rows = ( await peer.pool.query(`SELECT owner_id, owner_lock_version FROM ${first.qualified}.members ORDER BY owner_id`) From e2792bcf0321fc2053fd6760c21374f35be4225c Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:14:45 +0800 Subject: [PATCH 398/473] feat: link supported HA inference requests to recovery contexts --- src/server.mjs | 23 +++++++++++++++++++++-- test/unit/backup-service.test.mjs | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/server.mjs b/src/server.mjs index 27db07841..c8905622a 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1280,6 +1280,19 @@ function noteRequestExecution(req, execution) { async function withRequestActivity(req, res, callback, kind = 'http') { if (!activityCoordinator || requestActivities.has(req)) return callback() const requestId = haMode ? crypto.randomUUID() : null + const pathname = parseRequestUrl(req)?.pathname + const recoverable = + haMode && + req.method === 'POST' && + [ + '/v1/messages', + '/messages', + '/v1/chat/completions', + '/chat/completions', + '/v1/completions', + '/completions', + ].includes(pathname) + const trackHaRequest = haMode && (recoverable || kind === 'websocket_request') const gate = await activityCoordinator.admit({ kind, requestId }) const state = { gate, executions: new Map(), unknown: false } requestActivities.set(req, state) @@ -1294,13 +1307,19 @@ async function withRequestActivity(req, res, callback, kind = 'http') { try { return await gate.run(async () => { try { - const value = await withHaRequestContext({ db, requestId, activity: gate }, async () => { + const invoke = async () => { if (haCatalog) await haCatalog.refresh() const result = await callback(state) await responseDone await flushRuntimeWrites() return result - }) + } + const value = trackHaRequest + ? await withHaRequestContext( + { db, requestId, activity: gate, recoveryKind: recoverable ? 'claude_messages' : null }, + invoke, + ) + : await invoke() return value } catch (error) { state.unknown = true diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 13aa5912a..497cdb112 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -115,7 +115,7 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 19) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 20) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { From f67446b720e684007b0f152d2599f2366f110942 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:08:54 +0800 Subject: [PATCH 399/473] test: persist bounded node usage receipts with original execution completion --- .../slotgate/inference_receipt_test.go | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 worker/internal/slotgate/inference_receipt_test.go diff --git a/worker/internal/slotgate/inference_receipt_test.go b/worker/internal/slotgate/inference_receipt_test.go new file mode 100644 index 000000000..fa704453b --- /dev/null +++ b/worker/internal/slotgate/inference_receipt_test.go @@ -0,0 +1,154 @@ +package slotgate + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "sync" + "testing" +) + +const receiptClaudeJSON = `{"id":"msg_receipt","type":"message","role":"assistant","model":"claude-synthetic-actual","content":[{"type":"text","text":"private response body"}],"stop_reason":"end_turn","usage":{"input_tokens":9007199254740993,"output_tokens":7,"cache_read_input_tokens":11,"cache_creation_input_tokens":5,"cache_creation":{"ephemeral_5m_input_tokens":2,"ephemeral_1h_input_tokens":3},"server_tool_use":{"web_search_requests":1}},"speed":"fast","service_tier":"priority","headers":{"authorization":"synthetic-secret"}}` + +func receiptRecoveryReply(t *testing.T, addr string, request gate2Request) map[string]any { + t.Helper() + request.Op = "recovery-status" + request.Ticket, request.Kind = "", "" + c, reader := gate2Connect(t, addr, testToken, request, "") + defer c.Close() + var reply map[string]any + if err := json.NewDecoder(reader).Decode(&reply); err != nil { + t.Fatal(err) + } + return reply +} + +func receiptForward(t *testing.T, media, body string, detached bool) (string, *Server, gate2Request) { + t.Helper() + addr, root, server, state, _ := fencedGate(t) + resume := make(chan struct{}) + var once sync.Once + release := func() { once.Do(func() { close(resume) }) } + defer release() + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", media) + if detached { + fmt.Fprint(w, body[:1]) + w.(http.Flusher).Flush() + <-resume + fmt.Fprint(w, body[1:]) + } else { + fmt.Fprint(w, body) + } + })) + request := forwardRequest(t, addr, state, "receipt-job") + c, reader := gate2Connect(t, addr, testToken, request, claudeWire(request.CommandID)) + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + if detached { + if _, err = io.ReadFull(response.Body, make([]byte, 1)); err != nil { + t.Fatal(err) + } + resetControllerConnection(t, c) + release() + } else { + if _, err = io.Copy(io.Discard, response.Body); err != nil { + t.Fatal(err) + } + response.Body.Close() + c.Close() + } + waitCommandStatus(t, server.authority, request.CommandID, CommandCompleted) + return addr, server, request +} + +func requireInferenceReceipt(t *testing.T, reply map[string]any) map[string]any { + t.Helper() + receipt, ok := reply["receipt"].(map[string]any) + if reply["ok"] != true || reply["found"] != true || !ok || receipt["id"] == nil || receipt["id"] == "" { + t.Fatalf("completed command has no durable receipt: %+v", reply) + } + return receipt +} + +func TestNodeInferenceReceiptPersistsExactPublicUsageWithOriginalCommand(t *testing.T) { + addr, server, request := receiptForward(t, "application/json", receiptClaudeJSON, false) + receipt := requireInferenceReceipt(t, receiptRecoveryReply(t, addr, request)) + usage, ok := receipt["usage"].(map[string]any) + if !ok || receipt["usage_complete"] != true || receipt["model"] != "claude-synthetic-actual" || receipt["stop_reason"] != "end_turn" || receipt["service_tier"] != "priority" || receipt["speed"] != "fast" || usage["input_tokens"] != "9007199254740993" || usage["output_tokens"] != "7" { + t.Fatalf("receipt lost exact observed values: %+v", receipt) + } + if detail, ok := usage["cache_creation"].(map[string]any); !ok || detail["ephemeral_5m_input_tokens"] != "2" || detail["ephemeral_1h_input_tokens"] != "3" { + t.Fatalf("cache detail unavailable: %+v", usage) + } + encoded, _ := json.Marshal(receipt) + for _, forbidden := range []string{"private response body", "synthetic-secret", "authorization", "headers", "USD", "request_id"} { + if strings.Contains(string(encoded), forbidden) { + t.Fatalf("receipt retained forbidden data %s", forbidden) + } + } + if err := server.authority.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, server.cfg.AuthorityDir, terminatedExecutor) + state, err := reopened.State("v") + if err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(state.Commands[request.CommandID]) + var command map[string]any + json.Unmarshal(data, &command) + persisted, ok := command["terminal_receipt"].(map[string]any) + if !ok || persisted["id"] != receipt["id"] || command["status"] != string(CommandCompleted) { + t.Fatalf("receipt not committed with original command: %+v", command) + } +} + +func TestNodeInferenceReceiptMergesCumulativeSSEWithoutSumming(t *testing.T) { + body := "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_sse\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"claude-sse-actual\",\"usage\":{\"input_tokens\":8,\"output_tokens\":0,\"cache_read_input_tokens\":3,\"cache_creation_input_tokens\":2}}}\n\n" + + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{},\"usage\":{\"output_tokens\":4}}\n\n" + + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":9,\"cache_creation\":{\"ephemeral_5m_input_tokens\":2,\"ephemeral_1h_input_tokens\":0}},\"service_tier\":\"standard\"}\n\n" + + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" + addr, _, request := receiptForward(t, "text/event-stream", body, false) + receipt := requireInferenceReceipt(t, receiptRecoveryReply(t, addr, request)) + usage, ok := receipt["usage"].(map[string]any) + if !ok || receipt["usage_complete"] != true || receipt["model"] != "claude-sse-actual" || usage["input_tokens"] != "8" || usage["output_tokens"] != "9" || usage["cache_read_input_tokens"] != "3" || receipt["service_tier"] != "standard" { + t.Fatalf("SSE cumulative usage was lost or summed: %+v", receipt) + } +} + +func TestNodeInferenceReceiptMissingOrInvalidUsageDoesNotInventZeroOrHoldEndedExecution(t *testing.T) { + for _, usage := range []string{"", `,"usage":{"input_tokens":1,"output_tokens":-1,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}`, `,"usage":{"input_tokens":1,"output_tokens":1e3,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}`, `,"usage":{"input_tokens":1,"input_tokens":2,"output_tokens":1,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}`} { + t.Run(fmt.Sprintf("case-%d", len(usage)), func(t *testing.T) { + body := `{"id":"msg_incomplete","type":"message","role":"assistant","content":[],"model":"claude-actual","stop_reason":"end_turn"` + usage + `}` + addr, _, request := receiptForward(t, "application/json", body, false) + receipt := requireInferenceReceipt(t, receiptRecoveryReply(t, addr, request)) + if receipt["usage_complete"] != false { + t.Fatalf("invalid or missing usage was marked billable: %+v", receipt) + } + observed, ok := receipt["usage"].(map[string]any) + if !ok { + t.Fatal("missing usage object") + } + if _, exists := observed["output_tokens"]; exists { + t.Fatalf("missing/invalid count was substituted: %+v", observed) + } + }) + } +} + +func TestNodeInferenceReceiptSurvivesControllerDisconnect(t *testing.T) { + addr, _, request := receiptForward(t, "application/json", receiptClaudeJSON, true) + receipt := requireInferenceReceipt(t, receiptRecoveryReply(t, addr, request)) + if receipt["usage_complete"] != true { + t.Fatalf("detached consumption lost usage: %+v", receipt) + } +} From c0ad8dba1be0dc74ef0598a91fccce13c9514582 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:11:48 +0800 Subject: [PATCH 400/473] feat: persist public usage receipts with node execution completion --- docs/plans/2026-10-22-node-recovery.md | 12 +- worker/internal/slotgate/authority.go | 27 +- worker/internal/slotgate/claude_terminal.go | 3 + worker/internal/slotgate/inference_receipt.go | 274 ++++++++++++++++++ .../slotgate/inference_receipt_test.go | 70 +++++ worker/internal/slotgate/ksg2.go | 5 +- worker/internal/slotgate/recovery_status.go | 11 +- 7 files changed, 383 insertions(+), 19 deletions(-) create mode 100644 worker/internal/slotgate/inference_receipt.go diff --git a/docs/plans/2026-10-22-node-recovery.md b/docs/plans/2026-10-22-node-recovery.md index 71db58249..cba826987 100644 --- a/docs/plans/2026-10-22-node-recovery.md +++ b/docs/plans/2026-10-22-node-recovery.md @@ -26,8 +26,16 @@ bootID 不匹配仅拒绝新授权，不是旧 executor 终止证明，不更新 generation，也不释放 PG 容量。已有旧 boot 的 ACTIVE 状态仍需撤销、保留未知并走可信恢复。历史 recovery-status 按原 command/Fence 读取已持久结果，不要求当前 boot 或当前 generation 相同。 -## 后续 seal 和计费 receipt +## 正常终态的用量凭证 -当前本批尚未提供 seal-not-started 或持久 usage receipt。notfound 不等于未开始；未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 +失败规格 `c3a3bc6c` 证明已完成执行原先没有可供死 controller 后续对账的持久用量信息。正常 JSON/SSE 终态及完整 body/trailer 验证后，节点在同一次 journal 提交中记录 completed 和 `terminal_receipt`。凭证 ID 由节点生成，不接受客户端提交。原 command 的 ID/Fence 与凭证保存在同一条记录；`recovery-status` 在原 command 旁返回顶层 `receipt`，保持 command 的四个原字段不变。历史读取不会将其他 command 的凭证拼接到当前身份。 + +凭证仅保留有界的实际 model、usage、speed、service_tier 和白名单 stop_reason。usage 的四个主计数为 input_tokens、output_tokens、cache_read_input_tokens、cache_creation_input_tokens。原帧存在时才保留 cache_creation 的 5m/1h 细分和 server_tool_use.web_search_requests。计数统一为 `0` 至 PostgreSQL bigint 上限的规范十进制字符串，避免 JavaScript 数值转换丢失精度。SSE 累计值按帧覆盖，不相加；完成用量要求最终 message_delta 明确给出 output_tokens。 + +`usage_complete:true` 还要求实际 model 与四个主计数都明确存在，且未遇到非法、重复或越界的用量。缺失字段保持缺失，不填零，不以请求 model 替代实际 model。普通终态已经确认但用量不完整时仍可记录 completed；凭证标记 `usage_complete:false`，调用方保留待对账状态。凭证不含正文、thinking、工具参数、headers、凭据、USD 或定价输入。HTTP 错误终态、截断、尾随错误、无效 trailer 或 journal 失败不产生可用完成凭证。 + +## 后续 seal 与 PG 对账 + +当前本批尚未提供 seal-not-started。notfound 不等于未开始；未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 usage 缺失不得补零结算；节点只提供原始公开 model/usage/terminal 信息，PG 根据原 command 对应的 parent/attempt 和价格快照处理。node health、ready_slots 和取消 ACK 均不是恢复终止证明。以上节点能力不表示生产 HA 已完成。 diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index cb4c287bc..f9ce4b81d 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -78,17 +78,18 @@ type Fence struct { } type CommandRecord struct { - ID string `json:"id"` - Kind CommandKind `json:"kind"` - Status CommandStatus `json:"status"` - Fence Fence `json:"fence"` - Runtime string `json:"runtime"` - Token string `json:"token"` - Dispatched bool `json:"dispatched"` - TerminationID string `json:"termination_id,omitempty"` - LaunchTarget *DockerLaunchTarget `json:"launch_target,omitempty"` - Management *ManagementIntent `json:"management,omitempty"` - LaunchReceipt *LaunchReceipt `json:"launch_receipt,omitempty"` + ID string `json:"id"` + Kind CommandKind `json:"kind"` + Status CommandStatus `json:"status"` + Fence Fence `json:"fence"` + Runtime string `json:"runtime"` + Token string `json:"token"` + Dispatched bool `json:"dispatched"` + TerminationID string `json:"termination_id,omitempty"` + LaunchTarget *DockerLaunchTarget `json:"launch_target,omitempty"` + Management *ManagementIntent `json:"management,omitempty"` + LaunchReceipt *LaunchReceipt `json:"launch_receipt,omitempty"` + TerminalReceipt *inferenceReceipt `json:"terminal_receipt,omitempty"` } // DockerLaunchTarget is recorded after create returns and before start. A @@ -685,6 +686,7 @@ func cloneState(state AuthorityState) AuthorityState { } commands := make(map[string]CommandRecord, len(state.Commands)) for id, command := range state.Commands { + command.TerminalReceipt = cloneInferenceReceipt(command.TerminalReceipt) if command.Management != nil { intent := *command.Management command.Management = &intent @@ -808,6 +810,9 @@ func validateJournal(journal authorityJournal) error { return ErrInvalidAuthority } for id, command := range state.Commands { + if receipt := command.TerminalReceipt; receipt != nil && (command.Kind != CommandForward || command.Status != CommandCompleted || !command.Dispatched || !validInferenceReceipt(*receipt)) { + return ErrInvalidAuthority + } if id != command.ID || !validAuthorityID(id) || !validAuthorityID(command.Runtime) || !validAuthorityID(command.Token) || command.Fence.Generation == 0 || command.Fence.Generation > state.Fence.Generation || !validAuthorityID(command.Fence.Incarnation) { return ErrInvalidAuthority } diff --git a/worker/internal/slotgate/claude_terminal.go b/worker/internal/slotgate/claude_terminal.go index 0635d9626..c11272b13 100644 --- a/worker/internal/slotgate/claude_terminal.go +++ b/worker/internal/slotgate/claude_terminal.go @@ -33,6 +33,7 @@ type claudeTerminal struct { stopped bool blocks map[int]bool nextBlock int + receipt inferenceReceiptBuilder } func observeClaudeTerminal(response *http.Response, request *http.Request, body []byte, commandID, kind string) *claudeTerminal { @@ -220,6 +221,7 @@ func (o *claudeTerminal) observeEvent() { return } o.started = true + o.receipt.sse(o.data, true) case "message_stop": if !o.started || o.stopped || len(o.blocks) != 0 { o.reject() @@ -278,6 +280,7 @@ func (o *claudeTerminal) observeEvent() { o.reject() return } + o.receipt.sse(o.data, false) case "ping", "response_headers": // Kernel metadata may precede a message or follow message_stop. An // error/unknown event still invalidates even an earlier message_stop. diff --git a/worker/internal/slotgate/inference_receipt.go b/worker/internal/slotgate/inference_receipt.go new file mode 100644 index 000000000..a61c0b143 --- /dev/null +++ b/worker/internal/slotgate/inference_receipt.go @@ -0,0 +1,274 @@ +package slotgate + +import ( + "encoding/hex" + "encoding/json" + "regexp" + "strconv" +) + +// This is observed protocol metadata, not a bill. Missing counts stay absent; +// the PG parent/attempt owns identity, pricing and any later reconciliation. +type inferenceReceipt struct { + ID string `json:"id"` + Model string `json:"model,omitempty"` + UsageComplete bool `json:"usage_complete"` + Usage inferenceUsage `json:"usage"` + Speed string `json:"speed,omitempty"` + ServiceTier string `json:"service_tier,omitempty"` + StopReason string `json:"stop_reason,omitempty"` +} + +type inferenceUsage struct { + InputTokens string `json:"input_tokens,omitempty"` + OutputTokens string `json:"output_tokens,omitempty"` + CacheReadTokens string `json:"cache_read_input_tokens,omitempty"` + CacheCreationTokens string `json:"cache_creation_input_tokens,omitempty"` + CacheCreation *inferenceCacheUsage `json:"cache_creation,omitempty"` + ServerToolUse *inferenceServerUsage `json:"server_tool_use,omitempty"` +} + +type inferenceCacheUsage struct { + FiveMinute string `json:"ephemeral_5m_input_tokens,omitempty"` + OneHour string `json:"ephemeral_1h_input_tokens,omitempty"` +} + +type inferenceServerUsage struct { + WebSearchRequests string `json:"web_search_requests,omitempty"` +} + +var inferencePublicValue = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`) + +func validInferenceCount(value string) bool { + number, err := strconv.ParseUint(value, 10, 63) + return err == nil && strconv.FormatUint(number, 10) == value +} + +func completeInferenceUsage(receipt inferenceReceipt) bool { + u := receipt.Usage + return receipt.Model != "" && u.InputTokens != "" && u.OutputTokens != "" && u.CacheReadTokens != "" && u.CacheCreationTokens != "" +} + +func validInferenceReceipt(receipt inferenceReceipt) bool { + id, err := hex.DecodeString(receipt.ID) + if err != nil || len(id) != 16 || hex.EncodeToString(id) != receipt.ID || + (receipt.Model != "" && !inferencePublicValue.MatchString(receipt.Model)) || + (receipt.Speed != "" && (len(receipt.Speed) > 64 || !inferencePublicValue.MatchString(receipt.Speed))) || + (receipt.ServiceTier != "" && (len(receipt.ServiceTier) > 64 || !inferencePublicValue.MatchString(receipt.ServiceTier))) || + (receipt.StopReason != "" && !normalClaudeStop(receipt.StopReason)) || + (receipt.UsageComplete && !completeInferenceUsage(receipt)) { + return false + } + u := receipt.Usage + counts := []string{u.InputTokens, u.OutputTokens, u.CacheReadTokens, u.CacheCreationTokens} + if u.CacheCreation != nil { + if u.CacheCreation.FiveMinute == "" && u.CacheCreation.OneHour == "" { + return false + } + counts = append(counts, u.CacheCreation.FiveMinute, u.CacheCreation.OneHour) + } + if u.ServerToolUse != nil { + if u.ServerToolUse.WebSearchRequests == "" { + return false + } + counts = append(counts, u.ServerToolUse.WebSearchRequests) + } + for _, count := range counts { + if count != "" && !validInferenceCount(count) { + return false + } + } + return true +} + +func cloneInferenceReceipt(receipt *inferenceReceipt) *inferenceReceipt { + if receipt == nil { + return nil + } + copy := *receipt + if receipt.Usage.CacheCreation != nil { + detail := *receipt.Usage.CacheCreation + copy.Usage.CacheCreation = &detail + } + if receipt.Usage.ServerToolUse != nil { + detail := *receipt.Usage.ServerToolUse + copy.Usage.ServerToolUse = &detail + } + return © +} + +// Only the node's original response observer calls this after normal terminal +// and complete body/trailer verification. No RPC accepts a client receipt. +func (a *Authority) completeInference(handle CommandHandle, receipt inferenceReceipt) error { + a.mu.Lock() + defer a.mu.Unlock() + state, command, err := a.command(handle) + if err != nil { + return err + } + if command.Status == CommandCompleted { + return nil + } + if command.Kind != CommandForward || command.Status != CommandPending || !command.Dispatched { + return ErrCommandOwner + } + receipt.ID, err = randomAuthorityID() + if err != nil { + return err + } + if !validInferenceReceipt(receipt) { + return ErrInvalidAuthority + } + command.Status = CommandCompleted + command.TerminalReceipt = cloneInferenceReceipt(&receipt) + state.Commands[handle.ID] = command + _, err = a.commit(handle.VMID, state) + return err +} + +type inferenceReceiptBuilder struct { + receipt inferenceReceipt + invalid bool + finalOutput bool +} + +func (b *inferenceReceiptBuilder) publicValue(raw json.RawMessage, target *string, limit int) { + if len(raw) == 0 { + return + } + var value string + if json.Unmarshal(raw, &value) != nil || len(value) > limit || !inferencePublicValue.MatchString(value) { + b.invalid = true + return + } + *target = value +} + +func (b *inferenceReceiptBuilder) count(source map[string]json.RawMessage, key string, target *string) bool { + raw, exists := source[key] + if !exists { + return false + } + value := string(raw) + if !validInferenceCount(value) { + *target = "" + b.invalid = true + return false + } + *target = value + return true +} + +func (b *inferenceReceiptBuilder) metadata(source map[string]json.RawMessage) { + b.publicValue(source["speed"], &b.receipt.Speed, 64) + b.publicValue(source["service_tier"], &b.receipt.ServiceTier, 64) +} + +func (b *inferenceReceiptBuilder) usage(raw json.RawMessage, final bool) { + if len(raw) == 0 { + if final { + b.finalOutput = false + } + return + } + var source map[string]json.RawMessage + if strictJSON(raw, &source) != nil || source == nil { + b.invalid = true + if final { + b.finalOutput = false + } + return + } + u := &b.receipt.Usage + b.count(source, "input_tokens", &u.InputTokens) + output := b.count(source, "output_tokens", &u.OutputTokens) + if final { + b.finalOutput = output + } + b.count(source, "cache_read_input_tokens", &u.CacheReadTokens) + b.count(source, "cache_creation_input_tokens", &u.CacheCreationTokens) + if raw, exists := source["cache_creation"]; exists { + var detail map[string]json.RawMessage + if strictJSON(raw, &detail) != nil || detail == nil { + b.invalid = true + } else { + if u.CacheCreation == nil { + u.CacheCreation = &inferenceCacheUsage{} + } + b.count(detail, "ephemeral_5m_input_tokens", &u.CacheCreation.FiveMinute) + b.count(detail, "ephemeral_1h_input_tokens", &u.CacheCreation.OneHour) + if *u.CacheCreation == (inferenceCacheUsage{}) { + u.CacheCreation = nil + } + } + } + if raw, exists := source["server_tool_use"]; exists { + var detail map[string]json.RawMessage + if strictJSON(raw, &detail) != nil || detail == nil { + b.invalid = true + } else { + if u.ServerToolUse == nil { + u.ServerToolUse = &inferenceServerUsage{} + } + b.count(detail, "web_search_requests", &u.ServerToolUse.WebSearchRequests) + if u.ServerToolUse.WebSearchRequests == "" { + u.ServerToolUse = nil + } + } + } else if _, exists := source["web_search_requests"]; exists { + if u.ServerToolUse == nil { + u.ServerToolUse = &inferenceServerUsage{} + } + b.count(source, "web_search_requests", &u.ServerToolUse.WebSearchRequests) + if u.ServerToolUse.WebSearchRequests == "" { + u.ServerToolUse = nil + } + } + b.metadata(source) +} + +func (b *inferenceReceiptBuilder) message(raw json.RawMessage) { + var source map[string]json.RawMessage + if strictJSON(raw, &source) != nil || source == nil { + b.invalid = true + return + } + b.publicValue(source["model"], &b.receipt.Model, 256) + b.metadata(source) + if raw := source["stop_reason"]; presentJSON(raw) { + var reason string + if json.Unmarshal(raw, &reason) == nil && normalClaudeStop(reason) { + b.receipt.StopReason = reason + } + } + b.usage(source["usage"], false) +} + +func (b *inferenceReceiptBuilder) sse(raw json.RawMessage, start bool) { + var source map[string]json.RawMessage + if strictJSON(raw, &source) != nil { + b.invalid = true + return + } + if start { + b.message(source["message"]) + return + } + b.metadata(source) + var delta struct { + StopReason string `json:"stop_reason"` + } + if json.Unmarshal(source["delta"], &delta) == nil && normalClaudeStop(delta.StopReason) { + b.receipt.StopReason = delta.StopReason + } + b.usage(source["usage"], true) +} + +func (o *claudeTerminal) inferenceReceipt() inferenceReceipt { + if o.mode == "application/json" { + o.receipt.message(o.jsonBody) + } + receipt := o.receipt.receipt + receipt.UsageComplete = !o.receipt.invalid && completeInferenceUsage(receipt) && (o.mode == "application/json" || o.receipt.finalOutput) + return receipt +} diff --git a/worker/internal/slotgate/inference_receipt_test.go b/worker/internal/slotgate/inference_receipt_test.go index fa704453b..4ce7f40a0 100644 --- a/worker/internal/slotgate/inference_receipt_test.go +++ b/worker/internal/slotgate/inference_receipt_test.go @@ -1,6 +1,7 @@ package slotgate import ( + "context" "encoding/json" "fmt" "io" @@ -110,6 +111,17 @@ func TestNodeInferenceReceiptPersistsExactPublicUsageWithOriginalCommand(t *test if !ok || persisted["id"] != receipt["id"] || command["status"] != string(CommandCompleted) { t.Fatalf("receipt not committed with original command: %+v", command) } + state.Commands[request.CommandID].TerminalReceipt.Usage.CacheCreation.FiveMinute = "999" + if _, err = reopened.Revoke("v", request.Fence); err != nil { + t.Fatal(err) + } + if _, err = reopened.Advance(context.Background(), "v", request.Fence); err != nil { + t.Fatal(err) + } + historical, err := reopened.recoveryStatus("v", request.Fence, request.CommandID) + if err != nil || historical == nil || historical.Receipt == nil || historical.Receipt.ID != receipt["id"] || historical.Receipt.Usage.CacheCreation.FiveMinute != "2" { + t.Fatalf("historical receipt mutated or disappeared across generation: %+v %v", historical, err) + } } func TestNodeInferenceReceiptMergesCumulativeSSEWithoutSumming(t *testing.T) { @@ -152,3 +164,61 @@ func TestNodeInferenceReceiptSurvivesControllerDisconnect(t *testing.T) { t.Fatalf("detached consumption lost usage: %+v", receipt) } } + +func TestNodeInferenceReceiptRequiresObservedModelAndFinalSSEOutput(t *testing.T) { + for _, tc := range []struct{ name, media, body string }{ + {"missing-model", "application/json", strings.Replace(receiptClaudeJSON, `"model":"claude-synthetic-actual",`, "", 1)}, + {"final-output-missing", "text/event-stream", "data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_final\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"claude-actual\",\"usage\":{\"input_tokens\":1,\"output_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation_input_tokens\":0}}}\n\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n"}, + {"counter-overflow", "application/json", strings.Replace(receiptClaudeJSON, `"input_tokens":9007199254740993`, `"input_tokens":9223372036854775808`, 1)}, + } { + t.Run(tc.name, func(t *testing.T) { + addr, _, request := receiptForward(t, tc.media, tc.body, false) + receipt := requireInferenceReceipt(t, receiptRecoveryReply(t, addr, request)) + if receipt["usage_complete"] != false { + t.Fatalf("partial observed metadata was treated as billable: %+v", receipt) + } + if tc.name == "missing-model" && receipt["model"] != nil { + t.Fatal("receipt substituted the request model") + } + }) + } +} + +func TestNodeInferenceReceiptNeverPersistsForErrorTerminalOrBadTrailer(t *testing.T) { + for _, mode := range []string{"trailing-error", "bad-trailer"} { + t.Run(mode, func(t *testing.T) { + addr, root, server, state, _ := fencedGate(t) + unixHTTP(t, filepath.Join(root, "vms", "v", "incarnations", state.Fence.Incarnation, "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if mode == "bad-trailer" { + w.Header().Set("Trailer", "X-Terminal-State") + } + fmt.Fprint(w, receiptClaudeJSON) + if mode == "bad-trailer" { + w.Header().Set("X-Terminal-State", "incomplete") + } else { + fmt.Fprint(w, `{"type":"error","error":{"message":"client disconnected"}}`) + } + })) + request := forwardRequest(t, addr, state, "receipt-invalid-terminal") + c, reader := gate2Connect(t, addr, testToken, request, claudeWire(request.CommandID)) + defer c.Close() + if line, err := reader.ReadString('\n'); err != nil || line != "OK\n" { + t.Fatalf("forward: %q %v", line, err) + } + response, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatal(err) + } + if _, err = io.Copy(io.Discard, response.Body); err != nil { + t.Fatal(err) + } + response.Body.Close() + reply := receiptRecoveryReply(t, addr, request) + current, _ := server.authority.State("v") + if reply["receipt"] != nil || current.Commands[request.CommandID].Status != CommandPending { + t.Fatalf("unconfirmed execution acquired a usage receipt: %+v", reply) + } + }) + } +} diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index f288f1615..89424e840 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -151,6 +151,9 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["found"] = command != nil if command != nil { reply["command"] = command + if command.Receipt != nil { + reply["receipt"] = command.Receipt + } } else { reply["unknown"] = true } @@ -416,7 +419,7 @@ func (s *Server) forwardFenced(ctx context.Context, conn net.Conn, reader *bufio // read the authenticated command status before releasing its PG job. // If this commit fails the Authority poisons itself and cannot report // completed; no new execution or retry is issued here. - if err = s.authority.CompleteCommand(handle); err != nil { + if err = s.authority.completeInference(handle, terminal.inferenceReceipt()); err != nil { return } } diff --git a/worker/internal/slotgate/recovery_status.go b/worker/internal/slotgate/recovery_status.go index 1fda86652..2c994dcf2 100644 --- a/worker/internal/slotgate/recovery_status.go +++ b/worker/internal/slotgate/recovery_status.go @@ -9,10 +9,11 @@ import ( // authorize dispatch and deliberately does not require a dead controller's // nonce/runtime or match the current generation of the slot. type recoveryCommandStatus struct { - ID string `json:"id"` - Fence Fence `json:"fence"` - Status CommandStatus `json:"status"` - Dispatched bool `json:"dispatched"` + ID string `json:"id"` + Fence Fence `json:"fence"` + Status CommandStatus `json:"status"` + Dispatched bool `json:"dispatched"` + Receipt *inferenceReceipt `json:"-"` } func (a *Authority) recoveryStatus(vm string, fence Fence, id string) (*recoveryCommandStatus, error) { @@ -39,5 +40,5 @@ func (a *Authority) recoveryStatus(vm string, fence Fence, id string) (*recovery if command.Kind != CommandForward { return nil, ErrManagementDenied } - return &recoveryCommandStatus{ID: command.ID, Fence: command.Fence, Status: command.Status, Dispatched: command.Dispatched}, nil + return &recoveryCommandStatus{ID: command.ID, Fence: command.Fence, Status: command.Status, Dispatched: command.Dispatched, Receipt: cloneInferenceReceipt(command.TerminalReceipt)}, nil } From 8d5ff2286817afe1013b45cc87318b5aa7c9d824 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:07:26 +0800 Subject: [PATCH 401/473] test: require original node proof for exact orphan request recovery --- test/unit/ha-request-recovery.test.mjs | 249 ++++++++++++++++++++++++- 1 file changed, 248 insertions(+), 1 deletion(-) diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs index 2650f0788..30ed4d0fe 100644 --- a/test/unit/ha-request-recovery.test.mjs +++ b/test/unit/ha-request-recovery.test.mjs @@ -4,7 +4,12 @@ import pg from 'pg' import { getDb, quoteIdentifier, registerActivityCoordinator, withTransaction } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator, coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' import { createRepositoryTestDatabase } from '../repository-database.mjs' -import { withHaRequestContext } from '../../src/lib/core/ha-request-context.mjs' +import { withHaRequestContext, withHaRequestAttempt } from '../../src/lib/core/ha-request-context.mjs' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' +import { SharedSessionTurns } from '../../src/lib/pool/shared-session-turns.mjs' +import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' +import { syntheticFence } from '../fixtures/fake-slotgate.mjs' async function fixture(t) { const db = getDb() @@ -44,6 +49,61 @@ async function parent(first, db, requestId, recoveryKind = 'claude_messages') { return activity } +async function graph({ db, first }, requestId, { withJob = true, graceMs = 30000 } = {}) { + const activity = await first.admit({ kind: 'request', requestId }) + const nodeBinding = { + protocol: 'KSG2', + nodeId: 'original-node', + vmId: 'vm-fixture', + kind: 'kernel', + fence: syntheticFence(), + } + let keyLease, turn, admission + await activity.run(() => + withHaRequestContext({ db, requestId, activity, recoveryKind: 'claude_messages' }, async () => { + const keys = new ApiKeyStore({ db }) + const key = await keys.create({ name: 'synthetic-key-' + requestId }) + keyLease = await keys.reserve(key) + assert.equal(keyLease.ok, true) + const turns = new SharedSessionTurns(db) + turn = await turns.enqueue('session-' + requestId, 'not-a-parent-link') + assert.equal(await turns.tryActivate(turn), true) + await turns.markUncertain(turn) + admission = await new SharedExecutionAdmissions(db).tryReserve({ + accountId: 'fixture-account', + vmId: 'vm-fixture', + accountLimit: 10, + vmLimit: 10, + fableLimit: 0, + rpmLimit: 0, + seatKey: 'seat-' + requestId, + seatCap: 10, + seatGraceMs: graceMs, + seatReservePct: 0, + seatHeadroom: 1, + }) + assert.equal(admission.ok, true) + if (withJob) { + await new RequestAttemptsRepo(db).begin({ + requestId, + attemptNo: 1, + vmId: 'vm-fixture', + accountId: 'fixture-account', + model: 'claude-synthetic', + }) + await withHaRequestAttempt(1, () => admission.bindExecution({ requestId: 'command-' + requestId, nodeBinding })) + } + await admission.markUncertain() + }), + ) + await activity.complete({ ended: false, reason: 'synthetic_owner_lost' }) + return { activity, keyLease, turn, admission, nodeBinding, commandId: 'command-' + requestId } +} + +function completed(original) { + return { id: original.commandId, fence: original.nodeBinding.fence, status: 'completed', dispatched: true } +} + test('a live owner keeps its lifetime lock regardless of old member timestamps', async (t) => { const { peer, first, second, gate } = await fixture(t) await peer.pool.query(`UPDATE ${first.qualified}.members SET started_at='2000-01-01' WHERE owner_id=$1`, [ @@ -137,3 +197,190 @@ test('untyped legacy parents and stale epochs cannot be sealed from absent owner ) }) }) + +test('original completed node proof recovers only exact execution, seat, turn and activity while billing stays held', async (t) => { + const f = await fixture(t) + const original = await graph(f, 'ended-request') + const unrelated = await graph(f, 'still-unknown') + await f.first.stop() + const { HaRequestRecovery } = await import('../../src/lib/core/ha-request-recovery.mjs') + const recovery = new HaRequestRecovery(f.peer, { coordinator: f.second }) + await f.gate.run(async () => { + let calls = 0 + const outcome = await recovery.recover('ended-request', { + readCompletion: async ({ requestId, nodeBinding }) => { + calls++ + assert.equal(f.peer.isTransaction, false) + assert.equal(requestId, original.commandId) + assert.deepEqual(nodeBinding, original.nodeBinding) + return completed(original) + }, + }) + assert.equal(outcome.recovered, true) + assert.equal(outcome.billingHeld, true) + assert.equal(calls, 1) + const lease = await f.peer + .prepare('SELECT * FROM execution_admissions WHERE id=$1') + .get(original.admission.admissionId) + assert.ok(lease.released_at) + assert.ok(lease.uncertain_at, 'historical uncertainty remains in the audit row') + const seat = await f.peer + .prepare('SELECT *,clock_timestamp() AS now FROM execution_seats WHERE seat_key=$1') + .get(original.admission.seatKey) + assert.ok(seat.grace_until > seat.now) + assert.equal( + (await f.peer.prepare('SELECT count(*) AS n FROM session_turns WHERE id=$1').get(original.turn.id)).n, + 0, + ) + assert.ok( + ( + await f.peer.query(`SELECT completed_at FROM ${f.second.qualified}.activities WHERE id=$1`, [ + original.activity.activityId, + ]) + ).rows[0].completed_at, + ) + assert.equal( + ( + await f.peer + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(original.keyLease.reservationId) + ).released_at, + null, + ) + assert.equal( + ( + await f.peer + .prepare('SELECT released_at FROM execution_admissions WHERE id=$1') + .get(unrelated.admission.admissionId) + ).released_at, + null, + ) + assert.equal( + ( + await f.peer.query(`SELECT completed_at FROM ${f.second.qualified}.activities WHERE id=$1`, [ + unrelated.activity.activityId, + ]) + ).rows[0].completed_at, + null, + ) + const job = await f.peer.prepare('SELECT * FROM execution_jobs WHERE request_id=$1').get(original.commandId) + assert.ok(job.ended_at) + assert.deepEqual(job.node_completion, completed(original)) + assert.equal( + (await f.peer.prepare('SELECT terminal_state FROM request_attempts WHERE ha_request_id=$1').get('ended-request')) + .terminal_state, + 'started', + 'no usage/result is fabricated', + ) + }) +}) + +for (const scenario of ['notfound', 'pending', 'undispatched', 'different-command', 'different-fence']) { + test(`recovery preserves all claims when original node proof is ${scenario}`, async (t) => { + const f = await fixture(t) + const original = await graph(f, 'unproven') + await f.first.stop() + const { HaRequestRecovery } = await import('../../src/lib/core/ha-request-recovery.mjs') + const recovery = new HaRequestRecovery(f.peer, { coordinator: f.second }) + const proof = completed(original) + if (scenario === 'pending') proof.status = 'pending' + if (scenario === 'undispatched') proof.dispatched = false + if (scenario === 'different-command') proof.id = 'new-command' + if (scenario === 'different-fence') proof.fence = { ...proof.fence, incarnation: 'replacement-container' } + await f.gate.run(async () => { + const outcome = await recovery.recover('unproven', { + readCompletion: async () => (scenario === 'notfound' ? null : proof), + }) + assert.equal(outcome.recovered, false) + assert.equal( + ( + await f.peer + .prepare('SELECT released_at FROM execution_admissions WHERE id=$1') + .get(original.admission.admissionId) + ).released_at, + null, + ) + assert.equal( + (await f.peer.prepare('SELECT ended_at FROM execution_jobs WHERE request_id=$1').get(original.commandId)) + .ended_at, + null, + ) + assert.equal( + (await f.peer.prepare('SELECT count(*) AS n FROM session_turns WHERE id=$1').get(original.turn.id)).n, + 1, + ) + assert.equal( + ( + await f.peer.query(`SELECT completed_at FROM ${f.second.qualified}.activities WHERE id=$1`, [ + original.activity.activityId, + ]) + ).rows[0].completed_at, + null, + ) + assert.equal( + ( + await f.peer + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(original.keyLease.reservationId) + ).released_at, + null, + ) + }) + }) +} + +test('sealed supported requests with no bound node jobs can release execution and turns without inventing usage', async (t) => { + const f = await fixture(t) + const original = await graph(f, 'never-bound', { withJob: false, graceMs: 0 }) + await f.first.stop() + const { HaRequestRecovery } = await import('../../src/lib/core/ha-request-recovery.mjs') + const recovery = new HaRequestRecovery(f.peer, { coordinator: f.second }) + await f.gate.run(async () => { + const outcome = await recovery.recover('never-bound', { + readCompletion: async () => assert.fail('no original node job'), + }) + assert.equal(outcome.recovered, true) + assert.equal(outcome.billingHeld, true) + assert.ok( + ( + await f.peer + .prepare('SELECT released_at FROM execution_admissions WHERE id=$1') + .get(original.admission.admissionId) + ).released_at, + ) + assert.equal( + await f.peer.prepare('SELECT seat_key FROM execution_seats WHERE seat_key=$1').get(original.admission.seatKey), + undefined, + ) + }) +}) + +test('concurrent recoveries are idempotent and cannot extend recovered seat grace on retry', async (t) => { + const f = await fixture(t) + const original = await graph(f, 'concurrent') + await f.first.stop() + const { HaRequestRecovery } = await import('../../src/lib/core/ha-request-recovery.mjs') + const recovery = new HaRequestRecovery(f.peer, { coordinator: f.second }) + await f.gate.run(async () => { + const options = { readCompletion: async () => completed(original) } + const outcomes = await Promise.all([ + recovery.recover('concurrent', options), + recovery.recover('concurrent', options), + ]) + assert.equal( + outcomes.every((outcome) => outcome.recovered === true), + true, + ) + const before = await f.peer + .prepare('SELECT grace_until FROM execution_seats WHERE seat_key=$1') + .get(original.admission.seatKey) + const again = await recovery.recover('concurrent', { + readCompletion: async () => assert.fail('already persisted original completion'), + }) + assert.equal(again.recovered, true) + assert.deepEqual( + await f.peer.prepare('SELECT grace_until FROM execution_seats WHERE seat_key=$1').get(original.admission.seatKey), + before, + ) + }) +}) From ea104d8faed0ee8d4ef8118c93bc6979485f096f Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:10:21 +0800 Subject: [PATCH 402/473] feat: recover exact orphan execution graphs from original node completion --- docs/plans/2026-10-22-ha-orphan-recovery.md | 30 +++ src/lib/core/ha-request-recovery.mjs | 225 ++++++++++++++++++-- test/fixtures/orphan-owner.mjs | 9 + test/unit/ha-request-recovery.test.mjs | 39 ++++ 4 files changed, 282 insertions(+), 21 deletions(-) create mode 100644 docs/plans/2026-10-22-ha-orphan-recovery.md create mode 100644 test/fixtures/orphan-owner.mjs diff --git a/docs/plans/2026-10-22-ha-orphan-recovery.md b/docs/plans/2026-10-22-ha-orphan-recovery.md new file mode 100644 index 000000000..29590fd54 --- /dev/null +++ b/docs/plans/2026-10-22-ha-orphan-recovery.md @@ -0,0 +1,30 @@ +# HA 请求的显式孤儿恢复 + +本阶段提供显式服务，不安装定时器，也不在 server 自动调用。仅支持服务器明确标记 `recoveryKind: 'claude_messages'` 的 HA 普通 Claude Messages 与已支持的 OpenAI 转换路径。旧行默认 NULL，不能从 HTTP 路径字符串或同名 request ID 推断恢复资格。 + +## Owner 退出证明 + +`GlobalMaintenanceCoordinator.start()` 在专用 activity PG 连接上取得 owner session advisory lock，然后才登记 `owner_lock_version=1` 的成员。锁没有 TTL，连接关闭或进程 SIGKILL 才释放。成员 UUID 不允许重新登记。旧私有 schema 增加 nullable 列，旧成员仍为 NULL；无锁不构成其退出证明。 + +`lockAbsentOwner(ownerId)` 只允许在当前活跃 activity 的短 PG 事务调用。它核验 open epoch、成员协议版本，并尝试同键 transaction advisory lock。`HaRequestRecovery.seal(requestId)` 在同事务锁定精确 parent 与原 activity，随后关闭 parent 的 child 准入。它不按 owner 批量处理请求。未知历史协议、epoch/owner 不匹配、一个 activity 对应多个 parent、尚未结束的 activity handoff operation 均拒绝。 + +## 原节点终态 + +`recover(requestId, {readCompletion})` 封闭 parent 后释放数据库事务，再逐一调用： + +```js +readCompletion({requestId: originalCommandId, nodeBinding: originalPgBinding}) +// -> {id, fence, status: 'completed', dispatched: true} +``` + +调用方必须用可信固定 TLS 链接查询原 node。Go `recovery-status` 返回 `found:true` 时才可提取 `command`；`found:false`、未 dispatch、pending、unknown、失联、不同 command/Fence 均不能释放。服务不读最新 node State，也不依赖死 Pod 的 ticket。已持久化的原 completion 可以直接复核。 + +所有关联来自 019 的 parent/admission/attempt/job 外键与显式字段。服务再次核验原图，在 parent→device→account→VM→session 的锁顺序下原子记录 completion 并归还该 parent 的执行准入、席位和会话 turn；席位宽限期采用 PG 时间。原 uncertain_at 保留在审计行，重试不延长宽限期。无 bound job 的已封闭受支持请求可归还执行资源，因为该协议在任何节点提交前必须持久 bind。 + +原节点正常 completed 仅依赖现有 kernel 正常终态加 EOF 的协议假设。它不证明未知异常的 provider 行为已经结束。未 dispatch 的已绑定 command 仍需要节点持久 tombstone 或后续可靠停止证明，本阶段保留占位。 + +## 计费与维护限制 + +本阶段不写 usage、model、请求成功状态或金额，也不释放 API key reservation。缺少节点用量、实际模型和 PG 定价快照时，不能按 0 结算。API key 的业务 claim 继续阻止破坏性维护。恢复不会释放任何 VM operation、background task 或其他 parent 的资源。 + +后续用量回执应在节点 completed 的同一 journal 事务持久化，通过原 node binding + command ID 关联 `execution_jobs`。回执仅包含 receipt ID、实际 model、usage_complete 和明确计数白名单；缺字段不得补 0，不保存响应正文、headers 或凭据。控制面按原 pricing snapshot 结算；它不能重发原 command 或重新 mint grant。 diff --git a/src/lib/core/ha-request-recovery.mjs b/src/lib/core/ha-request-recovery.mjs index 0ff80872f..f805430e7 100644 --- a/src/lib/core/ha-request-recovery.mjs +++ b/src/lib/core/ha-request-recovery.mjs @@ -1,4 +1,22 @@ import { getActivityCoordinator, getDb, withTransaction } from '../db/database.mjs' +import { isDeepStrictEqual } from 'node:util' +import { normalizeNodeBinding, normalizeNodeCompletion } from './node-execution-proof.mjs' +import { SharedExecutionAdmissions } from '../pool/shared-execution-admissions.mjs' +import { SharedSessionTurns } from '../pool/shared-session-turns.mjs' + +function unchangedGraph(graph) { + return { + admissions: graph.admissions.map((row) => [row.id, row.account_id, row.vm_id, row.seat_key, row.seat_generation]), + jobs: graph.jobs.map((row) => [ + row.request_id, + row.admission_id, + row.ha_request_id, + row.attempt_no, + row.node_binding, + ]), + attempts: graph.attempts.map((row) => [row.id, row.attempt_no, row.vm_id, row.account_id]), + } +} /** Explicit recovery only. Closing a parent proves no late child can be admitted, not that a node ended. */ export class HaRequestRecovery { @@ -10,31 +28,196 @@ export class HaRequestRecovery { async seal(requestId) { return withTransaction(this.db, async () => { - const request = await this.db.prepare('SELECT * FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get(requestId) - if (!request) return { sealed: false, reason: 'request_missing' } - if (request.recovery_kind !== 'claude_messages') return { sealed: false, reason: 'request_kind_unproven' } - const absent = await this.coordinator.lockAbsentOwner(request.activity_owner_id) - if (!absent.absent) return { sealed: false, reason: absent.reason } - const scope = this.coordinator.assertActivity() - const activity = ( - await this.db.query(`SELECT * FROM ${this.coordinator.qualified}.activities WHERE id=$1 FOR UPDATE`, [ - request.activity_id, - ]) - ).rows[0] + const { request, ...result } = await this._seal(requestId) + return result + }) + } + + async _seal(requestId) { + const request = await this.db.prepare('SELECT * FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get(requestId) + if (!request) return { sealed: false, reason: 'request_missing' } + if (request.recovery_kind !== 'claude_messages') return { sealed: false, reason: 'request_kind_unproven' } + const absent = await this.coordinator.lockAbsentOwner(request.activity_owner_id) + if (!absent.absent) return { sealed: false, reason: absent.reason } + const scope = this.coordinator.assertActivity() + const activity = ( + await this.db.query(`SELECT * FROM ${this.coordinator.qualified}.activities WHERE id=$1 FOR UPDATE`, [ + request.activity_id, + ]) + ).rows[0] + if ( + !activity || + activity.owner_id !== request.activity_owner_id || + String(activity.epoch) !== String(request.activity_epoch) || + String(request.activity_epoch) !== String(scope.epoch) + ) { + return { sealed: false, reason: 'request_activity_mismatch' } + } + const sibling = await this.db + .prepare('SELECT id FROM ha_request_contexts WHERE activity_id=$1 AND id<>$2 LIMIT 1') + .get(request.activity_id, requestId) + if (sibling) return { sealed: false, reason: 'request_activity_ambiguous' } + const operation = ( + await this.db.query( + `SELECT id FROM ${this.coordinator.qualified}.operations WHERE source_activity_id=$1 AND ended_at IS NULL LIMIT 1`, + [request.activity_id], + ) + ).rows[0] + if (operation) return { sealed: false, reason: 'request_operation_pending' } + const sealed = await this.db + .prepare(`UPDATE ha_request_contexts SET accepting_children=false, + recovery_sealed_at=COALESCE(recovery_sealed_at,clock_timestamp()), + recovery_sealed_by=COALESCE(recovery_sealed_by,$2) WHERE id=$1 RETURNING *`) + .get(requestId, this.coordinator.ownerId) + return { sealed: true, request: sealed } + } + + async _graph(requestId) { + const admissions = await this.db + .prepare('SELECT * FROM execution_admissions WHERE ha_request_id=$1 ORDER BY id') + .all(requestId) + const jobs = await this.db + .prepare( + `SELECT * FROM execution_jobs WHERE ha_request_id=$1 OR admission_id=ANY($2::text[]) ORDER BY request_id`, + ) + .all( + requestId, + admissions.map((row) => row.id), + ) + const attempts = await this.db + .prepare('SELECT id,attempt_no,vm_id,account_id FROM request_attempts WHERE ha_request_id=$1 ORDER BY id') + .all(requestId) + const turns = await this.db.prepare('SELECT * FROM session_turns WHERE ha_request_id=$1 ORDER BY id').all(requestId) + return { admissions, jobs, attempts, turns } + } + + _validGraph(requestId, graph) { + for (const job of graph.jobs) { + const admission = graph.admissions.find((row) => row.id === job.admission_id) + const attempts = graph.attempts.filter( + (row) => + row.attempt_no === job.attempt_no && + row.vm_id === admission?.vm_id && + row.account_id === admission?.account_id, + ) + if (!admission || job.ha_request_id !== requestId || attempts.length !== 1 || !job.node_binding) return false + try { + if (normalizeNodeBinding(job.node_binding).vmId !== admission.vm_id) return false + } catch { + return false + } + } + return true + } + + async _billingHeld(requestId) { + return !!(await this.db + .prepare('SELECT id FROM api_key_admissions WHERE ha_request_id=$1 AND released_at IS NULL LIMIT 1') + .get(requestId)) + } + + /** readCompletion authenticates the original node; it must never substitute current State or a new command. */ + async recover(requestId, { readCompletion } = {}) { + const snapshot = await withTransaction(this.db, async () => { + const sealed = await this._seal(requestId) + if (!sealed.sealed) return { reason: sealed.reason } + if (sealed.request.execution_recovered_at) return { already: true } + const graph = await this._graph(requestId) + return this._validGraph(requestId, graph) ? { graph } : { reason: 'request_job_binding_unproven' } + }) + if (snapshot.reason) return { recovered: false, reason: snapshot.reason } + if (snapshot.already) return { recovered: true, billingHeld: await this._billingHeld(requestId) } + const proofs = new Map() + for (const job of snapshot.graph.jobs) { + try { + const value = + job.ended_at && job.node_completion + ? job.node_completion + : await readCompletion?.({ requestId: job.request_id, nodeBinding: structuredClone(job.node_binding) }) + proofs.set( + job.request_id, + normalizeNodeCompletion(value, { requestId: job.request_id, nodeBinding: job.node_binding }), + ) + } catch { + return { recovered: false, reason: 'node_completion_unproven' } + } + } + return withTransaction(this.db, async () => { + const sealed = await this._seal(requestId) + if (!sealed.sealed) return { recovered: false, reason: sealed.reason } + if (sealed.request.execution_recovered_at) + return { recovered: true, billingHeld: await this._billingHeld(requestId) } + const graph = await this._graph(requestId) if ( - !activity || - activity.owner_id !== request.activity_owner_id || - String(activity.epoch) !== String(request.activity_epoch) || - String(request.activity_epoch) !== String(scope.epoch) + !this._validGraph(requestId, graph) || + !isDeepStrictEqual(unchangedGraph(graph), unchangedGraph(snapshot.graph)) ) { - return { sealed: false, reason: 'request_activity_mismatch' } + return { recovered: false, reason: 'request_graph_changed' } } + const shared = new SharedExecutionAdmissions(this.db) + const turns = new SharedSessionTurns(this.db) + for (const key of [...new Set(graph.admissions.map((row) => row.seat_key).filter(Boolean))].sort()) + await shared.lockDevice(key) + const seats = await Promise.all( + graph.admissions.filter((row) => row.seat_key).map((row) => shared.readSeat(row.seat_key)), + ) + for (const accountId of [...new Set(graph.admissions.map((row) => row.account_id))].sort()) + await shared.executions.lockResources({ accountId }) + await shared.executions.lockResources({ + vmIds: [ + ...new Set([...graph.admissions.map((row) => row.vm_id), ...seats.filter(Boolean).map((row) => row.vm_id)]), + ], + }) + for (const key of [...new Set(graph.turns.map((row) => row.session_key))].sort()) await turns.lock(key) await this.db - .prepare(`UPDATE ha_request_contexts SET accepting_children=false, - recovery_sealed_at=COALESCE(recovery_sealed_at,clock_timestamp()), - recovery_sealed_by=COALESCE(recovery_sealed_by,$2) WHERE id=$1`) - .run(requestId, this.coordinator.ownerId) - return { sealed: true } + .prepare('SELECT id FROM execution_admissions WHERE ha_request_id=$1 ORDER BY id FOR UPDATE') + .all(requestId) + await this.db + .prepare('SELECT request_id FROM execution_jobs WHERE ha_request_id=$1 ORDER BY request_id FOR UPDATE') + .all(requestId) + const current = await this._graph(requestId) + if (!isDeepStrictEqual(unchangedGraph(current), unchangedGraph(snapshot.graph))) + return { recovered: false, reason: 'request_graph_changed' } + for (const job of current.jobs) { + await this.db + .prepare(`UPDATE execution_jobs SET ended_at=COALESCE(ended_at,clock_timestamp()),started=true, + reason='node_completed_recovery',node_completion=$3::jsonb WHERE request_id=$1 AND ha_request_id=$2`) + .run(job.request_id, requestId, JSON.stringify(proofs.get(job.request_id))) + } + for (const admission of current.admissions) { + const result = await this.db + .prepare(`UPDATE execution_admissions SET released_at=clock_timestamp() + WHERE id=$1 AND ha_request_id=$2 AND released_at IS NULL`) + .run(admission.id, requestId) + if (!result.changes) continue + await shared.executions.circuits.releaseProbe(admission.id) + if (!admission.seat_key) continue + const seat = await shared.readSeat(admission.seat_key) + if (!seat || seat.generation !== admission.seat_generation || seat.inflight) continue + if (seat.grace_ms > 0) { + await this.db + .prepare(`UPDATE execution_seats SET grace_until=clock_timestamp()+grace_ms*interval '1 millisecond' + WHERE seat_key=$1 AND generation=$2`) + .run(admission.seat_key, admission.seat_generation) + } else { + await this.db + .prepare('DELETE FROM execution_seats WHERE seat_key=$1 AND generation=$2') + .run(admission.seat_key, admission.seat_generation) + } + } + for (const turn of current.turns) + await this.db + .prepare('DELETE FROM session_turns WHERE id=$1 AND owner_id=$2 AND ha_request_id=$3') + .run(turn.id, turn.owner_id, requestId) + await this.db.query( + `UPDATE ${this.coordinator.qualified}.activities SET completed_at=COALESCE(completed_at,clock_timestamp()), + reason='ha_request_execution_recovered' WHERE id=$1 AND owner_id=$2 AND epoch=$3`, + [sealed.request.activity_id, sealed.request.activity_owner_id, sealed.request.activity_epoch], + ) + await this.db + .prepare('UPDATE ha_request_contexts SET execution_recovered_at=clock_timestamp() WHERE id=$1') + .run(requestId) + return { recovered: true, billingHeld: await this._billingHeld(requestId) } }) } } diff --git a/test/fixtures/orphan-owner.mjs b/test/fixtures/orphan-owner.mjs new file mode 100644 index 000000000..9c3c9de60 --- /dev/null +++ b/test/fixtures/orphan-owner.mjs @@ -0,0 +1,9 @@ +import { PostgresDatabase } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' + +const url = process.env.VM2API_TEST_DATABASE_URL +if (!url || !/_test$/.test(new URL(url).pathname)) throw new Error('dedicated test database required') +const db = new PostgresDatabase({ connectionString: url, schema: process.env.KIN_DATABASE_SCHEMA }) +const coordinator = new GlobalMaintenanceCoordinator(db, { ownerId: 'synthetic-killed-owner' }) +await coordinator.start() +process.send({ ready: true }) diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs index 30ed4d0fe..c50b9f7b0 100644 --- a/test/unit/ha-request-recovery.test.mjs +++ b/test/unit/ha-request-recovery.test.mjs @@ -384,3 +384,42 @@ test('concurrent recoveries are idempotent and cannot extend recovered seat grac ) }) }) + +test('SIGKILL of a synthetic owner process releases only its connection lock, with no timestamp timeout', async (t) => { + const { spawn } = await import('node:child_process') + const { fileURLToPath } = await import('node:url') + const f = await fixture(t) + const child = spawn(process.execPath, [fileURLToPath(new URL('../fixtures/orphan-owner.mjs', import.meta.url))], { + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], + }) + t.after(() => { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + }) + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('synthetic owner readiness timeout')), 5000) + child.once('error', (error) => { + clearTimeout(timer) + reject(error) + }) + child.once('exit', () => { + clearTimeout(timer) + reject(new Error('synthetic owner exited before readiness')) + }) + child.once('message', (message) => { + clearTimeout(timer) + message.ready === true ? resolve() : reject(new Error('invalid readiness')) + }) + }) + await f.gate.run(async () => { + assert.deepEqual(await withTransaction(f.peer, () => f.second.lockAbsentOwner('synthetic-killed-owner')), { + absent: false, + reason: 'owner_connected', + }) + const exited = new Promise((resolve) => child.once('exit', resolve)) + assert.equal(child.kill('SIGKILL'), true) + await exited + assert.deepEqual(await withTransaction(f.peer, () => f.second.lockAbsentOwner('synthetic-killed-owner')), { + absent: true, + }) + }) +}) From bf29220a673ef05e201f65c1fff81a03fe2c86d0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:11:12 +0800 Subject: [PATCH 403/473] test: retain unsettled node proofs and permit fresh HA activity contexts --- test/unit/ha-request-context.test.mjs | 35 ++++++++++++++++++ test/unit/ha-request-recovery.test.mjs | 49 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/test/unit/ha-request-context.test.mjs b/test/unit/ha-request-context.test.mjs index fea0aaef8..7eb756583 100644 --- a/test/unit/ha-request-context.test.mjs +++ b/test/unit/ha-request-context.test.mjs @@ -241,3 +241,38 @@ for (const mode of ['standalone', 'kubernetes']) { }) }) } + +test('a closed inherited context permits a newly admitted activity but cannot revive its original activity', async () => { + await fixture(async ({ db, activity, withHaRequestContext }) => { + let resume + const later = new Promise((resolve) => { + resume = resolve + }) + let inherited + await withHaRequestContext({ db, activity, requestId: 'closed-parent' }, async () => { + inherited = later.then(async () => { + await assert.rejects( + withHaRequestContext({ db, activity, requestId: 'revived-parent' }, async () => {}), + { code: 'ha_request_context_closed' }, + ) + const fresh = await getActivityCoordinator(db).admit({ kind: 'request', requestId: 'fresh-after-closed' }) + try { + await fresh.run(() => + withHaRequestContext({ db, activity: fresh, requestId: 'fresh-after-closed' }, async () => { + const lease = await new ExecutionAdmissionsRepo(db).tryReserve(input()) + assert.equal(lease.ok, true) + await lease.release() + }), + ) + } finally { + await fresh.complete({ ended: true }) + } + }) + inherited.catch(() => {}) + }) + resume() + await inherited + assert.equal(await db.prepare('SELECT id FROM ha_request_contexts WHERE id=$1').get('revived-parent'), undefined) + assert.ok(await db.prepare('SELECT id FROM ha_request_contexts WHERE id=$1').get('fresh-after-closed')) + }) +}) diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs index c50b9f7b0..dd0ff5ee3 100644 --- a/test/unit/ha-request-recovery.test.mjs +++ b/test/unit/ha-request-recovery.test.mjs @@ -9,6 +9,7 @@ import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' import { SharedExecutionAdmissions } from '../../src/lib/pool/shared-execution-admissions.mjs' import { SharedSessionTurns } from '../../src/lib/pool/shared-session-turns.mjs' import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { syntheticFence } from '../fixtures/fake-slotgate.mjs' async function fixture(t) { @@ -423,3 +424,51 @@ test('SIGKILL of a synthetic owner process releases only its connection lock, wi }) }) }) + +for (const heldBy of ['pending-billing', 'sealed-recovery', 'normal-finished']) { + test(`execution history cleanup preserves original proof for ${heldBy}`, async (t) => { + const f = await fixture(t) + const original = await graph(f, 'old-history') + await f.first.stop() + const { HaRequestRecovery } = await import('../../src/lib/core/ha-request-recovery.mjs') + const recovery = new HaRequestRecovery(f.peer, { coordinator: f.second }) + await f.gate.run(async () => { + if (heldBy === 'pending-billing') { + assert.equal( + (await recovery.recover('old-history', { readCompletion: async () => completed(original) })).recovered, + true, + ) + } else { + if (heldBy === 'sealed-recovery') assert.equal((await recovery.seal('old-history')).sealed, true) + await f.peer + .prepare('UPDATE execution_jobs SET ended_at=clock_timestamp(),node_completion=$2::jsonb WHERE request_id=$1') + .run(original.commandId, JSON.stringify(completed(original))) + await f.peer + .prepare('UPDATE execution_admissions SET released_at=clock_timestamp() WHERE id=$1') + .run(original.admission.admissionId) + await f.peer + .prepare('UPDATE api_key_admissions SET released_at=clock_timestamp() WHERE id=$1') + .run(original.keyLease.reservationId) + } + await f.peer + .prepare("UPDATE execution_admissions SET created_at=clock_timestamp()-interval '2 minutes' WHERE id=$1") + .run(original.admission.admissionId) + await withHaRequestContext({ db: f.peer, requestId: 'fresh-request', activity: f.gate }, async () => { + const lease = await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + accountId: 'fixture-account', + vmId: 'vm-fixture', + accountLimit: 10, + vmLimit: 10, + fableLimit: 0, + rpmLimit: 0, + }) + assert.equal(lease.ok, true) + await lease.release() + }) + const job = await f.peer.prepare('SELECT * FROM execution_jobs WHERE request_id=$1').get(original.commandId) + if (heldBy === 'normal-finished') assert.equal(job, undefined) + else + assert.deepEqual(job?.node_completion, completed(original), 'the only durable node proof must survive cleanup') + }) + }) +} From 1d0e0f7c00cfba40114929d3c91dea4e9895930c Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:11:59 +0800 Subject: [PATCH 404/473] fix: preserve unsettled HA proofs and admit fresh activity contexts --- docs/plans/2026-10-22-ha-orphan-recovery.md | 4 ++++ src/lib/core/ha-request-context.mjs | 13 ++++++++++++- src/lib/db/repos/execution-admissions-repo.mjs | 7 ++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/plans/2026-10-22-ha-orphan-recovery.md b/docs/plans/2026-10-22-ha-orphan-recovery.md index 29590fd54..26c867fc6 100644 --- a/docs/plans/2026-10-22-ha-orphan-recovery.md +++ b/docs/plans/2026-10-22-ha-orphan-recovery.md @@ -28,3 +28,7 @@ readCompletion({requestId: originalCommandId, nodeBinding: originalPgBinding}) 本阶段不写 usage、model、请求成功状态或金额，也不释放 API key reservation。缺少节点用量、实际模型和 PG 定价快照时，不能按 0 结算。API key 的业务 claim 继续阻止破坏性维护。恢复不会释放任何 VM operation、background task 或其他 parent 的资源。 后续用量回执应在节点 completed 的同一 journal 事务持久化，通过原 node binding + command ID 关联 `execution_jobs`。回执仅包含 receipt ID、实际 model、usage_complete 和明确计数白名单；缺字段不得补 0，不保存响应正文、headers 或凭据。控制面按原 pricing snapshot 结算；它不能重发原 command 或重新 mint grant。 + +历史清理保留两类原始执行图：仍有未 release 的同 parent API key reservation；已封闭但 execution_recovered_at 仍为空的 parent。正常已结束且无这些待处理关联的历史仍遵循旧清理策略。节点 proof 的长期独立审计存储和完整计费恢复尚未实现。 + +新 HTTP/异步请求可能继承已关闭的 HA context。新 context 仅在新真实 coordinator gate 的 scope 下允许建立；继承的 active context 仍拒绝嵌套，原 activity 无法借 closed context 重新建立 parent。attempt context 共用父 context 的关闭状态。 diff --git a/src/lib/core/ha-request-context.mjs b/src/lib/core/ha-request-context.mjs index 46b36bf40..b37a0ad9d 100644 --- a/src/lib/core/ha-request-context.mjs +++ b/src/lib/core/ha-request-context.mjs @@ -38,7 +38,8 @@ export async function lockHaRequestContext(db) { /** Correlation only: closing this context never releases a lease or unknown work. */ export async function withHaRequestContext({ db = getDb(), requestId, activity, recoveryKind = null }, callback) { if (!isKubernetesHaMode()) return callback() - if (requests.getStore()) throw failure('ha_request_context_nested') + const inherited = requests.getStore() + if (inherited?.isActive()) throw failure('ha_request_context_nested') const coordinator = getActivityCoordinator(db) const scope = coordinator?.assertActivity() if ( @@ -48,6 +49,15 @@ export async function withHaRequestContext({ db = getDb(), requestId, activity, String(activity?.epoch) !== String(scope.epoch) ) throw failure('ha_request_activity_mismatch') + if ( + inherited && + inherited.db.schema === db.schema && + inherited.db.connectionString === db.connectionString && + inherited.activityId === scope.id && + inherited.activityOwnerId === coordinator.ownerId && + String(inherited.activityEpoch) === String(scope.epoch) + ) + throw failure('ha_request_context_closed') if (typeof requestId !== 'string' || !requestId.trim() || requestId.length > 128) throw failure('ha_request_id_invalid') if (recoveryKind !== null && recoveryKind !== 'claude_messages') throw failure('ha_request_recovery_kind_invalid') @@ -64,6 +74,7 @@ export async function withHaRequestContext({ db = getDb(), requestId, activity, activityOwnerId: coordinator.ownerId, activityEpoch: scope.epoch, attemptNo: null, + isActive: () => active, assertActive: () => { if (!active) throw failure('ha_request_context_closed') }, diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 27be49ac3..a6de91281 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -132,7 +132,12 @@ export class ExecutionAdmissionsRepo { const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() await this.db .prepare(`DELETE FROM execution_admissions WHERE account_id = $1 - AND released_at IS NOT NULL AND created_at <= $2::timestamptz - interval '60 seconds'`) + AND released_at IS NOT NULL AND created_at <= $2::timestamptz - interval '60 seconds' + AND NOT EXISTS (SELECT 1 FROM api_key_admissions k + WHERE k.ha_request_id=execution_admissions.ha_request_id AND k.released_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM ha_request_contexts r + WHERE r.id=execution_admissions.ha_request_id AND r.recovery_sealed_at IS NOT NULL + AND r.execution_recovered_at IS NULL)`) .run(accountId, now) const counts = await this._counts(accountId, vmId, now) // Concurrency zero disables execution; RPM/Fable zero adds no sublimit. From b0ee367d037055f3d538a612993772c39431e004 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:21:49 +0800 Subject: [PATCH 405/473] test: require host-free HA container entrypoint and shared secrets --- test/unit/ha-entrypoint.test.mjs | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test/unit/ha-entrypoint.test.mjs diff --git a/test/unit/ha-entrypoint.test.mjs b/test/unit/ha-entrypoint.test.mjs new file mode 100644 index 000000000..2619f3829 --- /dev/null +++ b/test/unit/ha-entrypoint.test.mjs @@ -0,0 +1,44 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { spawnSync } from 'node:child_process' + +test('HA image startup stages only local config and requires shared secrets without Docker or payload writes', (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-648-ha-entrypoint-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const image = path.join(root, 'image') + const project = path.join(root, 'project') + const bin = path.join(root, 'stub-bin') + fs.mkdirSync(path.join(image, 'image-config'), { recursive: true }) + fs.mkdirSync(bin) + fs.writeFileSync(path.join(image, 'image-config', 'routing.json'), '{}') + const called = path.join(root, 'node-called') + fs.writeFileSync(path.join(bin, 'node'), `#!/bin/sh\nprintf '%s\\n' "$*" > '${called}'\n`, { mode: 0o700 }) + const source = fs.readFileSync(new URL('../../scripts/docker-entrypoint.sh', import.meta.url), 'utf8') + const script = path.join(root, 'entrypoint.sh') + // Relocate only the image's absolute fixture path; execute the actual shell logic. + fs.writeFileSync(script, source.replaceAll('/opt/vm2api', image)) + const env = { + ...process.env, PATH: bin + path.delimiter + process.env.PATH, + VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', KIN_PROJECT_ROOT: project, + VM2API_DATABASE_URL: 'postgresql://synthetic@127.0.0.1:1/unreachable_test', + VM2API_API_KEY: 'synthetic-ha-key', VM2API_ADMIN_PASSWORD: 'synthetic-ha-password', + VM2API_DB_SECRET: 'synthetic-ha-encryption-secret', + } + const result = spawnSync('sh', [script], { env, encoding: 'utf8', timeout: 10_000 }) + assert.equal(result.status, 0, result.stdout + result.stderr) + assert.equal(fs.readFileSync(called, 'utf8').trim(), 'src/server.mjs') + assert.equal(fs.readFileSync(path.join(project, 'src/config/routing.json'), 'utf8'), '{}') + assert.equal(fs.existsSync(path.join(project, 'bin')), false) + assert.equal(fs.existsSync(path.join(project, 'share')), false) + assert.equal(fs.existsSync(path.join(project, '.env')), false) + fs.rmSync(called) + delete env.VM2API_DB_SECRET + delete env.KIN_DB_SECRET + const refused = spawnSync('sh', [script], { env, encoding: 'utf8', timeout: 10_000 }) + assert.notEqual(refused.status, 0) + assert.match(refused.stderr, /VM2API_DB_SECRET/) + assert.equal(fs.existsSync(called), false, 'missing shared encryption key cannot start a divergent replica') +}) From a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:22:10 +0800 Subject: [PATCH 406/473] feat: start HA images with shared secrets and without host assets --- scripts/docker-entrypoint.sh | 19 +++++++++++++++++++ test/unit/ha-entrypoint.test.mjs | 9 ++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 959baa650..c33303104 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -7,6 +7,25 @@ if [ -z "${VM2API_DATABASE_URL:-${KIN_DATABASE_URL:-}}" ]; then echo "vm2api: PostgreSQL 17 connection is required. Configure VM2API_DATABASE_URL; see docs/deploy/postgresql.md for SQLite import." >&2 exit 1 fi + +# Runtime HA Pods only use PostgreSQL and the fixed TLS node interface. Assets +# and execution privileges stay on trusted nodes; per-Pod random secrets would +# split authentication and encrypted state between replicas. +if [ "${VM2API_CONTROL_PLANE_MODE:-}" = kubernetes-ha ]; then + if [ -z "${VM2API_API_KEY:-${KIN_API_KEY:-}}" ] || [ -z "${VM2API_ADMIN_PASSWORD:-${KIN_ADMIN_PASSWORD:-}}" ] || [ -z "${VM2API_DB_SECRET:-${KIN_DB_SECRET:-}}" ]; then + echo "vm2api: kubernetes-ha requires shared VM2API_API_KEY, VM2API_ADMIN_PASSWORD and VM2API_DB_SECRET." >&2 + exit 1 + fi + KIN_DATA_DIR="${KIN_DATA_DIR:-$ROOT/data}" + KIN_CAPTURES_DIR="${KIN_CAPTURES_DIR:-$KIN_DATA_DIR/captures}" + export KIN_DATA_DIR KIN_CAPTURES_DIR + mkdir -p "$ROOT/src/config" "$KIN_DATA_DIR" "$KIN_CAPTURES_DIR" + if [ ! -f "$ROOT/src/config/routing.json" ]; then + cp /opt/vm2api/image-config/routing.json "$ROOT/src/config/routing.json" + fi + cd /opt/vm2api + exec node src/server.mjs +fi mkdir -p "$ROOT/vms" "$ROOT/data" "$ROOT/bin" # 镜像安装没有仓库 checkout：把镜像内默认配置补进被 bind-mount 覆盖的 src/config。 diff --git a/test/unit/ha-entrypoint.test.mjs b/test/unit/ha-entrypoint.test.mjs index 2619f3829..99fdcd7d6 100644 --- a/test/unit/ha-entrypoint.test.mjs +++ b/test/unit/ha-entrypoint.test.mjs @@ -21,10 +21,13 @@ test('HA image startup stages only local config and requires shared secrets with // Relocate only the image's absolute fixture path; execute the actual shell logic. fs.writeFileSync(script, source.replaceAll('/opt/vm2api', image)) const env = { - ...process.env, PATH: bin + path.delimiter + process.env.PATH, - VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', KIN_PROJECT_ROOT: project, + ...process.env, + PATH: bin + path.delimiter + process.env.PATH, + VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', + KIN_PROJECT_ROOT: project, VM2API_DATABASE_URL: 'postgresql://synthetic@127.0.0.1:1/unreachable_test', - VM2API_API_KEY: 'synthetic-ha-key', VM2API_ADMIN_PASSWORD: 'synthetic-ha-password', + VM2API_API_KEY: 'synthetic-ha-key', + VM2API_ADMIN_PASSWORD: 'synthetic-ha-password', VM2API_DB_SECRET: 'synthetic-ha-encryption-secret', } const result = spawnSync('sh', [script], { env, encoding: 'utf8', timeout: 10_000 }) From e72112ece78678b5a1b938242c56b4b039154e5d Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:21:04 +0800 Subject: [PATCH 407/473] test: seal an original undispatched node command before delayed send --- .../slotgate/seal_not_started_test.go | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 worker/internal/slotgate/seal_not_started_test.go diff --git a/worker/internal/slotgate/seal_not_started_test.go b/worker/internal/slotgate/seal_not_started_test.go new file mode 100644 index 000000000..3ccdd443c --- /dev/null +++ b/worker/internal/slotgate/seal_not_started_test.go @@ -0,0 +1,226 @@ +package slotgate + +import ( + "bufio" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +func sealWire(addr string, request gate2Request) (recoveryStatusReply, error) { + var reply recoveryStatusReply + c, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS13}) + if err != nil { + return reply, err + } + defer c.Close() + c.SetDeadline(time.Now().Add(5 * time.Second)) + data, _ := json.Marshal(request) + if _, err = fmt.Fprintf(c, "KSG2 %s\n%s\n", testToken, data); err != nil { + return reply, err + } + err = json.NewDecoder(c).Decode(&reply) + return reply, err +} + +func readSeal(t *testing.T, addr string, request gate2Request) recoveryStatusReply { + t.Helper() + reply, err := sealWire(addr, request) + if err != nil { + t.Fatal(err) + } + return reply +} + +func requireNotStarted(t *testing.T, reply recoveryStatusReply, request gate2Request) { + t.Helper() + if !reply.OK || reply.Found == nil || !*reply.Found || reply.Command == nil || reply.Command.ID != request.CommandID || reply.Command.Fence != request.Fence || reply.Command.Status != "not_started" || reply.Command.Dispatched || reply.Unknown || len(reply.State) != 0 { + t.Fatalf("no original durable not_started proof: %+v", reply) + } +} + +func TestNodeSealNotStartedPersistsAbsentCommandAndRejectsLatePrepare(t *testing.T) { + _, addr, s, state := activeHealthGate(t) + request := gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: "absent-sealed-command"} + before := readRecoveryStatus(t, addr, gate2Request{Op: "recovery-status", VMID: "v", Fence: state.Fence, CommandID: request.CommandID}) + if before.Found == nil || *before.Found || !before.Unknown { + t.Fatal("fixture already contains command") + } + requireNotStarted(t, readSeal(t, addr, request), request) + requireNotStarted(t, readSeal(t, addr, request), request) + late := gate2Control(t, addr, gate2Request{Op: "prepare", VMID: "v", Fence: state.Fence, CommandID: request.CommandID, Kind: "kernel"}) + if late.OK || late.Ticket != "" { + t.Fatal("sealed ID authorized delayed prepare") + } + if err := s.authority.Close(); err != nil { + t.Fatal(err) + } + reopened := authorityForTest(t, s.cfg.AuthorityDir, terminatedExecutor) + historical, err := reopened.recoveryStatus("v", state.Fence, request.CommandID) + if err != nil || historical == nil || historical.Status != "not_started" || historical.Dispatched { + t.Fatalf("restart lost tombstone: %+v %v", historical, err) + } + if _, err = reopened.BeginCommand("v", state.Fence, request.CommandID, CommandForward); err == nil { + t.Fatal("restarted daemon reissued sealed ID") + } +} + +func TestNodeSealNotStartedPreventsAnAlreadyPreparedDelayedForward(t *testing.T) { + f, addr, s, state := activeHealthGate(t) + request := forwardRequest(t, addr, state, "prepared-sealed-command") + seal := gate2Request{Op: "seal-not-started", VMID: "v", Fence: request.Fence, CommandID: request.CommandID} + requireNotStarted(t, readSeal(t, addr, seal), seal) + var calls atomic.Int64 + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + fmt.Fprint(w, normalClaudeJSON) + })) + c, reader := gate2Connect(t, addr, testToken, request, claudeWire(request.CommandID)) + defer c.Close() + line, err := reader.ReadString('\n') + if err != nil || line == "OK\n" || calls.Load() != 0 { + t.Fatalf("sealed prepared request reached kernel: %q %v calls=%d", line, err, calls.Load()) + } + current, _ := s.authority.State("v") + if current.Commands[request.CommandID].Dispatched { + t.Fatal("delayed forward changed sealed command") + } +} + +func TestNodeSealNotStartedAndDispatchAreMutuallyExclusive(t *testing.T) { + f, addr, s, state := activeHealthGate(t) + var calls atomic.Int64 + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, normalClaudeJSON) + })) + for i := 0; i < 12; i++ { + request := forwardRequest(t, addr, state, fmt.Sprintf("seal-dispatch-race-%d", i)) + seal := gate2Request{Op: "seal-not-started", VMID: "v", Fence: request.Fence, CommandID: request.CommandID} + start := make(chan struct{}) + type result struct { + reply recoveryStatusReply + line string + err error + } + sealDone, forwardDone := make(chan result, 1), make(chan result, 1) + go func() { + <-start + reply, err := sealWire(addr, seal) + sealDone <- result{reply: reply, err: err} + }() + go func() { + <-start + c, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS13}) + if err != nil { + forwardDone <- result{err: err} + return + } + defer c.Close() + c.SetDeadline(time.Now().Add(5 * time.Second)) + data, _ := json.Marshal(request) + _, err = fmt.Fprintf(c, "KSG2 %s\n%s\n%s", testToken, data, claudeWire(request.CommandID)) + reader := bufio.NewReader(c) + var line string + if err == nil { + line, err = reader.ReadString('\n') + } + if err == nil && line == "OK\n" { + _, err = io.Copy(io.Discard, reader) + } + forwardDone <- result{line: line, err: err} + }() + before := calls.Load() + close(start) + sealed, forwarded := <-sealDone, <-forwardDone + if sealed.err != nil || forwarded.err != nil { + t.Fatalf("race transport failed: %v / %v", sealed.err, forwarded.err) + } + if sealed.reply.OK { + requireNotStarted(t, sealed.reply, seal) + if forwarded.line == "OK\n" || calls.Load() != before { + t.Fatal("not_started proof overlapped an actual dispatch") + } + } else { + if forwarded.line != "OK\n" || calls.Load() != before+1 { + t.Fatalf("neither dispatch nor seal won: %+v / %+v", sealed, forwarded) + } + waitCommandStatus(t, s.authority, request.CommandID, CommandCompleted) + } + } +} + +func TestNodeSealNotStartedRejectsEveryDispatchedState(t *testing.T) { + _, addr, s, state := activeHealthGate(t) + for _, completed := range []bool{false, true} { + id := fmt.Sprintf("already-dispatched-%t", completed) + handle, err := s.authority.BeginCommand("v", state.Fence, id, CommandForward) + if err != nil { + t.Fatal(err) + } + if err = s.authority.DispatchCommand(handle, func(ExecutorIdentity) error { return nil }); err != nil { + t.Fatal(err) + } + if completed { + if err = s.authority.CompleteCommand(handle); err != nil { + t.Fatal(err) + } + } + reply := readSeal(t, addr, gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: id}) + if reply.OK || reply.Command != nil { + t.Fatalf("dispatched execution was relabeled not_started: %+v", reply) + } + } +} + +func TestNodeSealNotStartedRejectsNonManagedOrRelabeledAuthority(t *testing.T) { + addr, _, _, legacy, _ := fencedGate(t) + if reply := readSeal(t, addr, gate2Request{Op: "seal-not-started", VMID: "v", Fence: legacy.Fence, CommandID: "unmanaged-deny"}); reply.OK { + t.Fatal("runtime token gained unconfigured management authority") + } + _, addr, s, state := activeHealthGate(t) + wrong := state.Fence + wrong.Generation++ + for _, request := range []gate2Request{ + {Op: "seal-not-started", VMID: "v", Fence: wrong, CommandID: "wrong-generation"}, + {Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: "wrong-kind", Kind: "worker"}, + {Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: "wrong-ticket", Ticket: "arbitrary"}, + } { + if reply := readSeal(t, addr, request); reply.OK || reply.Command != nil { + t.Fatalf("relabeled/expanded deny request accepted: %+v", reply) + } + } + current, _ := s.authority.State("v") + if len(current.Commands) != len(state.Commands) { + t.Fatal("rejected request created a tombstone") + } +} + +func TestNodeSealNotStartedPersistenceFailureNeverReturnsProof(t *testing.T) { + _, addr, s, state := activeHealthGate(t) + journal := filepath.Join(s.cfg.AuthorityDir, "authority.json") + if err := os.Rename(journal, journal+".saved"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(journal, 0o700); err != nil { + t.Fatal(err) + } + request := gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: "seal-fsync-failure"} + reply := readSeal(t, addr, request) + if reply.OK || reply.Command != nil { + t.Fatalf("failed journal write returned a proof: %+v", reply) + } + request.Op = "recovery-status" + reply = readRecoveryStatus(t, addr, request) + if reply.OK || reply.Command != nil { + t.Fatalf("failed seal remained authoritative: %+v", reply) + } +} From 59487d0583ac89ba2158685cac08b69ddf5763b4 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:24:49 +0800 Subject: [PATCH 408/473] feat: persist a deny-only seal for original undispatched node commands --- docs/plans/2026-10-22-node-recovery.md | 14 +- worker/internal/slotgate/authority.go | 6 +- worker/internal/slotgate/ksg2.go | 16 +- worker/internal/slotgate/seal_not_started.go | 56 +++++++ .../slotgate/seal_not_started_test.go | 156 +++++++++++++++++- 5 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 worker/internal/slotgate/seal_not_started.go diff --git a/docs/plans/2026-10-22-node-recovery.md b/docs/plans/2026-10-22-node-recovery.md index cba826987..eeb74ee86 100644 --- a/docs/plans/2026-10-22-node-recovery.md +++ b/docs/plans/2026-10-22-node-recovery.md @@ -26,6 +26,16 @@ bootID 不匹配仅拒绝新授权，不是旧 executor 终止证明，不更新 generation，也不释放 PG 容量。已有旧 boot 的 ACTIVE 状态仍需撤销、保留未知并走可信恢复。历史 recovery-status 按原 command/Fence 读取已持久结果，不要求当前 boot 或当前 generation 相同。 +## 原 command 未发送封存 + +失败规格 `2e3b6af2` 覆盖原先缺少未发送封存接口的情况。新增 authenticated KSG2 请求 `{op:"seal-not-started",vm_id,fence,command_id}`。它只拒绝该 command ID 的未来发送，不发 kernel/Engine 请求，不停止任何作业，也不增加 revoke、advance、stop 或 refresh 权限。原 runtime token 已能通过 prepare 占用同 ID；此 deny 操作使用同一 token，但仅允许当前 ACTIVE、当前 boot、完整原 Fence 完全匹配且已受信任 allocate/publish 的 managed kernel incarnation。未知管理动作、旧 generation、非 managed 状态或不匹配的 manifest incarnation 均拒绝。接口不接受 ticket、kind、management grant、manifest 替换或 body。 + +Authority 在与 Dispatch 相同的锁下检查原 ID。不存在的 ID 或原同 Fence 的 pending 且未 dispatch 的 inference command 可以写入持久 tombstone。响应为 `{ok:true,found:true,command:{id,fence,status:"not_started",dispatched:false}}`，不含用量凭证或最新 state。`not_started` 独立于表示已验证 executor 终止的 `terminated`，也不属于 dispatched completed。既有 dispatched 标记在任何状态都拒绝 seal。 + +先封存成功时，之后的同 ID prepare/forward 永久拒绝；先 Dispatch 持久化时，seal 不返回未发送证明。两者不通过连接关闭或时间推断胜负。写盘失败时不返回证明，Authority 保持不可用。节点重启保留已持久 tombstone；原 recovery-status 可按原 Fence 读取它。重复 seal 仅在同一当前授权下幂等，不重新授权旧 generation 的 notfound，也不把请求身份替换为最新 state。 + +本批测试使用 fake Engine、临时 Unix/TLS 监听和合成 journal。覆盖与 prepare/forward 的并发、迟到发送、重启、原 Fence 拒绝和写盘故障。PG 释放须另行核对原持久 job/attempt 身份；本接口没有 PG 释放逻辑，不以 TTL 释放容量。 + ## 正常终态的用量凭证 失败规格 `c3a3bc6c` 证明已完成执行原先没有可供死 controller 后续对账的持久用量信息。正常 JSON/SSE 终态及完整 body/trailer 验证后，节点在同一次 journal 提交中记录 completed 和 `terminal_receipt`。凭证 ID 由节点生成，不接受客户端提交。原 command 的 ID/Fence 与凭证保存在同一条记录；`recovery-status` 在原 command 旁返回顶层 `receipt`，保持 command 的四个原字段不变。历史读取不会将其他 command 的凭证拼接到当前身份。 @@ -34,8 +44,8 @@ bootID 不匹配仅拒绝新授权，不是旧 executor 终止证明，不更新 `usage_complete:true` 还要求实际 model 与四个主计数都明确存在，且未遇到非法、重复或越界的用量。缺失字段保持缺失，不填零，不以请求 model 替代实际 model。普通终态已经确认但用量不完整时仍可记录 completed；凭证标记 `usage_complete:false`，调用方保留待对账状态。凭证不含正文、thinking、工具参数、headers、凭据、USD 或定价输入。HTTP 错误终态、截断、尾随错误、无效 trailer 或 journal 失败不产生可用完成凭证。 -## 后续 seal 与 PG 对账 +## 后续 PG 对账 -当前本批尚未提供 seal-not-started。notfound 不等于未开始；未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 +notfound 不等于未开始；原 command 未持久 seal 时，未 dispatch 也不能直接释放，因为旧 owner 仍可能延迟发送。 usage 缺失不得补零结算；节点只提供原始公开 model/usage/terminal 信息，PG 根据原 command 对应的 parent/attempt 和价格快照处理。node health、ready_slots 和取消 ACK 均不是恢复终止证明。以上节点能力不表示生产 HA 已完成。 diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index f9ce4b81d..d73e0156a 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -52,6 +52,7 @@ const ( CommandUnknown CommandStatus = "unknown" CommandCompleted CommandStatus = "completed" CommandTerminated CommandStatus = "terminated" + CommandNotStarted CommandStatus = "not_started" ) // ExecutorIdentity names an incarnation, never a reusable container name or @@ -810,6 +811,9 @@ func validateJournal(journal authorityJournal) error { return ErrInvalidAuthority } for id, command := range state.Commands { + if command.Status == CommandNotStarted && (command.Kind != CommandForward || command.Dispatched || command.TerminationID != "" || command.TerminalReceipt != nil || validateExecutor(command.Fence.Executor) != nil) { + return ErrInvalidAuthority + } if receipt := command.TerminalReceipt; receipt != nil && (command.Kind != CommandForward || command.Status != CommandCompleted || !command.Dispatched || !validInferenceReceipt(*receipt)) { return ErrInvalidAuthority } @@ -840,7 +844,7 @@ func validateJournal(journal authorityJournal) error { return ErrInvalidAuthority } } - if command.Status != CommandPending && command.Status != CommandUnknown && command.Status != CommandCompleted && command.Status != CommandTerminated { + if command.Status != CommandPending && command.Status != CommandUnknown && command.Status != CommandCompleted && command.Status != CommandTerminated && command.Status != CommandNotStarted { return ErrInvalidAuthority } if pendingCommand(command) && command.Fence != state.Fence { diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index 89424e840..994873daa 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -158,6 +158,20 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["unknown"] = true } } + case "seal-not-started": + if request.Kind != "" || request.Ticket != "" || request.AllocationID != "" || request.OperationID != "" || request.OwnerID != "" || request.Management != nil || request.ContentLength != 0 { + err = ErrInvalidAuthority + break + } + if err = s.currentBootFence(request.Fence); err != nil { + break + } + var command *recoveryCommandStatus + command, err = s.authority.sealNotStarted(request.VMID, request.Fence, request.CommandID) + if err == nil { + reply["found"] = true + reply["command"] = command + } case "allocate": if s.allocator == nil || request.Fence != (Fence{}) || request.CommandID != "" || request.Kind != "" || request.Ticket != "" { err = ErrManagementDenied @@ -257,7 +271,7 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["error"] = gate2Code(err) } else { reply["ok"] = true - if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" && request.Op != "health" && request.Op != "recovery-status" { + if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" && request.Op != "health" && request.Op != "recovery-status" && request.Op != "seal-not-started" { reply["state"] = publicGateState(state) } } diff --git a/worker/internal/slotgate/seal_not_started.go b/worker/internal/slotgate/seal_not_started.go new file mode 100644 index 000000000..b6005ac87 --- /dev/null +++ b/worker/internal/slotgate/seal_not_started.go @@ -0,0 +1,56 @@ +package slotgate + +// sealNotStarted is a deny operation, not execution or termination authority. +// It shares DispatchCommand's lock and journal: once this commit succeeds no +// delayed prepare/forward with the same ID can send a byte. A dispatched marker +// always wins, even if the response or the controller has subsequently vanished. +func (a *Authority) sealNotStarted(vm string, fence Fence, id string) (*recoveryCommandStatus, error) { + a.mu.Lock() + defer a.mu.Unlock() + if !vmIDRe.MatchString(vm) || !validAuthorityID(id) || !publicHealthString(id) { + return nil, ErrInvalidAuthority + } + state, err := a.match(vm, fence) + if err != nil { + return nil, err + } + if state.Phase != PhaseActive || state.Allocation == nil || state.InitialManifest == nil || hasPendingMutation(state) { + return nil, ErrAuthorityPhase + } + if err = validateInitialManifest(vm, state); err != nil { + return nil, err + } + manifestFence := state.InitialManifest.Receipt.Intent.Fence + if fence.Generation != manifestFence.Generation || fence.Incarnation != manifestFence.Incarnation { + return nil, ErrFenceMismatch + } + command, exists := state.Commands[id] + if exists { + if command.Fence != fence { + return nil, ErrFenceMismatch + } + if command.Kind != CommandForward { + return nil, ErrManagementDenied + } + if command.Dispatched { + return nil, ErrCommandDispatched + } + if command.Status != CommandPending && command.Status != CommandNotStarted { + return nil, ErrCommandOwner + } + } else { + token, err := randomAuthorityID() + if err != nil { + return nil, err + } + command = CommandRecord{ID: id, Kind: CommandForward, Status: CommandPending, Fence: fence, Runtime: a.runtime, Token: token} + } + if command.Status != CommandNotStarted { + command.Status = CommandNotStarted + state.Commands[id] = command + if _, err = a.commit(vm, state); err != nil { + return nil, err + } + } + return &recoveryCommandStatus{ID: command.ID, Fence: command.Fence, Status: command.Status, Dispatched: false}, nil +} diff --git a/worker/internal/slotgate/seal_not_started_test.go b/worker/internal/slotgate/seal_not_started_test.go index 3ccdd443c..d0e0eb4ff 100644 --- a/worker/internal/slotgate/seal_not_started_test.go +++ b/worker/internal/slotgate/seal_not_started_test.go @@ -2,8 +2,10 @@ package slotgate import ( "bufio" + "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -158,10 +160,45 @@ func TestNodeSealNotStartedAndDispatchAreMutuallyExclusive(t *testing.T) { } } +func TestNodeSealNotStartedCannotBeOverwrittenByConcurrentPrepare(t *testing.T) { + _, addr, s, state := activeHealthGate(t) + for i := 0; i < 12; i++ { + request := gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: fmt.Sprintf("seal-prepare-race-%d", i)} + prepare := request + prepare.Op, prepare.Kind = "prepare", "kernel" + start := make(chan struct{}) + type result struct { + reply recoveryStatusReply + err error + } + sealed, prepared := make(chan result, 1), make(chan result, 1) + go func() { + <-start + reply, err := sealWire(addr, request) + sealed <- result{reply, err} + }() + go func() { + <-start + reply, err := sealWire(addr, prepare) + prepared <- result{reply, err} + }() + close(start) + a, b := <-sealed, <-prepared + if a.err != nil || b.err != nil { + t.Fatalf("prepare race transport: %v / %v", a.err, b.err) + } + requireNotStarted(t, a.reply, request) + current, _ := s.authority.State("v") + if current.Commands[request.CommandID].Status != "not_started" || current.Commands[request.CommandID].Dispatched { + t.Fatal("a late prepare replaced the durable seal") + } + } +} + func TestNodeSealNotStartedRejectsEveryDispatchedState(t *testing.T) { _, addr, s, state := activeHealthGate(t) - for _, completed := range []bool{false, true} { - id := fmt.Sprintf("already-dispatched-%t", completed) + for _, status := range []CommandStatus{CommandPending, CommandCompleted, CommandUnknown, CommandTerminated} { + id := fmt.Sprintf("already-dispatched-%s", status) handle, err := s.authority.BeginCommand("v", state.Fence, id, CommandForward) if err != nil { t.Fatal(err) @@ -169,10 +206,23 @@ func TestNodeSealNotStartedRejectsEveryDispatchedState(t *testing.T) { if err = s.authority.DispatchCommand(handle, func(ExecutorIdentity) error { return nil }); err != nil { t.Fatal(err) } - if completed { + if status == CommandCompleted { if err = s.authority.CompleteCommand(handle); err != nil { t.Fatal(err) } + } else if status != CommandPending { + // Represent historical statuses directly so the dispatch guard is + // tested independently of a separate phase/generation rejection. + s.authority.mu.Lock() + stored := cloneState(s.authority.journal.Slots["v"]) + command := stored.Commands[id] + command.Status = status + stored.Commands[id] = command + _, err = s.authority.commit("v", stored) + s.authority.mu.Unlock() + if err != nil { + t.Fatal(err) + } } reply := readSeal(t, addr, gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: id}) if reply.OK || reply.Command != nil { @@ -204,6 +254,106 @@ func TestNodeSealNotStartedRejectsNonManagedOrRelabeledAuthority(t *testing.T) { } } +func TestNodeSealNotStartedRejectsUnknownManagementAndOldBoot(t *testing.T) { + _, addr, s, state := activeHealthGate(t) + if _, err := s.authority.BeginCommand("v", state.Fence, "unknown-management-before-seal", CommandMutation); err != nil { + t.Fatal(err) + } + request := gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: "not-authorized-during-mutation"} + if reply := readSeal(t, addr, request); reply.OK || reply.Command != nil { + t.Fatalf("unknown management allowed a not_started proof: %+v", reply) + } + f := publishingFixture(t) + f.publish(t) + f.l.boot = "previous-node-boot" + state, _ = f.a.State("v") + if _, err := f.l.LaunchManifest(context.Background(), launchIntentForTest(state, "old-boot-seal-launch"), allowNodeCommandForTest); err != nil { + t.Fatal(err) + } + addr, s = publishingGate(t, f) + state, _ = s.authority.State("v") + request = gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: "old-boot-no-proof"} + if reply := readSeal(t, addr, request); reply.OK || reply.Command != nil { + t.Fatalf("old boot authorized a new tombstone: %+v", reply) + } +} + +func TestNodeSealNotStartedDoesNotRelabelAbsentHistoryWithNewGeneration(t *testing.T) { + f := publishingFixture(t) + f.publish(t) + state, _ := f.a.State("v") + if _, err := f.l.LaunchManifest(context.Background(), launchIntentForTest(state, "historical-seal-launch"), allowNodeCommandForTest); err != nil { + t.Fatal(err) + } + state, _ = f.a.State("v") + original := state.Fence + if _, err := f.a.Revoke("v", original); err != nil { + t.Fatal(err) + } + next, err := f.a.Advance(context.Background(), "v", original) + if err != nil { + t.Fatal(err) + } + // Trusted test adapter binds a subsequent executor but does not publish a + // new manifest. Neither its Fence nor the old manifest may reauthorize IDs. + handle, err := f.a.BeginLaunch("v", next.Fence, "trusted-subsequent-launch") + if err != nil { + t.Fatal(err) + } + if err = f.a.DispatchCommand(handle, func(ExecutorIdentity) error { return nil }); err != nil { + t.Fatal(err) + } + executor := testExecutor("d") + executor.NodeBootID = f.l.boot + state, err = f.a.BindExecutor(handle, executor) + if err != nil { + t.Fatal(err) + } + addr, s := publishingGate(t, f) + for _, fence := range []Fence{original, state.Fence} { + reply := readSeal(t, addr, gate2Request{Op: "seal-not-started", VMID: "v", Fence: fence, CommandID: "historically-absent-job"}) + if reply.OK || reply.Command != nil { + t.Fatalf("historical absence was relabeled with new authority: %+v", reply) + } + } + current, _ := s.authority.State("v") + if _, exists := current.Commands["historically-absent-job"]; exists { + t.Fatal("a replacement manifest created a historical tombstone") + } +} + +func TestNodeSealNotStartedJournalRejectsContradictoryDispatchedMarker(t *testing.T) { + _, addr, s, state := activeHealthGate(t) + request := gate2Request{Op: "seal-not-started", VMID: "v", Fence: state.Fence, CommandID: "contradictory-seal"} + requireNotStarted(t, readSeal(t, addr, request), request) + if err := s.authority.Close(); err != nil { + t.Fatal(err) + } + journal := filepath.Join(s.cfg.AuthorityDir, "authority.json") + data, err := os.ReadFile(journal) + if err != nil { + t.Fatal(err) + } + var stored authorityJournal + if err = json.Unmarshal(data, &stored); err != nil { + t.Fatal(err) + } + command := stored.Slots["v"].Commands[request.CommandID] + command.Dispatched = true + stored.Slots["v"].Commands[request.CommandID] = command + data, _ = json.Marshal(stored) + if err = os.WriteFile(journal, data, 0o600); err != nil { + t.Fatal(err) + } + opened, err := OpenAuthority(s.cfg.AuthorityDir, terminatedExecutor) + if opened != nil { + opened.Close() + } + if !errors.Is(err, ErrInvalidAuthority) { + t.Fatalf("contradictory not_started journal accepted: %v", err) + } +} + func TestNodeSealNotStartedPersistenceFailureNeverReturnsProof(t *testing.T) { _, addr, s, state := activeHealthGate(t) journal := filepath.Join(s.cfg.AuthorityDir, "authority.json") From ec077369ec5f9543d68f61fb84a1e5c4dfadaff9 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:32:45 +0800 Subject: [PATCH 409/473] test: stabilize egress fixture and cover HA credential migration guard --- test/unit/egress-stop-proof.test.mjs | 8 +++++--- test/unit/oauth-credentials.test.mjs | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/test/unit/egress-stop-proof.test.mjs b/test/unit/egress-stop-proof.test.mjs index e9410edb1..c6d5d2f16 100644 --- a/test/unit/egress-stop-proof.test.mjs +++ b/test/unit/egress-stop-proof.test.mjs @@ -16,10 +16,12 @@ function fixture(t) { const script = path.join(root, 'kin-egress-fixture.mjs') fs.writeFileSync( script, - "process.on('SIGTERM', () => setTimeout(() => process.exit(0), 80)); setInterval(() => {}, 1000)\n", + `#!${process.execPath}\nprocess.on('SIGTERM', () => setTimeout(() => process.exit(0), 80)); setInterval(() => {}, 1000)\n`, + { mode: 0o700 }, ) - const bin = path.join(root, 'kin-egress-fixture.sh') - fs.writeFileSync(bin, `#!/bin/sh\nexec '${process.execPath}' '${script}' "$@"\n`, { mode: 0o700 }) + // A shell wrapper can be receipted as sh just before exec changes its image. + // A direct shebang has the final Node image when the config argv is visible. + const bin = script const pids = [] t.after(() => { for (const pid of pids) { diff --git a/test/unit/oauth-credentials.test.mjs b/test/unit/oauth-credentials.test.mjs index 69c5a0d52..9a28437af 100644 --- a/test/unit/oauth-credentials.test.mjs +++ b/test/unit/oauth-credentials.test.mjs @@ -426,12 +426,24 @@ for (const subscriptionType of [undefined, 'pro']) { ) const source = fs.readFileSync(new URL('../../src/server.mjs', import.meta.url), 'utf8') const start = source.indexOf( - 'for (const vm of listVms(cfg.paths.project)) {\n if (isCodexVm(vm)) continue\n try {\n ensureSlotSubscriptionType', + 'for (const vm of haMode ? [] : listVms(cfg.paths.project)) {\n if (isCodexVm(vm)) continue\n try {\n ensureSlotSubscriptionType', ) assert.ok(start >= 0, 'the startup credential migration must be exercised') const end = source.indexOf('\ncliNodeGuard =', start) assert.ok(end > start) + const original = fs.readFileSync(file, 'utf8') runInNewContext(source.slice(start, end), { + haMode: true, + listVms, + isCodexVm, + ensureSlotSubscriptionType, + path, + cfg: { paths: { project: root } }, + console, + }) + assert.equal(fs.readFileSync(file, 'utf8'), original, 'HA startup must not migrate Pod credentials') + runInNewContext(source.slice(start, end), { + haMode: false, listVms, isCodexVm, ensureSlotSubscriptionType, @@ -447,6 +459,7 @@ for (const subscriptionType of [undefined, 'pro']) { const written = fs.readFileSync(file, 'utf8') const mtime = fs.statSync(file).mtimeMs runInNewContext(source.slice(start, end), { + haMode: false, listVms, isCodexVm, ensureSlotSubscriptionType, From 0149edc9be13b7976f9ef0c7c726d693fb0e204e Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:34:55 +0800 Subject: [PATCH 410/473] test: reproduce HA Responses status regression --- test/e2e/ha-bootstrap.e2e.test.mjs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/ha-bootstrap.e2e.test.mjs b/test/e2e/ha-bootstrap.e2e.test.mjs index 2ab6e7667..22410890b 100644 --- a/test/e2e/ha-bootstrap.e2e.test.mjs +++ b/test/e2e/ha-bootstrap.e2e.test.mjs @@ -63,6 +63,15 @@ test('two HA processes start with PG-only metadata and never enter legacy host b }) assert.equal(blocked.status, 503) assert.equal((await blocked.json()).error.code, 'ha_operation_unsupported') + for (const endpoint of ['/v1/responses', '/responses']) { + const response = await fetch(`${gateways[1].baseUrl}${endpoint}`, { + method: 'POST', + headers: { authorization: `Bearer ${gateways[1].apiKey}`, 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'claude-sonnet-4-5', input: 'synthetic rejected request' }), + }) + assert.equal(response.status, 400) + assert.equal((await response.json()).error.code, 'protocol_not_allowed') + } assert.equal(fs.existsSync(barrier) ? fs.readFileSync(barrier, 'utf8') : '', '') for (const gateway of gateways) { assert.equal(fs.existsSync(gateway.runtimeCallsFile) ? fs.readFileSync(gateway.runtimeCallsFile, 'utf8') : '', '') From 1f62f85f94ada7881a7d3d110616d8aac7cfed3c Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:36:00 +0800 Subject: [PATCH 411/473] fix: preserve HA Responses protocol rejection context --- src/server.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server.mjs b/src/server.mjs index c8905622a..4783787d3 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1292,7 +1292,8 @@ async function withRequestActivity(req, res, callback, kind = 'http') { '/v1/completions', '/completions', ].includes(pathname) - const trackHaRequest = haMode && (recoverable || kind === 'websocket_request') + const protocolContext = req.method === 'POST' && ['/v1/responses', '/responses'].includes(pathname) + const trackHaRequest = haMode && (recoverable || protocolContext || kind === 'websocket_request') const gate = await activityCoordinator.admit({ kind, requestId }) const state = { gate, executions: new Map(), unknown: false } requestActivities.set(req, state) From ed717a07bc127911d7c9809e05f1c9bec9d937a2 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:36:00 +0800 Subject: [PATCH 412/473] test: reproduce HA routing restarting unsupported monitors --- test/unit/ha-routing-monitors.test.mjs | 80 ++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/unit/ha-routing-monitors.test.mjs diff --git a/test/unit/ha-routing-monitors.test.mjs b/test/unit/ha-routing-monitors.test.mjs new file mode 100644 index 000000000..11036c521 --- /dev/null +++ b/test/unit/ha-routing-monitors.test.mjs @@ -0,0 +1,80 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import pg from 'pg' +import { getDb, quoteIdentifier, registerActivityCoordinator } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { createRoutingRuntime } from '../../src/lib/admin/routing-runtime.mjs' +import { createUsageProbeMonitor } from '../../src/lib/oauth/usage-probe-monitor.mjs' +import { createHealthProbeMonitor } from '../../src/lib/admin/health-probe.mjs' +import { HaVmCatalog, bindHaVmCatalog } from '../../src/lib/vm/ha-vm-catalog.mjs' + +test('HA routing updates keep unsupported monitors stopped and reject on-demand probes before claiming work', async (t) => { + const db = getDb() + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-648-ha-monitors-')) + const coordinator = new GlobalMaintenanceCoordinator(db) + await coordinator.start() + registerActivityCoordinator(db, coordinator) + const previous = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + const file = path.join(root, 'routing.json') + fs.writeFileSync(file, JSON.stringify({ health_probe: { enabled: true }, usage_probe: { enabled: true } })) + let config + let probes = 0 + let pool + const usage = createUsageProbeMonitor({ config: { enabled: true }, listTargets: () => [] }) + const health = createHealthProbeMonitor({ config: { enabled: true }, listTargets: () => [] }) + t.after(async () => { + await Promise.all([usage.stop(), health.stop()]) + if (previous == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = previous + bindHaVmCatalog(null) + await coordinator.stop() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + fs.rmSync(root, { recursive: true, force: true }) + } + }) + const runtime = createRoutingRuntime({ + db, + cfg: { paths: { project: root } }, + routingConfigPath: file, + getRoutingConfig: () => config, + setRoutingConfig: (next) => { config = next }, + accountQuota: { reloadConfig() {}, attachRuntimeRepo() {} }, + getPoolScheduler: () => pool, + setPoolScheduler: (next) => { pool = next }, + healthMonitor: health, + usageProbeMonitor: usage, + probeUsageOne: () => { probes++ }, + }) + const bootstrap = await coordinator.admitBootstrap() + await bootstrap.run(async () => { + const catalog = new HaVmCatalog(db) + bindHaVmCatalog(catalog) + await catalog.refresh() + config = await runtime.initRoutingConfig() + await runtime.initPoolRuntime() + }) + await coordinator.acknowledgeEpoch(bootstrap.epoch) + await bootstrap.complete({ ended: true }) + const intervals = [] + const original = globalThis.setInterval + t.mock.method(globalThis, 'setInterval', (callback, ms, ...args) => { + intervals.push(ms) + return original(callback, ms, ...args) + }) + await coordinator.runActivity({ kind: 'routing_update' }, async () => { + await runtime.persistRoutingPatch({ pool: { queue_max: 23 } }) + return { ended: true } + }) + assert.deepEqual(intervals, [], 'a routing update must not install unsupported HA monitor timers') + pool.rateLimitService.onUsageProbe({ vmId: 'synthetic-vm' }) + assert.equal(probes, 0) +}) From f44b646a56e28a88f99c0abedb6ba1c33ba57724 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:36:29 +0800 Subject: [PATCH 413/473] fix: keep unsupported HA probes disabled after routing updates --- src/lib/admin/routing-runtime.mjs | 6 +++--- src/server.mjs | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/lib/admin/routing-runtime.mjs b/src/lib/admin/routing-runtime.mjs index e99f810a8..f9a923c54 100644 --- a/src/lib/admin/routing-runtime.mjs +++ b/src/lib/admin/routing-runtime.mjs @@ -166,8 +166,8 @@ export function createRoutingRuntime(ctx) { configureUnitCircuit(config) const pending = [] for (const [name, callback] of [ - ['health_monitor', () => getHealth()?.setConfig(config.health_probe)], - ['usage_monitor', () => getUsage()?.setConfig(config.usage_probe)], + ['health_monitor', () => getHealth()?.setConfig(config.health_probe, { restart: !isKubernetesHaMode() })], + ['usage_monitor', () => getUsage()?.setConfig(config.usage_probe, { restart: !isKubernetesHaMode() })], ['notify_monitor', () => getNotify()?.setConfig(config.notify)], [ 'logging', @@ -864,7 +864,7 @@ export function createRoutingRuntime(ctx) { accountQuota: ctx.accountQuota, config: routingConfig.rate_limit || {}, onUsageProbe: ({ vmId }) => { - if (!vmId || typeof ctx.probeUsageOne !== 'function') return + if (isKubernetesHaMode() || !vmId || typeof ctx.probeUsageOne !== 'function') return Promise.resolve(ctx.probeUsageOne(vmId)).catch(() => {}) }, }) diff --git a/src/server.mjs b/src/server.mjs index 4783787d3..7698e1f5d 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -345,7 +345,10 @@ const routingRt = createRoutingRuntime({ }, // Lazy: the monitor is built after the runtime. An unknown 429 asks for one // /usage hop through the same single-flight, bounded queue as the timer. - probeUsageOne: (vmId) => usageProbeMonitor?.probeNow?.(vmId, 'rate_limited_unknown'), + probeUsageOne: (vmId) => { + if (haMode) throw haUnsupported() + return usageProbeMonitor?.probeNow?.(vmId, 'rate_limited_unknown') + }, }) const { @@ -692,10 +695,12 @@ backupService.onRestored(async (db) => { maxMb: routingConfig.logging?.max_mb, mutedErrorClasses: routingConfig.logging?.muted_error_classes, }) - healthMonitor?.setConfig(routingConfig.health_probe) - credentialRefreshMonitor?.setConfig(normalizeCredentialRefreshConfig(routingConfig.credential_refresh)) - kernelWatchdog?.setConfig(normalizeKernelWatchdogConfig(routingConfig.kernel_watchdog)) - usageProbeMonitor?.setConfig(routingConfig.usage_probe) + if (!haMode) { + healthMonitor?.setConfig(routingConfig.health_probe) + credentialRefreshMonitor?.setConfig(normalizeCredentialRefreshConfig(routingConfig.credential_refresh)) + kernelWatchdog?.setConfig(normalizeKernelWatchdogConfig(routingConfig.kernel_watchdog)) + usageProbeMonitor?.setConfig(routingConfig.usage_probe) + } notifyMonitor?.setConfig(routingConfig.notify) await initPoolRuntime() From cfe50d6e9abf31230ada8e9d403bb03879260dca Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:38:24 +0800 Subject: [PATCH 414/473] test: bind KSG2 fixture to the current node boot --- test/fixtures/slotgate-client-node.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/fixtures/slotgate-client-node.go b/test/fixtures/slotgate-client-node.go index 25ebd10f6..4faebd852 100644 --- a/test/fixtures/slotgate-client-node.go +++ b/test/fixtures/slotgate-client-node.go @@ -36,8 +36,10 @@ func main() { launch, err := authority.BeginLaunch("vm-synthetic", state.Fence, "synthetic-bootstrap") check(err) check(authority.StartLaunch(launch)) + boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + check(err) state, err = authority.BindExecutor(launch, slotgate.ExecutorIdentity{ - Kind: "container", NodeBootID: "synthetic-boot", ContainerID: strings.Repeat("a", 64), + Kind: "container", NodeBootID: strings.TrimSpace(string(boot)), ContainerID: strings.Repeat("a", 64), DockerDaemonID: "synthetic-engine", StartedAt: "2026-10-21T00:00:00.000000001Z", }) check(err) From cf25009d7bac99a13e294442defe44e98e0c6dae Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:38:44 +0800 Subject: [PATCH 415/473] test: execute normal HA HTTP requests through PG node authority --- test/e2e/ha-node-flow.e2e.test.mjs | 328 ++++++++++++++++++++++ test/fixtures/ha-flow-no-host-loader.mjs | 24 ++ test/fixtures/node-management-fixture.mjs | 171 +++++++++++ test/unit/node-management-client.test.mjs | 160 +---------- 4 files changed, 524 insertions(+), 159 deletions(-) create mode 100644 test/e2e/ha-node-flow.e2e.test.mjs create mode 100644 test/fixtures/ha-flow-no-host-loader.mjs create mode 100644 test/fixtures/node-management-fixture.mjs diff --git a/test/e2e/ha-node-flow.e2e.test.mjs b/test/e2e/ha-node-flow.e2e.test.mjs new file mode 100644 index 000000000..ed527048c --- /dev/null +++ b/test/e2e/ha-node-flow.e2e.test.mjs @@ -0,0 +1,328 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import http from 'node:http' +import os from 'node:os' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { nodeManagementFixture } from '../fixtures/node-management-fixture.mjs' +import { startGateway, api } from '../harness.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' + +const model = 'claude-haiku-4-5-20251001' +const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +test('HA flow barrier observes legacy I/O even with the gateway mock loader installed', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ha-flow-canary-')) + const barrier = path.join(root, 'effects.log') + try { + const cases = [ + [new URL('../../src/lib/oauth/oauth-credentials.mjs', import.meta.url).href, 'readSlotCredentialIdentity'], + [new URL('../../src/lib/cluster/docker-remote.mjs', import.meta.url).href, 'dockerInfo'], + [new URL('../../src/lib/cluster/ssh-link.mjs', import.meta.url).href, 'connectSsh'], + [new URL('../../src/lib/cluster/remote-fs.mjs', import.meta.url).href, 'openSftp'], + ] + const script = `for (const [url,name] of ${JSON.stringify(cases)}) { const module=await import(url);try { await module[name]() } catch {} }` + await promisify(execFile)( + process.execPath, + [ + '--import', + new URL('../fixtures/mock-gateway-runtime-loader.mjs', import.meta.url).pathname, + '--input-type=module', + '-e', + script, + ], + { + env: { + ...process.env, + KIN_CRS_MOCK: '1', + VM2API_TEST_HOST_BARRIER_FILE: barrier, + NODE_OPTIONS: `--import=${new URL('../fixtures/ha-flow-no-host-loader.mjs', import.meta.url).href}`, + }, + timeout: 10000, + }, + ) + assert.deepEqual( + fs.readFileSync(barrier, 'utf8').trim().split('\n'), + cases.map(([, name]) => name), + ) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) + +test('two normal HA HTTP processes execute through PG grants and node completion before releasing shared admissions', { + timeout: 120000, +}, async (t) => { + const fx = await nodeManagementFixture(t) + const gateways = [] + const manifestFile = path.join(os.tmpdir(), `vm2api-648-ha-flow-${process.pid}-resources.json`) + const manifest = { + root: fx.root, + nodeRoot: fx.node.root, + ...fx.resources, + projectDirs: [], + gatewayPids: [], + state: 'active', + } + const saveManifest = () => fs.writeFileSync(manifestFile, JSON.stringify(manifest, null, 2) + '\n', { mode: 0o600 }) + saveManifest() + t.diagnostic(`resource manifest: ${manifestFile}`) + let kernel + let release + let deadline + let inFlight = [] + try { + const credential = { ...JSON.parse(fx.payload).credential, expires_at: Date.now() + 3600000 } + const payload = Buffer.from(JSON.stringify({ credential })) + const published = await fx.run('node_publish_initial', (operation) => + fx.service.publishInitial(fx.input(operation, { payload })), + ) + const launched = await fx.run('node_launch', (operation) => + fx.service.launch( + fx.input(operation, { + payload, + commandId: 'http-flow-launch', + manifestDigest: published.manifest_digest, + }), + ), + ) + assert.equal(launched.fence.executor.kind, 'container') + const managementCalls = [...fx.calls] + const nodeRoot = path.join(fx.node.root, 'runtime', 'vms', fx.vmId, 'incarnations', launched.fence.incarnation) + const arrivals = [] + let active = 0 + let peak = 0 + let healthCalls = 0 + let released = false + let signalEntered + const bothEntered = new Promise((resolve) => { + signalEntered = resolve + }) + const overlap = new Promise((resolve) => { + release = () => { + released = true + resolve() + } + }) + kernel = http.createServer(async (req, res) => { + if (req.method === 'GET' && req.url === '/internal/health') { + healthCalls++ + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ ok: true, healthy: true, ready_slots: 8, version: 'synthetic-kernel' })) + return + } + if (req.method !== 'POST' || req.url !== '/internal/v1/messages') { + res.writeHead(404).end() + return + } + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + const envelope = JSON.parse(Buffer.concat(chunks)) + arrivals.push({ + id: envelope.request_id, + nodeAuthenticated: + !!req.headers['x-kin-internal-token'] && req.headers['x-kin-internal-token'] !== 'synthetic-client-forgery', + }) + active++ + peak = Math.max(peak, active) + if (active === 2 && !released) signalEntered(true) + await overlap + const id = 'msg_' + envelope.request_id.replaceAll('-', '') + const events = [ + { + type: 'message_start', + message: { + id, + type: 'message', + role: 'assistant', + model, + content: [], + stop_reason: null, + usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'synthetic-node-response' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 7 } }, + { type: 'message_stop' }, + ] + res.writeHead(200, { 'content-type': 'text/event-stream', 'x-terminal-state': 'verified' }) + for (const event of events) res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) + res.end() + active-- + }) + await new Promise((resolve, reject) => { + kernel.once('error', reject) + kernel.listen(path.join(nodeRoot, 'run', 'kernel.sock'), resolve) + }) + await new VmsRepo(fx.db).upsertFromVmJson({ + id: fx.vmId, + name: 'synthetic-ha-http', + node_id: fx.nodeId, + status: 'running', + schedulable: true, + timezone: 'UTC', + locale: 'en_US.UTF-8', + fingerprint: { device_id: 'ha-http-device', session_id: 'ha-http-native-session', machineID: 'ha-http-machine' }, + policy: { maxConcurrency: 2, maxConcurrencyOverride: true, weight: 1 }, + claude: { + account_uuid: credential.account_uuid, + org_uuid: credential.org_uuid, + email: credential.email, + has_access: true, + has_refresh: true, + expires_at: credential.expires_at, + source: 'synthetic-node-publish', + }, + }) + const trust = path.join(fx.root, 'trusted-nodes.json') + fs.writeFileSync( + trust, + JSON.stringify({ + nodes: { [fx.nodeId]: { ...fx.node.link, policyDigest: fx.policyDigest, presetId: 'claude-docker-v1' } }, + }), + { mode: 0o600 }, + ) + const barrier = path.join(fx.root, 'legacy-effects.log') + for (let index = 0; index < 2; index++) { + const project = path.join(fx.root, `controller-${index}`) + fs.mkdirSync(path.join(project, 'vms'), { recursive: true }) + fs.writeFileSync(path.join(project, 'vms', 'active.json'), '{}') + gateways.push( + await startGateway({ + project, + readyMs: 15000, + env: { + VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', + KIN_DATABASE_SCHEMA: fx.db.schema, + VM2API_NODE_CONFIG_FILE: trust, + VM2API_TEST_HOST_BARRIER_FILE: barrier, + NODE_OPTIONS: `--import=${new URL('../fixtures/ha-flow-no-host-loader.mjs', import.meta.url).href}`, + KIN_BACKUP_DISABLED: '1', + }, + }), + ) + manifest.projectDirs.push(project) + manifest.gatewayPids.push(gateways.at(-1).child.pid) + saveManifest() + } + assert.notEqual(gateways[0].child.pid, gateways[1].child.pid) + const created = await api(gateways[0], 'POST', '/api/panel/api-keys', { + body: { name: 'ha-node-flow', max_concurrency: 2, rpm: 0, quota_requests: 20 }, + }) + assert.equal(created.status, 201, created.text) + const key = created.json.item.key + deadline = setTimeout(() => { + signalEntered(false) + release() + }, 10000) + inFlight = gateways.map((gateway, index) => + api(gateway, 'POST', '/v1/messages', { + headers: { + authorization: `Bearer ${key}`, + 'x-session-id': `ha-http-session-${index}`, + 'x-internal-token': 'synthetic-client-forgery', + }, + body: { + model, + max_tokens: 32, + stream: index === 1, + messages: [{ role: 'user', content: 'synthetic request' }], + }, + }), + ) + const entered = await Promise.race([bothEntered, Promise.all(inFlight).then(() => false)]) + assert.equal(entered, true, 'both controllers must dispatch before either kernel execution finishes') + const activeClaims = ( + await fx.peer.query(`SELECT + (SELECT count(*)::int FROM execution_admissions WHERE released_at IS NULL) AS executions, + (SELECT count(*)::int FROM api_key_admissions WHERE released_at IS NULL) AS keys, + (SELECT count(*)::int FROM session_turns WHERE state='active') AS turns, + (SELECT count(*)::int FROM execution_jobs WHERE ended_at IS NULL AND node_binding IS NOT NULL) AS bound_jobs`) + ).rows[0] + assert.deepEqual(activeClaims, { executions: 2, keys: 2, turns: 2, bound_jobs: 2 }) + release() + const responses = await Promise.all(inFlight) + clearTimeout(deadline) + for (const response of responses) assert.equal(response.status, 200, response.text) + assert.equal(responses[0].json?.type, 'message', responses[0].text) + assert.match(responses[0].text, /synthetic-node-response/) + assert.match(responses[1].headers.get('content-type'), /text\/event-stream/) + assert.match(responses[1].text, /event: message_stop/) + assert.match(responses[1].text, /synthetic-node-response/) + assert.ok(peak >= 2, `actual simultaneous kernel executions=${peak}`) + assert.equal(arrivals.length, 2) + assert.equal(new Set(arrivals.map((row) => row.id)).size, 2) + assert.ok(arrivals.every((row) => row.nodeAuthenticated)) + assert.ok(healthCalls >= 2) + const jobs = ( + await fx.peer.query( + 'SELECT request_id,node_binding,node_completion,ended_at FROM execution_jobs ORDER BY request_id', + ) + ).rows + assert.equal(jobs.length, 2) + for (const job of jobs) { + assert.ok(arrivals.some((row) => row.id === job.request_id)) + assert.equal(job.node_binding.protocol, 'KSG2') + assert.equal(job.node_binding.nodeId, fx.nodeId) + assert.equal(job.node_completion.id, job.request_id) + assert.equal(job.node_completion.status, 'completed') + assert.equal(job.node_completion.dispatched, true) + assert.deepEqual(job.node_completion.fence, job.node_binding.fence) + assert.ok(job.ended_at) + } + let counts + for (let poll = 0; poll < 60; poll++) { + counts = ( + await fx.peer.query(`SELECT + (SELECT count(*)::int FROM execution_admissions WHERE released_at IS NULL) AS executions, + (SELECT count(*)::int FROM api_key_admissions WHERE released_at IS NULL) AS keys, + (SELECT count(*)::int FROM session_turns) AS turns`) + ).rows[0] + if (Object.values(counts).every((count) => count === 0)) break + await pause(25) + } + assert.deepEqual(counts, { executions: 0, keys: 0, turns: 0 }) + assert.equal((await fx.peer.query('SELECT count(*)::int AS n FROM api_key_admissions')).rows[0].n, 2) + assert.equal( + (await fx.peer.query('SELECT count(*)::int AS n FROM api_key_admissions WHERE settled_at IS NOT NULL')).rows[0].n, + 2, + ) + assert.deepEqual(fx.calls, managementCalls, 'controllers must not call legacy Docker or mutate executors') + assert.equal(fs.existsSync(barrier) ? fs.readFileSync(barrier, 'utf8') : '', '') + for (const gateway of gateways) { + assert.equal(fs.existsSync(gateway.runtimeCallsFile) ? fs.readFileSync(gateway.runtimeCallsFile, 'utf8') : '', '') + assert.equal(fs.existsSync(path.join(gateway.project, 'vms', fx.vmId, 'cli-home')), false) + } + t.diagnostic( + `normal HTTP processes=2 kernel peak=${peak} jobs=${jobs.length} PG active executions/keys/turns=0/0/0; legacy I/O=0`, + ) + } finally { + clearTimeout(deadline) + release?.() + for (const gateway of gateways.reverse()) await gateway.stop() + await Promise.allSettled(inFlight) + if (kernel) { + kernel.closeAllConnections() + await new Promise((resolve) => kernel.close(resolve)) + } + await fx.close() + manifest.state = 'cleaned' + manifest.remainingDirs = [fx.root, fx.node.root].filter((file) => fs.existsSync(file)) + manifest.remainingRoles = ( + await fx.db.pool.query('SELECT rolname FROM pg_roles WHERE rolname=$1', [fx.resources.readerRole]) + ).rows + manifest.remainingPrivateSchemas = ( + await fx.db.pool.query('SELECT nspname FROM pg_namespace WHERE nspname=ANY($1::text[])', [ + [fx.resources.authoritySchema, fx.resources.maintenanceSchema], + ]) + ).rows + saveManifest() + assert.deepEqual(manifest.remainingDirs, []) + assert.deepEqual(manifest.remainingRoles, []) + assert.deepEqual(manifest.remainingPrivateSchemas, []) + } +}) diff --git a/test/fixtures/ha-flow-no-host-loader.mjs b/test/fixtures/ha-flow-no-host-loader.mjs new file mode 100644 index 000000000..9a62c5396 --- /dev/null +++ b/test/fixtures/ha-flow-no-host-loader.mjs @@ -0,0 +1,24 @@ +import './ha-no-host-loader.mjs' +import { registerHooks } from 'node:module' + +// Fail closed at legacy I/O entry points while leaving PG, TLS, protocol and +// scheduler implementations intact. The real Go fixture owns its fake Engine. +const denied = new Map([ + ['../../src/lib/oauth/oauth-credentials.mjs', ['readWorkerCredentialFile', 'readSlotCredentialIdentity', 'mirrorWorkerCredentialsToVm', 'writeWorkerCredentialFile', 'ensureOfficialCredentialLink']], + ['../../src/lib/cluster/docker-remote.mjs', ['sshDocker', 'localDocker', 'dockerRequest', 'dockerJson', 'dockerPing', 'dockerInfo', 'listContainers', 'pullImage', 'ensureImage', 'createAndStartContainer', 'containerAction', 'removeContainer', 'renameContainer', 'containerLogs', 'inspectContainerOrNull', 'createContainerRaw', 'execDetached', 'openExecTty', 'resizeExec', 'inspectExec', 'imagePresent', 'inspectNetworkOrNull', 'createNetwork', 'removeNetwork', 'disconnectContainerFromNetwork', 'connectContainerToNetwork', 'buildImage']], + ['../../src/lib/cluster/ssh-link.mjs', ['connectSsh', 'forwardOut', 'forwardOutStreamLocal', 'execCollect']], + ['../../src/lib/cluster/remote-fs.mjs', ['openSftp', 'writeRemoteFile', 'readRemoteFileNoFollow', 'remoteSymlink', 'runRemote']], +].map(([file, names]) => [new URL(file, import.meta.url).href, names])) + +registerHooks({ + resolve(specifier, context, nextResolve) { + const resolved = nextResolve(specifier, context) + const names = denied.get(resolved.url) + if (!names) return resolved + const original = JSON.stringify(resolved.url + '?ha-flow-original') + const source = `import fs from 'node:fs'; export * from ${original}; + function deny(name) { fs.appendFileSync(process.env.VM2API_TEST_HOST_BARRIER_FILE,name+'\\n'); throw new Error('HA attempted legacy request I/O: '+name) } + ${names.map((name) => `export function ${name}() { return deny(${JSON.stringify(name)}) }`).join('\n')}` + return { url: 'data:text/javascript,' + encodeURIComponent(source), shortCircuit: true } + }, +}) diff --git a/test/fixtures/node-management-fixture.mjs b/test/fixtures/node-management-fixture.mjs new file mode 100644 index 000000000..4a363eb4c --- /dev/null +++ b/test/fixtures/node-management-fixture.mjs @@ -0,0 +1,171 @@ +import crypto from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import pg from 'pg' +import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { realAllocationGate } from './real-allocation-gate.mjs' + +export const nodeId = 'command-node' +export const vmId = 'command-vm' +export const allocationId = 'command-allocation' +export const payload = Buffer.from( + JSON.stringify({ + credential: { + type: 'oauth', + access_token: 'synthetic-command-access', + refresh_token: 'synthetic-command-refresh', + account_uuid: 'synthetic-account', + org_uuid: 'synthetic-org', + email: 'synthetic@example.invalid', + }, + }), +) + +export async function nodeManagementFixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const coordinator = new GlobalMaintenanceCoordinator(db) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'nmc-')) + let node + let readerRole + let engine + let closed = false + const service = new NodeManagementGrants(db, { resolveNodeLink: async () => node.link }) + const close = async () => { + if (closed) return + closed = true + await node?.close() + if (engine) { + engine.closeAllConnections() + await new Promise((resolve) => engine.close(resolve)) + } + await coordinator.stop() + await peer.close() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + if (readerRole) + await cleanup.query(`DROP OWNED BY ${quoteIdentifier(readerRole)}; DROP ROLE ${quoteIdentifier(readerRole)}`) + await cleanup.query( + `DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE; DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`, + ) + } finally { + await cleanup.end() + fs.rmSync(root, { recursive: true, force: true }) + } + } + t.after(close) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + await service.initialize() + await service.initializeCommands() + readerRole = 'command_wire_' + crypto.randomBytes(8).toString('hex') + const password = crypto.randomBytes(24).toString('hex') + await db.pool.query( + `CREATE ROLE ${quoteIdentifier(readerRole)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, + ) + await db.pool.query(service.readerRoleSql({ role: readerRole, nodeId })) + await db.pool.query(service.commandReaderRoleSql({ role: readerRole })) + const url = new URL(db.connectionString) + url.username = readerRole + url.password = password + const policy = { + image_id: 'sha256:' + 'a'.repeat(64), + network_id: 'none', + uid: process.getuid() || 1000, + gid: process.getgid() || 1000, + } + const policyDigest = crypto + .createHash('sha256') + .update( + [ + 'slotgate-policy-v1', + 'claude-docker-v1', + policy.image_id, + policy.network_id, + String(policy.uid), + String(policy.gid), + ].join('\0'), + ) + .digest('hex') + const calls = [] + let spec + const containerId = 'b'.repeat(64) + engine = http.createServer(async (req, res) => { + calls.push(`${req.method} ${req.url}`) + const parsed = new URL(req.url, 'http://synthetic-engine') + const send = (body, status = 200) => { + res.writeHead(status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(body)) + } + if (parsed.pathname === '/info') return send({ ID: 'synthetic-daemon' }) + if (parsed.pathname.startsWith('/images/')) + return send({ Id: policy.image_id, Config: { Labels: { 'kin.slot.full': '1' } } }) + if (parsed.pathname === '/containers/create') { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + spec = JSON.parse(Buffer.concat(chunks).toString()) + return send({ Id: containerId }, 201) + } + if (parsed.pathname === `/containers/${containerId}/start`) return res.writeHead(204).end() + if (parsed.pathname === `/containers/${containerId}/json`) + return send({ + Id: containerId, + Image: policy.image_id, + Config: { Labels: spec.Labels }, + State: { Running: true, Restarting: false, StartedAt: '2026-10-22T00:00:00.000000001Z' }, + }) + res.writeHead(404).end() + }) + const socket = path.join(root, 'engine.sock') + await new Promise((resolve, reject) => { + engine.once('error', reject) + engine.listen(socket, resolve) + }) + node = await realAllocationGate( + t, + { connection_string: url.toString(), authority_schema: service.schema, node_id: nodeId }, + { launch: policy, dockerSocket: socket }, + ) + const run = (kind, callback) => new SharedVmOperations(db).run(vmId, callback, { kind }) + const allocated = await run('node_allocate', (operation) => + service.allocate({ operation, nodeId, vmId, allocationId }), + ) + const input = (operation, patch = {}) => ({ + operation, + nodeId, + vmId, + allocationId, + commandId: 'wire-publish', + capturedFence: allocated.fence, + presetId: 'claude-docker-v1', + policyDigest, + payload, + ...patch, + }) + return { + db, + peer, + coordinator, + service, + run, + input, + node, + calls, + root, + close, + allocated, + policy, + policyDigest, + nodeId, + vmId, + payload, + resources: { readerRole, authoritySchema: service.schema, maintenanceSchema: coordinator.schema }, + q: quoteIdentifier(service.schema), + } +} diff --git a/test/unit/node-management-client.test.mjs b/test/unit/node-management-client.test.mjs index 93a494ad1..102fa178b 100644 --- a/test/unit/node-management-client.test.mjs +++ b/test/unit/node-management-client.test.mjs @@ -2,171 +2,13 @@ import test from 'node:test' import assert from 'node:assert/strict' import crypto from 'node:crypto' import fs from 'node:fs' -import os from 'node:os' import path from 'node:path' -import http from 'node:http' import tls from 'node:tls' -import pg from 'pg' import * as management from '../../src/lib/transport/slotgate-management-client.mjs' import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' -import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' -import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' -import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' -const nodeId = 'command-node' -const vmId = 'command-vm' -const allocationId = 'command-allocation' -const payload = Buffer.from( - JSON.stringify({ - credential: { - type: 'oauth', - access_token: 'synthetic-command-access', - refresh_token: 'synthetic-command-refresh', - account_uuid: 'synthetic-account', - org_uuid: 'synthetic-org', - email: 'synthetic@example.invalid', - }, - }), -) - -async function fixture(t) { - const db = getDb() - const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) - const coordinator = new GlobalMaintenanceCoordinator(db) - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'nmc-')) - let node - let readerRole - let engine - let closed = false - const service = new NodeManagementGrants(db, { resolveNodeLink: async () => node.link }) - const close = async () => { - if (closed) return - closed = true - await node?.close() - if (engine) { - engine.closeAllConnections() - await new Promise((resolve) => engine.close(resolve)) - } - await coordinator.stop() - await peer.close() - const cleanup = new pg.Client({ connectionString: db.connectionString }) - await cleanup.connect() - try { - if (readerRole) - await cleanup.query(`DROP OWNED BY ${quoteIdentifier(readerRole)}; DROP ROLE ${quoteIdentifier(readerRole)}`) - await cleanup.query( - `DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE; DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`, - ) - } finally { - await cleanup.end() - fs.rmSync(root, { recursive: true, force: true }) - } - } - t.after(close) - await coordinator.start() - await coordinator.acknowledgeEpoch(1) - await service.initialize() - await service.initializeCommands() - readerRole = 'command_wire_' + crypto.randomBytes(8).toString('hex') - const password = crypto.randomBytes(24).toString('hex') - await db.pool.query( - `CREATE ROLE ${quoteIdentifier(readerRole)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, - ) - await db.pool.query(service.readerRoleSql({ role: readerRole, nodeId })) - await db.pool.query(service.commandReaderRoleSql({ role: readerRole })) - const url = new URL(db.connectionString) - url.username = readerRole - url.password = password - const policy = { - image_id: 'sha256:' + 'a'.repeat(64), - network_id: 'none', - uid: process.getuid() || 1000, - gid: process.getgid() || 1000, - } - const policyDigest = crypto - .createHash('sha256') - .update( - [ - 'slotgate-policy-v1', - 'claude-docker-v1', - policy.image_id, - policy.network_id, - String(policy.uid), - String(policy.gid), - ].join('\0'), - ) - .digest('hex') - const calls = [] - let spec - const containerId = 'b'.repeat(64) - engine = http.createServer(async (req, res) => { - calls.push(`${req.method} ${req.url}`) - const parsed = new URL(req.url, 'http://synthetic-engine') - const send = (body, status = 200) => { - res.writeHead(status, { 'content-type': 'application/json' }) - res.end(JSON.stringify(body)) - } - if (parsed.pathname === '/info') return send({ ID: 'synthetic-daemon' }) - if (parsed.pathname.startsWith('/images/')) - return send({ Id: policy.image_id, Config: { Labels: { 'kin.slot.full': '1' } } }) - if (parsed.pathname === '/containers/create') { - const chunks = [] - for await (const chunk of req) chunks.push(chunk) - spec = JSON.parse(Buffer.concat(chunks).toString()) - return send({ Id: containerId }, 201) - } - if (parsed.pathname === `/containers/${containerId}/start`) return res.writeHead(204).end() - if (parsed.pathname === `/containers/${containerId}/json`) - return send({ - Id: containerId, - Image: policy.image_id, - Config: { Labels: spec.Labels }, - State: { Running: true, Restarting: false, StartedAt: '2026-10-22T00:00:00.000000001Z' }, - }) - res.writeHead(404).end() - }) - const socket = path.join(root, 'engine.sock') - await new Promise((resolve, reject) => { - engine.once('error', reject) - engine.listen(socket, resolve) - }) - node = await realAllocationGate( - t, - { connection_string: url.toString(), authority_schema: service.schema, node_id: nodeId }, - { launch: policy, dockerSocket: socket }, - ) - const run = (kind, callback) => new SharedVmOperations(db).run(vmId, callback, { kind }) - const allocated = await run('node_allocate', (operation) => - service.allocate({ operation, nodeId, vmId, allocationId }), - ) - const input = (operation, patch = {}) => ({ - operation, - nodeId, - vmId, - allocationId, - commandId: 'wire-publish', - capturedFence: allocated.fence, - presetId: 'claude-docker-v1', - policyDigest, - payload, - ...patch, - }) - return { - db, - peer, - coordinator, - service, - run, - input, - node, - calls, - root, - close, - allocated, - q: quoteIdentifier(service.schema), - } -} +import { nodeManagementFixture as fixture, vmId, payload } from '../fixtures/node-management-fixture.mjs' function nodeCase(name, callback) { test(name, async (t) => { From b4fff2fc25a299d49705ff894189a820fcae7508 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:40:51 +0800 Subject: [PATCH 416/473] feat: add explicit multi-node HA Kubernetes deployment overlay --- deploy/k8s-ha/deployment.yaml | 59 +++++++++++++++++++ deploy/k8s-ha/hpa.example.yaml | 20 +++++++ deploy/k8s-ha/kustomization.yaml | 23 ++++++++ deploy/k8s-ha/node-trust.example.json | 12 ++++ deploy/k8s-ha/poddisruptionbudget.yaml | 9 +++ docs/deploy/kubernetes-ha.md | 78 ++++++++++++++++++++++++++ docs/deploy/kubernetes.md | 2 + 7 files changed, 203 insertions(+) create mode 100644 deploy/k8s-ha/deployment.yaml create mode 100644 deploy/k8s-ha/hpa.example.yaml create mode 100644 deploy/k8s-ha/kustomization.yaml create mode 100644 deploy/k8s-ha/node-trust.example.json create mode 100644 deploy/k8s-ha/poddisruptionbudget.yaml create mode 100644 docs/deploy/kubernetes-ha.md diff --git a/deploy/k8s-ha/deployment.yaml b/deploy/k8s-ha/deployment.yaml new file mode 100644 index 000000000..0376cfcc6 --- /dev/null +++ b/deploy/k8s-ha/deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vm2api +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: vm2api + topologyKey: kubernetes.io/hostname + containers: + - name: control-plane + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - name: state + mountPath: /opt/vm2api/vms + subPath: vms + - name: state + mountPath: /opt/vm2api/data + subPath: data + - name: state + mountPath: /opt/vm2api/src/config + subPath: config + - name: run + mountPath: /opt/vm2api/data/run + - name: bin + $patch: delete + - name: share + $patch: delete + - name: node-trust + mountPath: /etc/vm2api + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: state + persistentVolumeClaim: null + emptyDir: {} + - name: bin + $patch: delete + - name: share + $patch: delete + - name: node-trust + secret: + secretName: vm2api-node-trust + defaultMode: 0440 + - name: tmp + emptyDir: {} diff --git a/deploy/k8s-ha/hpa.example.yaml b/deploy/k8s-ha/hpa.example.yaml new file mode 100644 index 000000000..558ecefa0 --- /dev/null +++ b/deploy/k8s-ha/hpa.example.yaml @@ -0,0 +1,20 @@ +# Optional. Install metrics-server and choose limits for your available nodes. +# Do not manage spec.replicas with kubectl apply after enabling this HPA. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: vm2api +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: vm2api + minReplicas: 2 + maxReplicas: 6 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 diff --git a/deploy/k8s-ha/kustomization.yaml b/deploy/k8s-ha/kustomization.yaml new file mode 100644 index 000000000..9a1f049cb --- /dev/null +++ b/deploy/k8s-ha/kustomization.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../k8s + - poddisruptionbudget.yaml +patches: + - path: deployment.yaml + - target: + kind: ConfigMap + name: vm2api-config + patch: |- + - op: replace + path: /data/VM2API_CONTROL_PLANE_MODE + value: kubernetes-ha + - op: add + path: /data/VM2API_NODE_CONFIG_FILE + value: /etc/vm2api/nodes.json + - patch: |- + $patch: delete + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: vm2api-state diff --git a/deploy/k8s-ha/node-trust.example.json b/deploy/k8s-ha/node-trust.example.json new file mode 100644 index 000000000..44daea8fc --- /dev/null +++ b/deploy/k8s-ha/node-trust.example.json @@ -0,0 +1,12 @@ +{ + "nodes": { + "node-01": { + "address": "192.0.2.10", + "port": 18443, + "token": "REPLACE_WITH_EXISTING_NODE_RUNTIME_TOKEN", + "certSha256": "REPLACE_WITH_PINNED_NODE_CERTIFICATE_SHA256", + "presetId": "claude-docker-v1", + "policyDigest": "REPLACE_WITH_APPROVED_NODE_POLICY_SHA256" + } + } +} diff --git a/deploy/k8s-ha/poddisruptionbudget.yaml b/deploy/k8s-ha/poddisruptionbudget.yaml new file mode 100644 index 000000000..d803a5d86 --- /dev/null +++ b/deploy/k8s-ha/poddisruptionbudget.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: vm2api +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: vm2api diff --git a/docs/deploy/kubernetes-ha.md b/docs/deploy/kubernetes-ha.md new file mode 100644 index 000000000..b81704edf --- /dev/null +++ b/docs/deploy/kubernetes-ha.md @@ -0,0 +1,78 @@ +# Kubernetes 多副本控制面（Draft） + +`deploy/k8s-ha` 使用 `VM2API_CONTROL_PLANE_MODE=kubernetes-ha`。默认部署两个控制面 Pod，通过 PostgreSQL 共享状态，通过固定证书指纹的 TLS 节点接口执行请求。控制面不选举推理 leader，各副本独立处理请求。 + +该部署清单已通过 Kustomize 渲染和隔离 k3d 集群的服务端试运行。双进程正常 HTTP/SSE 测试使用真实 PostgreSQL、Go 节点协议和合成执行器。多节点滚动更新、异常退出恢复及既有预编译内核的业务请求验收尚未完成。本页不作为生产启用依据。 + +## 支持范围 + +| 对象 | HA 模式行为 | +| --- | --- | +| 普通 Claude Docker 槽位 | 必须使用已批准的 `claude-docker-v1` 节点策略、私有授权和原始启动回执 | +| Anthropic Messages、Claude 的 OpenAI Chat/Completions 转换 | 通过共享请求准入和原节点执行身份处理，支持 HTTP 与 SSE | +| Claude Responses | 保持 `400 protocol_not_allowed` | +| GPT/Codex、外部 API 后端、Search、原生执行器、KVM | 拒绝 HA 调度 | +| 旧 SSH、SFTP、Docker 管理、终端、在线导入、备份恢复 | 拒绝 HA HTTP 操作；不能沿用旧管理连接绕过节点授权 | +| 旧主动健康探测、用量探测、内核重启和目录同步 | HA 下停止；路由配置刷新不会重新启动 | + +面板登录、API key、配额、路由配置、会话顺序、粘滞路由及执行状态存放在 PostgreSQL。各 Pod 的 `vms`、`data`、`src/config` 和运行目录使用独立 `emptyDir`。节点持有预编译执行包、凭据和执行状态日志；这些文件不放在共享卷。 + +未知执行不会按超时释放容量。恢复必须确认原控制面已失去 PostgreSQL 会话锁，并获取原节点、原 command ID、原执行身份对应的持久完成证明。计费缺少可信用量或原始定价依据时继续保留相应预约。 + +## 安装前提 + +1. PostgreSQL 17、相同业务 schema、相同业务加密密钥。所有副本使用相同管理员配置和初始 API key。 +2. 至少两个可调度的 Kubernetes 节点。必需的 Pod 反亲和保证副本分布在不同节点。默认滚动更新为 `maxUnavailable: 0, maxSurge: 1`，因此两个副本更新时需要第三个可调度节点。 +3. 已安装并校验固定镜像 ID、网络 ID 和 UID/GID 的远端 Go 节点服务。节点的原始 Docker/SSH 权限不能交给控制面 Pod。 +4. 通过可信初始化安装独立 allocation、command 私有 authority、节点专用只读 PostgreSQL role，以及授权的 allocation → 首次凭据发布 → launch 回执链。普通应用启动不会自行安装这些授权。相关接口见 [节点授权](../plans/2026-10-22-node-command-authority.md) 和 [首次发布](../plans/2026-10-22-node-initial-publication.md)。目前尚无面向操作者的完整安装命令。 + +从旧安装切换前停止旧控制面并核实在途任务。`kubernetes` 单实例模式与 `kubernetes-ha` 多副本模式使用互斥的 PostgreSQL 模式锁，不能并存。旧版独立进程不持有该锁，必须由操作者停止。 + +## Secret 与清单 + +在目标命名空间创建现有 `vm2api-secrets`，键与 [单实例部署](kubernetes.md) 一致。所有值来自受控文件，不写入命令参数或 Git。 + +另准备权限为 `0600` 的 `nodes.json`。格式见 `deploy/k8s-ha/node-trust.example.json`。每个节点必须包含固定 IP、端口、运输认证 token、证书 SHA-256、preset 和批准策略 SHA-256。占位内容不能使用。 + +```sh +kubectl -n vm2api create secret generic vm2api-node-trust --from-file=nodes.json=/secure/nodes.json +``` + +在仓库 `.local/ha` 建立自己的覆盖目录，设置命名空间、经过验证的镜像 digest 和对外地址： + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: vm2api +resources: + - ../../deploy/k8s-ha +images: + - name: ghcr.io/dofastted/vm2api + newName: registry.example.com/vm2api + digest: sha256:REPLACE_WITH_VERIFIED_IMAGE_DIGEST +patches: + - target: + kind: ConfigMap + name: vm2api-config + patch: |- + - op: add + path: /data/PUBLIC_BASE_URL + value: https://vm2api.example.com +``` + +```sh +kubectl kustomize .local/ha +kubectl apply -k .local/ha +kubectl -n vm2api rollout status deployment/vm2api +kubectl -n vm2api get pods -l app.kubernetes.io/name=vm2api -o wide +``` + +Pod 使用 UID/GID 1000、只读根文件系统，不挂载 Kubernetes API token、宿主路径或 Docker socket。节点信任配置在启动时固定读取；更换该配置后需要滚动更新。 + +## 扩缩容与更新 + +手动扩容使用 `kubectl -n vm2api scale deployment/vm2api --replicas=N`。每个副本都需要不同的可调度节点。缩容及滚动更新先取消就绪状态，再排空请求。默认排空期限为 60 秒，Pod 终止宽限期为 75 秒。长于宽限期的连接需要客户端重连。 + +可选 `deploy/k8s-ha/hpa.example.yaml` 使用 CPU 指标，需要 metrics-server。调整最大副本数以匹配节点数和 PostgreSQL 容量。启用 HPA 后，不应继续通过应用清单覆盖 `spec.replicas`。 + +Pod 中断预算要求自愿驱逐时至少保留一个副本。它不替代节点故障处理，也不延长长连接的终止宽限期。入口层的 WebSocket、SSE 缓冲和超时仍需按实际网关配置验证。 diff --git a/docs/deploy/kubernetes.md b/docs/deploy/kubernetes.md index 69b248125..21812554b 100644 --- a/docs/deploy/kubernetes.md +++ b/docs/deploy/kubernetes.md @@ -2,6 +2,8 @@ 本阶段运行一个活动控制面，使用 PostgreSQL 17、持久卷和远端计算节点。默认 `replicas: 1`、`Recreate` 更新策略。更新期间会有服务中断。此部署不提供水平多活或节点故障后的无损接管；不要增加副本或添加 HPA。 +显式多副本模式使用独立的 [HA 部署清单](kubernetes-ha.md)。其支持范围、节点授权和存储前提与本清单不同。 + `VM2API_CONTROL_PLANE_MODE=kubernetes` 启用 PG 独占控制面锁及 PG 面板会话。第二个实例连接同一数据库和 schema 时，在业务初始化前退出。连接丢失后，控制面停止准入并退出。该锁不向远端执行器提供 fencing，不能证明失联的远端请求已终止。迁移时必须先停止旧控制面，并确认在途任务的状态。 ## 前提 From a6c0f0ea888c81c6164957c2a2448d2280ba4eaf Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:05:39 +0800 Subject: [PATCH 417/473] test: require node private refresh credentials and authoritative health --- .../slotgate/node_private_credential_test.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 worker/internal/slotgate/node_private_credential_test.go diff --git a/worker/internal/slotgate/node_private_credential_test.go b/worker/internal/slotgate/node_private_credential_test.go new file mode 100644 index 000000000..0bfe58866 --- /dev/null +++ b/worker/internal/slotgate/node_private_credential_test.go @@ -0,0 +1,55 @@ +package slotgate + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestNodeRefreshTokenIsPrivateAndNeverProjectedIntoGuest(t *testing.T) { + f := publishingFixture(t) + f.publish(t) + guest, err := os.ReadFile(filepath.Join(f.private(), "claude", "credentials.json")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(guest), "synthetic-refresh") || strings.Contains(string(guest), "refreshToken") { + t.Fatal("refresh token reached the guest credential projection") + } + private, err := os.ReadFile(filepath.Join(f.a.root, "credentials", "v", f.intent.Fence.Incarnation, "credentials.json")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(private), "synthetic-refresh") { + t.Fatal("node canonical credential lost its refresh token") + } + state, _ := f.a.State("v") + data, _ := json.Marshal(state.InitialManifest) + if !strings.Contains(string(data), `"credential_authority":"node-private-v1"`) { + t.Fatal("manifest lacks node-private ownership proof") + } +} + +func TestNodePrivateCredentialHealthRejectsStableGuestReplacement(t *testing.T) { + f, addr, _, state := activeHealthGate(t) + file := filepath.Join(f.private(), "claude", "credentials.json") + if err := os.Chmod(file, 0o600); err != nil { + t.Fatal(err) + } + // Same account, plausible newer generation and a stable file for the whole + // health read still cannot become the canonical node credential. + if err := os.WriteFile(file, []byte(`{"type":"oauth","claudeAiOauth":{"accessToken":"guest-replacement","accountUuid":"synthetic-account","kinGeneration":2}}`), 0o400); err != nil { + t.Fatal(err) + } + unixHTTP(t, filepath.Join(f.private(), "run", "kernel.sock"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"ok":true}`) + })) + if reply := readNodeHealth(t, addr, capturedHealthRequest(state)); reply.OK { + t.Fatal("guest replacement became the authoritative credential generation") + } +} From 2ef24bf8e72bbb439a7b21907b0a8f2a93792d04 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:06:40 +0800 Subject: [PATCH 418/473] fix: keep node refresh tokens outside guest credential projections --- worker/internal/slotgate/authority.go | 1 + worker/internal/slotgate/management_intent.go | 20 +++-- worker/internal/slotgate/node_health.go | 9 +++ .../slotgate/node_private_credential.go | 73 ++++++++++++++++++ .../slotgate/node_private_credential_linux.go | 75 +++++++++++++++++++ .../slotgate/node_private_credential_other.go | 9 +++ worker/internal/slotgate/publish_initial.go | 15 +++- 7 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 worker/internal/slotgate/node_private_credential.go create mode 100644 worker/internal/slotgate/node_private_credential_linux.go create mode 100644 worker/internal/slotgate/node_private_credential_other.go diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index d73e0156a..78e03938c 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -106,6 +106,7 @@ type DockerLaunchTarget struct { type AuthorityState struct { Allocation *AllocationReceipt `json:"allocation,omitempty"` InitialManifest *initialManifest `json:"initial_manifest,omitempty"` + Credential *nodeCredentialState `json:"credential,omitempty"` Fence Fence `json:"fence"` Phase AuthorityPhase `json:"phase"` Commands map[string]CommandRecord `json:"commands"` diff --git a/worker/internal/slotgate/management_intent.go b/worker/internal/slotgate/management_intent.go index b134c03ce..352824497 100644 --- a/worker/internal/slotgate/management_intent.go +++ b/worker/internal/slotgate/management_intent.go @@ -91,11 +91,13 @@ type LaunchReceipt struct { // Private file hashes stay in the journal; RPC receipts expose only the // aggregate manifest and never credentials, internal tokens or their hashes. type initialManifest struct { - Receipt PublicationReceipt `json:"receipt"` - KernelDigest string `json:"kernel_digest"` - TokenDigest string `json:"token_digest"` - CredentialDigest string `json:"credential_digest"` - Identity credentialIdentity `json:"identity"` + Receipt PublicationReceipt `json:"receipt"` + KernelDigest string `json:"kernel_digest"` + TokenDigest string `json:"token_digest"` + CredentialDigest string `json:"credential_digest"` + Identity credentialIdentity `json:"identity"` + CredentialAuthority string `json:"credential_authority,omitempty"` + PrivateCredentialDigest string `json:"private_credential_digest,omitempty"` } type credentialIdentity struct { @@ -109,6 +111,10 @@ func validCredentialIdentity(identity credentialIdentity) bool { } func (m initialManifest) digest() string { - return hashText(strings.Join([]string{"slotgate-manifest-v1", m.Receipt.Intent.digest(), "1", - m.KernelDigest, m.TokenDigest, m.CredentialDigest, m.Identity.AccountUUID, m.Identity.OrgUUID, m.Identity.Email}, "\x00")) + fields := []string{"slotgate-manifest-v1", m.Receipt.Intent.digest(), "1", + m.KernelDigest, m.TokenDigest, m.CredentialDigest, m.Identity.AccountUUID, m.Identity.OrgUUID, m.Identity.Email} + if m.CredentialAuthority != "" { + fields = append(fields, m.CredentialAuthority, m.PrivateCredentialDigest) + } + return hashText(strings.Join(fields, "\x00")) } diff --git a/worker/internal/slotgate/node_health.go b/worker/internal/slotgate/node_health.go index 4c6b7107a..d2b408ea8 100644 --- a/worker/internal/slotgate/node_health.go +++ b/worker/internal/slotgate/node_health.go @@ -79,6 +79,12 @@ func (s *Server) readHealth(ctx context.Context, request gate2Request) (map[stri if err != nil { return nil, err } + if snapshot.InitialManifest.CredentialAuthority == nodePrivateCredentialAuthority { + before, err = s.authority.readCanonicalCredential(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy, snapshot) + if err != nil { + return nil, err + } + } summary, identity, err := nodeCredentialSummary(before, snapshot.InitialManifest.Identity) if err != nil { return nil, err @@ -129,6 +135,9 @@ func (s *Server) readHealth(ctx context.Context, request gate2Request) (map[stri return nil, err } after, err := readPublishedCredential(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy) + if snapshot.InitialManifest.CredentialAuthority == nodePrivateCredentialAuthority { + after, err = s.authority.readCanonicalCredential(s.cfg.Root, request.VMID, request.Fence, s.launcher.policy, snapshot) + } if err != nil || !bytes.Equal(before, after) { return nil, ErrManagementDenied } diff --git a/worker/internal/slotgate/node_private_credential.go b/worker/internal/slotgate/node_private_credential.go new file mode 100644 index 000000000..a371980db --- /dev/null +++ b/worker/internal/slotgate/node_private_credential.go @@ -0,0 +1,73 @@ +package slotgate + +import ( + "bytes" + "encoding/json" +) + +const nodePrivateCredentialAuthority = "node-private-v1" + +type nodeCredentialState struct { + Generation string `json:"generation"` + CanonicalDigest string `json:"canonical_digest"` + ProjectionDigest string `json:"projection_digest"` +} + +func validNodeCredentialState(s *nodeCredentialState) bool { + if s == nil || !validDigest(s.CanonicalDigest) || !validDigest(s.ProjectionDigest) { + return false + } + gen, err := credentialScalar(json.RawMessage(s.Generation)) + return err == nil && gen == s.Generation && gen != "0" +} + +// Only the private canonical document contains the rotating token. The guest +// projection retains the existing schema and cannot refresh by itself. +func credentialProjection(data []byte) ([]byte, error) { + var doc map[string]json.RawMessage + if strictJSON(data, &doc) != nil { + return nil, ErrInvalidAuthority + } + if raw, ok := doc["claudeAiOauth"]; ok { + var oauth map[string]json.RawMessage + if strictJSON(raw, &oauth) != nil { + return nil, ErrInvalidAuthority + } + delete(oauth, "refreshToken") + delete(oauth, "refresh_token") + var err error + doc["claudeAiOauth"], err = json.Marshal(oauth) + if err != nil { + return nil, err + } + } + delete(doc, "refreshToken") + delete(doc, "refresh_token") + return json.Marshal(doc) +} + +func (a *Authority) readCanonicalCredential(root, vm string, fence Fence, policy DockerLaunchPolicy, state AuthorityState) ([]byte, error) { + if state.InitialManifest == nil || state.InitialManifest.CredentialAuthority != nodePrivateCredentialAuthority || !validNodeCredentialState(state.Credential) { + return nil, ErrManagementDenied + } + canonical, err := readNodeCredential(a.root, vm, fence) + if err != nil || hashBytes(canonical) != state.Credential.CanonicalDigest { + return nil, ErrInvalidAuthority + } + projection, err := credentialProjection(canonical) + if err != nil || hashBytes(projection) != state.Credential.ProjectionDigest { + return nil, ErrInvalidAuthority + } + guest, err := readPublishedCredential(root, vm, fence, policy) + if err != nil || !bytes.Equal(guest, projection) { + return nil, ErrManagementDenied + } + if err = checkCredentialLink(root, vm, fence, policy); err != nil { + return nil, err + } + summary, _, err := nodeCredentialSummary(canonical, state.InitialManifest.Identity) + if err != nil || summary["generation"] != state.Credential.Generation { + return nil, ErrInvalidAuthority + } + return canonical, nil +} diff --git a/worker/internal/slotgate/node_private_credential_linux.go b/worker/internal/slotgate/node_private_credential_linux.go new file mode 100644 index 000000000..7dca445ef --- /dev/null +++ b/worker/internal/slotgate/node_private_credential_linux.go @@ -0,0 +1,75 @@ +//go:build linux + +package slotgate + +import ( + "golang.org/x/sys/unix" + "os" +) + +func nodeCredentialDirectory(root, vm string, fence Fence, create bool) (*os.File, error) { + if !vmIDRe.MatchString(vm) || len(fence.Incarnation) != 32 { + return nil, ErrInvalidAuthority + } + dir, err := secureRoot(root) + if err != nil { + return nil, err + } + for _, name := range []string{"credentials", vm, fence.Incarnation} { + next, e := openDirectoryAt(dir, name, create, os.Geteuid()) + if e == nil && create { + e = dir.Sync() + } + dir.Close() + if e != nil { + if next != nil { + next.Close() + } + return nil, e + } + dir = next + } + return dir, nil +} +func writeInitialNodeCredential(root, vm string, fence Fence, data []byte) error { + dir, err := nodeCredentialDirectory(root, vm, fence, true) + if err != nil { + return err + } + defer dir.Close() + if err = writeInitialFile(dir, "credentials.json", data, os.Geteuid(), os.Getegid(), 0o600); err != nil { + return err + } + return dir.Sync() +} +func readNodeCredential(root, vm string, fence Fence) ([]byte, error) { + dir, err := nodeCredentialDirectory(root, vm, fence, false) + if err != nil { + return nil, err + } + defer dir.Close() + return readInitialFile(dir, "credentials.json", os.Geteuid(), initialPublishMax) +} +func checkCredentialLink(root, vm string, fence Fence, policy DockerLaunchPolicy) error { + parent, err := incarnationParent(root, vm, false) + if err != nil { + return err + } + defer parent.Close() + base, err := openDirectoryAt(parent, fence.Incarnation, false, os.Geteuid()) + if err != nil { + return err + } + defer base.Close() + dir, err := openDirectoryAt(base, "claude", false, policy.UID) + if err != nil { + return err + } + defer dir.Close() + link := make([]byte, 64) + n, err := unix.Readlinkat(int(dir.Fd()), ".credentials.json", link) + if err != nil || string(link[:n]) != "credentials.json" { + return ErrInvalidAuthority + } + return nil +} diff --git a/worker/internal/slotgate/node_private_credential_other.go b/worker/internal/slotgate/node_private_credential_other.go new file mode 100644 index 000000000..fc45b00a7 --- /dev/null +++ b/worker/internal/slotgate/node_private_credential_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package slotgate + +func writeInitialNodeCredential(string, string, Fence, []byte) error { return errPublishUnsupported } +func readNodeCredential(string, string, Fence) ([]byte, error) { return nil, errPublishUnsupported } +func checkCredentialLink(string, string, Fence, DockerLaunchPolicy) error { + return errPublishUnsupported +} diff --git a/worker/internal/slotgate/publish_initial.go b/worker/internal/slotgate/publish_initial.go index d13311eb8..76cdc56d2 100644 --- a/worker/internal/slotgate/publish_initial.go +++ b/worker/internal/slotgate/publish_initial.go @@ -208,6 +208,10 @@ func (p *initialPublisher) Publish(ctx context.Context, intent ManagementIntent, if err != nil { return PublicationReceipt{}, err } + projection, err := credentialProjection(credential) + if err != nil { + return PublicationReceipt{}, err + } var secret [32]byte if _, err = rand.Read(secret[:]); err != nil { return PublicationReceipt{}, err @@ -217,14 +221,17 @@ func (p *initialPublisher) Publish(ctx context.Context, intent ManagementIntent, if err != nil { return PublicationReceipt{}, err } - manifest := initialManifest{Receipt: PublicationReceipt{Intent: intent, IntentDigest: intent.digest(), ManifestRevision: "1"}, KernelDigest: hashBytes(kernel), TokenDigest: hashText(token + "\n"), CredentialDigest: hashBytes(credential)} + manifest := initialManifest{Receipt: PublicationReceipt{Intent: intent, IntentDigest: intent.digest(), ManifestRevision: "1"}, KernelDigest: hashBytes(kernel), TokenDigest: hashText(token + "\n"), CredentialDigest: hashBytes(projection), CredentialAuthority: nodePrivateCredentialAuthority, PrivateCredentialDigest: hashBytes(credential)} var payload initialPayload if err = json.Unmarshal(data, &payload); err != nil { return PublicationReceipt{}, ErrInvalidAuthority } manifest.Identity = credentialIdentity{payload.Credential.AccountUUID, payload.Credential.OrgUUID, payload.Credential.Email} manifest.Receipt.ManifestDigest = manifest.digest() - tree, err := stageInitialFiles(p.root, intent, p.policy, kernel, []byte(token+"\n"), credential, p.fault) + if err = writeInitialNodeCredential(p.authority.root, intent.VMID, intent.Fence, credential); err != nil { + return PublicationReceipt{}, err + } + tree, err := stageInitialFiles(p.root, intent, p.policy, kernel, []byte(token+"\n"), projection, p.fault) if err != nil { return PublicationReceipt{}, err } @@ -292,6 +299,7 @@ func (a *Authority) completeInitialPublish(handle CommandHandle, intent Manageme command.Status = CommandCompleted state.Commands[handle.ID] = command state.InitialManifest = &manifest + state.Credential = &nodeCredentialState{Generation: "1", CanonicalDigest: manifest.PrivateCredentialDigest, ProjectionDigest: manifest.CredentialDigest} _, err = a.commit(handle.VMID, state) return err } @@ -306,6 +314,9 @@ func validateInitialManifest(vm string, state AuthorityState) error { m.Receipt.IntentDigest != i.digest() || m.Receipt.ManifestRevision != "1" || !validDigest(m.KernelDigest) || !validDigest(m.TokenDigest) || !validDigest(m.CredentialDigest) || m.Receipt.ManifestDigest != m.digest() { return ErrInvalidAuthority } + if m.CredentialAuthority != "" && (m.CredentialAuthority != nodePrivateCredentialAuthority || !validDigest(m.PrivateCredentialDigest) || !validNodeCredentialState(state.Credential)) { + return ErrInvalidAuthority + } command, ok := state.Commands[i.CommandID] if !ok || command.Kind != commandPublish || command.Status != CommandCompleted || command.Management == nil || *command.Management != i { return ErrInvalidAuthority From 248af6e1783fa09eea29efb484a11ae85262a37d Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:07:58 +0800 Subject: [PATCH 419/473] test: specify node refresh attempt and generation fencing --- worker/internal/slotgate/node_refresh_test.go | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 worker/internal/slotgate/node_refresh_test.go diff --git a/worker/internal/slotgate/node_refresh_test.go b/worker/internal/slotgate/node_refresh_test.go new file mode 100644 index 000000000..18e6fe289 --- /dev/null +++ b/worker/internal/slotgate/node_refresh_test.go @@ -0,0 +1,111 @@ +package slotgate + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/dofastted/kin-gateway/worker/internal/credential" + kinoauth "github.com/dofastted/kin-gateway/worker/internal/oauth" +) + +func refreshFixture(t *testing.T, handler http.Handler) (*publicationFixture, *nodeCredentialRefresher, RefreshIntent, *atomic.Int64) { + t.Helper() + f := publishingFixture(t) + f.publish(t) + state, _ := f.a.State("v") + if _, err := f.l.LaunchManifest(context.Background(), launchIntentForTest(state, "refresh-launch"), allowNodeCommandForTest); err != nil { + t.Fatal(err) + } + state, _ = f.a.State("v") + calls := new(atomic.Int64) + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); handler.ServeHTTP(w, r) })) + t.Cleanup(provider.Close) + r := &nodeCredentialRefresher{authority: f.a, root: f.root, nodeID: f.intent.NodeID, policy: f.l.policy, authorize: func(_ context.Context, _ RefreshIntent, dispatch func() error) error { return dispatch() }, newRefresher: func(store *credential.Store) *kinoauth.Refresher { + return &kinoauth.Refresher{Store: store, Client: provider.Client(), TokenURL: provider.URL, MaxTries: 1} + }} + i := RefreshIntent{Action: "credential-refresh", NodeID: f.intent.NodeID, VMID: "v", AllocationID: f.intent.AllocationID, OperationID: "refresh-operation", OwnerID: "refresh-owner", CommandID: "refresh-command", AccountID: "synthetic-account", BGTaskKey: "oauth-refresh:claude:synthetic-account", BGExecutionID: "refresh-execution", BGOwnerID: "refresh-bg-owner", Epoch: "1", Fence: state.Fence, ExpectedGeneration: "1", ManifestRevision: "1", ManifestDigest: state.InitialManifest.Receipt.ManifestDigest, PresetID: initialPresetID, PolicyDigest: f.l.policy.digest(), Force: true} + return f, r, i, calls +} + +func TestNodeRefreshPersistsOneAttemptAndRejectsLateGeneration(t *testing.T) { + f, r, i, calls := refreshFixture(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"access_token":"synthetic-rotated","refresh_token":"synthetic-next","expires_in":3600}`) + })) + receipt, err := r.Refresh(context.Background(), i) + if err != nil || !receipt.OK || !receipt.Finished { + t.Fatalf("refresh did not complete: %+v %v", receipt, err) + } + if receipt.Credential.Generation == "1" || receipt.Credential.AccountUUID != "synthetic-account" { + t.Fatal("receipt lacks node canonical generation") + } + if _, err = r.Refresh(context.Background(), i); err == nil { + t.Fatal("completed attempt was resubmitted") + } + late := i + late.CommandID = "late-command" + if _, err = r.Refresh(context.Background(), late); err == nil { + t.Fatal("late generation reached provider") + } + if calls.Load() != 1 { + t.Fatalf("provider requests=%d want 1", calls.Load()) + } + state, _ := f.a.State("v") + if _, err = f.a.readCanonicalCredential(f.root, "v", i.Fence, f.l.policy, state); err != nil { + t.Fatal(err) + } +} + +func TestNodeRefreshUnknownNeverRetriesOrAcceptsReplacementAttempt(t *testing.T) { + f, r, i, calls := refreshFixture(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + c, b, _ := w.(http.Hijacker).Hijack() + fmt.Fprint(b, "HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{\"access_token\":") + b.Flush() + c.Close() + })) + if _, err := r.Refresh(context.Background(), i); err == nil { + t.Fatal("truncated accepted refresh completed") + } + replacement := i + replacement.CommandID = "replacement-command" + if _, err := r.Refresh(context.Background(), replacement); err == nil { + t.Fatal("unknown refresh was reissued") + } + if calls.Load() != 1 { + t.Fatalf("provider requests=%d want 1", calls.Load()) + } + state, _ := f.a.State("v") + if !hasPendingMutation(state) { + t.Fatal("unknown refresh released node mutation") + } +} + +func TestNodeRefreshRequiresPrivateOwnershipAndOriginalActiveFence(t *testing.T) { + f, r, i, calls := refreshFixture(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { t.Error("provider called after denied refresh") })) + stale := i + stale.Fence.Incarnation = strings.Repeat("f", 32) + if _, err := r.Refresh(context.Background(), stale); !errors.Is(err, ErrFenceMismatch) { + t.Fatalf("stale fence: %v", err) + } + f.a.mu.Lock() + state := f.a.journal.Slots["v"] + state.InitialManifest.CredentialAuthority = "" + state.InitialManifest.PrivateCredentialDigest = "" + state.Credential = nil + f.a.journal.Slots["v"] = state + f.a.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := r.Refresh(ctx, i); err == nil { + t.Fatal("legacy exposed refresh token was adopted") + } + if calls.Load() != 0 { + t.Fatal("denied refresh reached provider") + } +} From 98640ce33e0767297a86cd71f27f5887ee67fa50 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:11:23 +0800 Subject: [PATCH 420/473] feat: persist fenced node credential refresh attempts --- worker/internal/slotgate/authority.go | 21 +- .../slotgate/node_private_credential_linux.go | 34 +++ .../slotgate/node_private_credential_other.go | 4 + worker/internal/slotgate/node_refresh.go | 226 ++++++++++++++++++ 4 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 worker/internal/slotgate/node_refresh.go diff --git a/worker/internal/slotgate/authority.go b/worker/internal/slotgate/authority.go index 78e03938c..6d371bb3c 100644 --- a/worker/internal/slotgate/authority.go +++ b/worker/internal/slotgate/authority.go @@ -91,6 +91,8 @@ type CommandRecord struct { Management *ManagementIntent `json:"management,omitempty"` LaunchReceipt *LaunchReceipt `json:"launch_receipt,omitempty"` TerminalReceipt *inferenceReceipt `json:"terminal_receipt,omitempty"` + Refresh *RefreshIntent `json:"refresh,omitempty"` + RefreshReceipt *RefreshReceipt `json:"refresh_receipt,omitempty"` } // DockerLaunchTarget is recorded after create returns and before start. A @@ -497,7 +499,7 @@ func (a *Authority) CompleteCommand(handle CommandHandle) error { if command.Status != CommandPending { return ErrCommandOwner } - if command.Kind == commandPublish || (command.Kind == commandLaunch && command.Dispatched) { + if command.Kind == commandPublish || command.Kind == commandRefresh || (command.Kind == commandLaunch && command.Dispatched) { return ErrAuthorityPhase } command.Status = CommandCompleted @@ -678,6 +680,10 @@ func (a *Authority) persist(next authorityJournal) error { } func cloneState(state AuthorityState) AuthorityState { + if state.Credential != nil { + credential := *state.Credential + state.Credential = &credential + } if state.InitialManifest != nil { manifest := *state.InitialManifest state.InitialManifest = &manifest @@ -689,6 +695,14 @@ func cloneState(state AuthorityState) AuthorityState { commands := make(map[string]CommandRecord, len(state.Commands)) for id, command := range state.Commands { command.TerminalReceipt = cloneInferenceReceipt(command.TerminalReceipt) + if command.Refresh != nil { + intent := *command.Refresh + command.Refresh = &intent + } + if command.RefreshReceipt != nil { + receipt := *command.RefreshReceipt + command.RefreshReceipt = &receipt + } if command.Management != nil { intent := *command.Management command.Management = &intent @@ -821,9 +835,12 @@ func validateJournal(journal authorityJournal) error { if id != command.ID || !validAuthorityID(id) || !validAuthorityID(command.Runtime) || !validAuthorityID(command.Token) || command.Fence.Generation == 0 || command.Fence.Generation > state.Fence.Generation || !validAuthorityID(command.Fence.Incarnation) { return ErrInvalidAuthority } - if command.Kind != CommandForward && command.Kind != CommandMutation && command.Kind != commandLaunch && command.Kind != commandPublish { + if command.Kind != CommandForward && command.Kind != CommandMutation && command.Kind != commandLaunch && command.Kind != commandPublish && command.Kind != commandRefresh { return ErrInvalidAuthority } + if err := validateRefreshCommand(vm, command); err != nil { + return err + } if command.Kind == commandPublish && command.Management == nil { return ErrInvalidAuthority } diff --git a/worker/internal/slotgate/node_private_credential_linux.go b/worker/internal/slotgate/node_private_credential_linux.go index 7dca445ef..4cfdd3df6 100644 --- a/worker/internal/slotgate/node_private_credential_linux.go +++ b/worker/internal/slotgate/node_private_credential_linux.go @@ -73,3 +73,37 @@ func checkCredentialLink(root, vm string, fence Fence, policy DockerLaunchPolicy } return nil } + +func publishCredentialProjection(root, vm string, fence Fence, policy DockerLaunchPolicy, data []byte) error { + parent, err := incarnationParent(root, vm, false) + if err != nil { + return err + } + defer parent.Close() + base, err := openDirectoryAt(parent, fence.Incarnation, false, os.Geteuid()) + if err != nil { + return err + } + defer base.Close() + dir, err := openDirectoryAt(base, "claude", false, policy.UID) + if err != nil { + return err + } + defer dir.Close() + if _, err = readInitialFile(dir, "credentials.json", policy.UID, initialPublishMax); err != nil { + return err + } + id, err := randomAuthorityID() + if err != nil { + return err + } + name := ".node-credential-" + id + if err = writeInitialFile(dir, name, data, policy.UID, policy.GID, 0o400); err != nil { + return err + } + defer unix.Unlinkat(int(dir.Fd()), name, 0) + if err = unix.Renameat(int(dir.Fd()), name, int(dir.Fd()), "credentials.json"); err != nil { + return err + } + return dir.Sync() +} diff --git a/worker/internal/slotgate/node_private_credential_other.go b/worker/internal/slotgate/node_private_credential_other.go index fc45b00a7..21e1b1355 100644 --- a/worker/internal/slotgate/node_private_credential_other.go +++ b/worker/internal/slotgate/node_private_credential_other.go @@ -7,3 +7,7 @@ func readNodeCredential(string, string, Fence) ([]byte, error) { return ni func checkCredentialLink(string, string, Fence, DockerLaunchPolicy) error { return errPublishUnsupported } + +func publishCredentialProjection(string, string, Fence, DockerLaunchPolicy, []byte) error { + return errPublishUnsupported +} diff --git a/worker/internal/slotgate/node_refresh.go b/worker/internal/slotgate/node_refresh.go new file mode 100644 index 000000000..cf267a2dc --- /dev/null +++ b/worker/internal/slotgate/node_refresh.go @@ -0,0 +1,226 @@ +package slotgate + +import ( + "context" + "encoding/json" + "errors" + "github.com/dofastted/kin-gateway/worker/internal/credential" + kinoauth "github.com/dofastted/kin-gateway/worker/internal/oauth" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" +) + +const commandRefresh CommandKind = "credential-refresh" + +type RefreshIntent struct { + Action string `json:"action"` + NodeID string `json:"node_id"` + VMID string `json:"vm_id"` + AllocationID string `json:"allocation_id"` + OperationID string `json:"operation_id"` + OwnerID string `json:"owner_id"` + CommandID string `json:"command_id"` + AccountID string `json:"account_id"` + BGTaskKey string `json:"bg_task_key"` + BGExecutionID string `json:"bg_execution_id"` + BGOwnerID string `json:"bg_owner_id"` + Epoch string `json:"epoch"` + Fence Fence `json:"fence"` + ExpectedGeneration string `json:"expected_credential_generation"` + ManifestRevision string `json:"manifest_revision"` + ManifestDigest string `json:"manifest_digest"` + PresetID string `json:"preset_id"` + PolicyDigest string `json:"policy_digest"` + Force bool `json:"force"` +} + +func validRefreshIntent(i RefreshIntent) bool { + if i.Action != "credential-refresh" || !validAllocation(AllocationIntent{i.NodeID, i.VMID, i.AllocationID, i.OperationID, i.OwnerID}, i.Epoch) || !vmIDRe.MatchString(i.CommandID) || !vmIDRe.MatchString(i.AccountID) || !vmIDRe.MatchString(i.BGOwnerID) || i.BGTaskKey != "oauth-refresh:claude:"+i.AccountID || !vmIDRe.MatchString(i.BGExecutionID) || validateExecutor(i.Fence.Executor) != nil || i.Fence.Executor.Kind != "container" || i.Fence.Generation == 0 || !validAuthorityID(i.Fence.Incarnation) || i.ManifestRevision != "1" || !validDigest(i.ManifestDigest) || !validDigest(i.PolicyDigest) || i.PresetID != initialPresetID { + return false + } + n, err := strconv.ParseInt(i.ExpectedGeneration, 10, 64) + return err == nil && n > 0 && n < 1<<63-1 && strconv.FormatInt(n, 10) == i.ExpectedGeneration +} +func (i RefreshIntent) digest() string { + force := "0" + if i.Force { + force = "1" + } + e := i.Fence.Executor + return hashText(strings.Join([]string{"slotgate-refresh-v1", i.Action, i.NodeID, i.VMID, i.AllocationID, i.OperationID, i.OwnerID, i.CommandID, i.AccountID, i.BGTaskKey, i.BGExecutionID, i.BGOwnerID, i.Epoch, strconv.FormatUint(i.Fence.Generation, 10), i.Fence.Incarnation, e.Kind, e.NodeBootID, e.ContainerID, e.DockerDaemonID, e.StartedAt, i.ExpectedGeneration, i.ManifestRevision, i.ManifestDigest, i.PresetID, i.PolicyDigest, force}, "\x00")) +} + +type PublicCredential struct { + Generation string `json:"generation"` + ExpiresAt string `json:"expires_at"` + AccountUUID string `json:"account_uuid"` + OrgUUID string `json:"org_uuid,omitempty"` + Email string `json:"email,omitempty"` + HasAccess bool `json:"has_access"` + HasRefresh bool `json:"has_refresh"` + NeedsRefresh bool `json:"needs_refresh"` +} +type RefreshReceipt struct { + Intent RefreshIntent `json:"intent"` + IntentDigest string `json:"intent_digest"` + Finished bool `json:"finished"` + OK bool `json:"ok"` + Status int `json:"status"` + Refreshed bool `json:"refreshed"` + ErrorCode string `json:"error_code,omitempty"` + Credential PublicCredential `json:"credential"` +} +type RefreshAuthorizer func(context.Context, RefreshIntent, func() error) error +type nodeCredentialRefresher struct { + authority *Authority + root string + nodeID string + policy DockerLaunchPolicy + authorize RefreshAuthorizer + newRefresher func(*credential.Store) *kinoauth.Refresher + fault func(string) error +} + +func publicCredential(data []byte, identity credentialIdentity) (PublicCredential, error) { + summary, _, err := nodeCredentialSummary(data, identity) + if err != nil { + return PublicCredential{}, err + } + encoded, _ := json.Marshal(summary) + var result PublicCredential + if err = json.Unmarshal(encoded, &result); err != nil { + return result, err + } + return result, nil +} +func (r *nodeCredentialRefresher) Refresh(ctx context.Context, i RefreshIntent) (RefreshReceipt, error) { + if r.authorize == nil || i.NodeID != r.nodeID || !validRefreshIntent(i) || i.PolicyDigest != r.policy.digest() { + return RefreshReceipt{}, ErrManagementDenied + } + var state AuthorityState + var handle CommandHandle + err := r.authorize(ctx, i, func() error { var err error; handle, state, err = r.authority.beginCredentialRefresh(i); return err }) + if err != nil { + return RefreshReceipt{}, err + } + canonical, err := r.authority.readCanonicalCredential(r.root, i.VMID, i.Fence, r.policy, state) + if err != nil { + return RefreshReceipt{}, err + } + // The durable start precedes the provider request. An ambiguous failure + // retains this mutation; later file observations never authorize replay. + store := credential.NewStore(filepath.Join(r.authority.root, "credentials", i.VMID, i.Fence.Incarnation, "credentials.json")) + refresher := &kinoauth.Refresher{Store: store, Client: &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, TokenURL: "https://platform.claude.com/v1/oauth/token"} + if r.newRefresher != nil { + refresher = r.newRefresher(store) + } + result, refreshErr := refresher.Ensure(ctx, i.Force) + receipt := RefreshReceipt{Intent: i, IntentDigest: i.digest(), Finished: true, OK: refreshErr == nil, Status: 200, Refreshed: result.Refreshed} + if refreshErr != nil { + var rejection *kinoauth.RefreshError + if !errors.As(refreshErr, &rejection) || rejection.Unknown { + return RefreshReceipt{}, ErrAuthorityUnavailable + } + receipt.Status = rejection.Status + if receipt.Status < 400 || receipt.Status > 599 { + receipt.Status = 400 + } + receipt.ErrorCode = "credential_refresh_rejected" + } else { + canonical, err = readNodeCredential(r.authority.root, i.VMID, i.Fence) + if err != nil { + return RefreshReceipt{}, err + } + } + receipt.Credential, err = publicCredential(canonical, state.InitialManifest.Identity) + if err != nil { + return RefreshReceipt{}, err + } + projection, err := credentialProjection(canonical) + if err != nil { + return RefreshReceipt{}, err + } + if refreshErr == nil { + if err = publishFault(r.fault, "before-projection"); err != nil { + return RefreshReceipt{}, err + } + if err = publishCredentialProjection(r.root, i.VMID, i.Fence, r.policy, projection); err != nil { + return RefreshReceipt{}, err + } + if err = publishFault(r.fault, "after-projection"); err != nil { + return RefreshReceipt{}, err + } + } + err = r.authorize(ctx, i, func() error { + return r.authority.completeCredentialRefresh(handle, receipt, nodeCredentialState{Generation: receipt.Credential.Generation, CanonicalDigest: hashBytes(canonical), ProjectionDigest: hashBytes(projection)}) + }) + return receipt, err +} +func (a *Authority) beginCredentialRefresh(i RefreshIntent) (CommandHandle, AuthorityState, error) { + a.mu.Lock() + defer a.mu.Unlock() + state, err := a.match(i.VMID, i.Fence) + if err != nil { + return CommandHandle{}, state, err + } + if !validRefreshIntent(i) || state.Phase != PhaseActive || state.InitialManifest == nil || state.InitialManifest.CredentialAuthority != nodePrivateCredentialAuthority || state.Allocation == nil || state.Allocation.AllocationID != i.AllocationID || state.InitialManifest.Identity.AccountUUID != i.AccountID || state.InitialManifest.Receipt.ManifestDigest != i.ManifestDigest || state.InitialManifest.Receipt.Intent.PolicyDigest != i.PolicyDigest || !validNodeCredentialState(state.Credential) || state.Credential.Generation != i.ExpectedGeneration { + return CommandHandle{}, state, ErrManagementDenied + } + if _, ok := state.Commands[i.CommandID]; ok { + return CommandHandle{}, state, ErrCommandExists + } + for _, command := range state.Commands { + if pendingCommand(command) { + return CommandHandle{}, state, ErrPendingMutation + } + } + token, err := randomAuthorityID() + if err != nil { + return CommandHandle{}, state, err + } + copy := i + state.Commands[i.CommandID] = CommandRecord{ID: i.CommandID, Kind: commandRefresh, Status: CommandPending, Fence: i.Fence, Runtime: a.runtime, Token: token, Dispatched: true, Refresh: ©} + state, err = a.commit(i.VMID, state) + return CommandHandle{VMID: i.VMID, ID: i.CommandID, Fence: i.Fence, runtime: a.runtime, token: token}, state, err +} +func (a *Authority) completeCredentialRefresh(handle CommandHandle, receipt RefreshReceipt, credential nodeCredentialState) error { + a.mu.Lock() + defer a.mu.Unlock() + state, command, err := a.command(handle) + if err != nil { + return err + } + if state.Phase != PhaseActive || state.Fence != handle.Fence || command.Kind != commandRefresh || command.Status != CommandPending || command.Refresh == nil || *command.Refresh != receipt.Intent || !validRefreshReceipt(receipt) || !validNodeCredentialState(&credential) || credential.Generation != receipt.Credential.Generation { + return ErrAuthorityPhase + } + command.Status = CommandCompleted + command.RefreshReceipt = &receipt + state.Commands[handle.ID] = command + state.Credential = &credential + _, err = a.commit(handle.VMID, state) + return err +} +func validRefreshReceipt(r RefreshReceipt) bool { + return validRefreshIntent(r.Intent) && r.IntentDigest == r.Intent.digest() && r.Finished && r.Credential.AccountUUID == r.Intent.AccountID && r.Credential.Generation != "" && ((r.OK && r.Status == 200 && r.ErrorCode == "") || (!r.OK && r.Status >= 400 && r.Status <= 599 && r.ErrorCode == "credential_refresh_rejected")) +} +func validateRefreshCommand(vm string, c CommandRecord) error { + if c.Kind != commandRefresh { + if c.Refresh != nil || c.RefreshReceipt != nil { + return ErrInvalidAuthority + } + return nil + } + if c.Refresh == nil || !validRefreshIntent(*c.Refresh) || c.Refresh.VMID != vm || c.Refresh.CommandID != c.ID || c.Refresh.Fence != c.Fence { + return ErrInvalidAuthority + } + if c.RefreshReceipt != nil && (c.Status != CommandCompleted || c.RefreshReceipt.Intent != *c.Refresh || !validRefreshReceipt(*c.RefreshReceipt)) { + return ErrInvalidAuthority + } + if c.Status == CommandCompleted && c.RefreshReceipt == nil { + return ErrInvalidAuthority + } + return nil +} From 8c191783482b4bec22cbc3a4a432d9d702b78ede Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:12:05 +0800 Subject: [PATCH 421/473] test: require exact durable node credential refresh authorization --- test/unit/node-credential-refresh.test.mjs | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 test/unit/node-credential-refresh.test.mjs diff --git a/test/unit/node-credential-refresh.test.mjs b/test/unit/node-credential-refresh.test.mjs new file mode 100644 index 000000000..7a426a2a4 --- /dev/null +++ b/test/unit/node-credential-refresh.test.mjs @@ -0,0 +1,70 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import pg from 'pg' +import {getDb,createDatabase,quoteIdentifier} from '../../src/lib/db/database.mjs' +import {GlobalMaintenanceCoordinator} from '../../src/lib/db/global-maintenance.mjs' +import {NodeManagementGrants} from '../../src/lib/cluster/node-management-grants.mjs' +import {SharedVmOperations} from '../../src/lib/vm/shared-vm-operations.mjs' +import {SharedCredentialRefresh} from '../../src/lib/oauth/shared-credential-refresh.mjs' +import * as refreshModule from '../../src/lib/cluster/node-credential-refresh-grants.mjs' + +const nodeId='refresh-node',vmId='refresh-vm',accountId='refresh-account',allocationId='refresh-allocation' +const vacant={generation:'1',incarnation:'a'.repeat(32),executor:{kind:'',node_boot_id:''}} +const fence={generation:'1',incarnation:vacant.incarnation,executor:{kind:'container',node_boot_id:'synthetic-boot',container_id:'b'.repeat(64),docker_daemon_id:'synthetic-daemon',started_at:'2026-10-22T00:00:00.000000001Z'}} +const summary=(generation)=>({generation,expires_at:'2000000000000',account_uuid:accountId,has_access:true,has_refresh:true,needs_refresh:false}) +async function fixture(t){ + const db=getDb(),peer=await createDatabase({connectionString:db.connectionString,schema:db.schema}) + const maintenance=new GlobalMaintenanceCoordinator(db),base=new NodeManagementGrants(db) + const grants=new refreshModule.NodeCredentialRefreshGrants(db),q=quoteIdentifier(base.schema) + const readers=[],roles=[];let closed=false + const close=async()=>{if(closed)return;closed=true;await maintenance.stop();await peer.close();for(const c of readers)await c.end();const c=new pg.Client({connectionString:db.connectionString});await c.connect();try{for(const role of roles)await c.query(`DROP OWNED BY ${quoteIdentifier(role)};DROP ROLE ${quoteIdentifier(role)}`);await c.query(`DROP SCHEMA IF EXISTS ${q} CASCADE;DROP SCHEMA IF EXISTS ${quoteIdentifier(maintenance.schema)} CASCADE`)}finally{await c.end()}} + t.after(close);await maintenance.start();await maintenance.acknowledgeEpoch(1);await base.initialize();await base.initializeCommands() + const run=(kind,fn)=>new SharedVmOperations(db).run(vmId,fn,{kind,accountId}) + await run('node_allocate',async operation=>{const g=await base.issueAllocation({operation,nodeId,vmId,allocationId});await db.query(`UPDATE ${q}.allocations SET receipt=$2::jsonb WHERE allocation_id=$1`,[allocationId,JSON.stringify({node_id:nodeId,vm_id:vmId,allocation_id:allocationId,operation_id:g.operationId,owner_id:g.ownerId,epoch:g.epoch,intent_digest:g.intentDigest,fence:vacant})])}) + await run('node_publish_initial',async operation=>{const g=await base.issueInitialPublish({operation,nodeId,vmId,allocationId,commandId:'initial-publish',capturedFence:vacant,expectedManifestRevision:'0',revision:'1',payloadDigest:'c'.repeat(64),presetId:'claude-docker-v1',policyDigest:'d'.repeat(64)});await base.recordInitialPublishReceipt(g,{intent:g.intent,intent_digest:g.intentDigest,manifest_revision:'1',manifest_digest:'e'.repeat(64)})}) + await run('node_launch',async operation=>{const g=await base.issueLaunch({operation,nodeId,vmId,allocationId,commandId:'initial-launch',capturedFence:vacant,expectedManifestRevision:'1',revision:'1',payloadDigest:'',presetId:'claude-docker-v1',policyDigest:'d'.repeat(64),manifestDigest:'e'.repeat(64)});await base.recordLaunchReceipt(g,{intent:g.intent,intent_digest:g.intentDigest,manifest_revision:'1',manifest_digest:'e'.repeat(64),fence})}) + let current={generation:'1',credential:summary('1')} + const refresh=new SharedCredentialRefresh(db) + const input=(operation,patch={})=>({operation,nodeId,vmId,accountId,commandId:crypto.randomUUID(),capturedFence:fence,expectedGeneration:current.generation,force:true,...patch}) + const attempt=(callback)=>refresh.run({platform:'claude',vmId,accountId,expectedGeneration:current.generation},{readCurrent:async()=>current,refresh:callback,publish:async(_old,value)=>{current={generation:value.generation,credential:summary(value.generation)};return current}}) + const receipt=(g,generation='2')=>({intent:g.intent,intent_digest:g.intentDigest,finished:true,ok:true,status:200,refreshed:true,credential:summary(generation)}) + const reader=async()=>{const role='refresh_reader_'+crypto.randomBytes(8).toString('hex'),password=crypto.randomBytes(20).toString('hex');roles.push(role);await db.pool.query(`CREATE ROLE ${quoteIdentifier(role)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`);await db.pool.query(base.readerRoleSql({role,nodeId}));await db.pool.query(grants.readerRoleSql({role}));const url=new URL(db.connectionString);url.username=role;url.password=password;const c=new pg.Client({connectionString:url.toString()});await c.connect();readers.push(c);return c} + return{db,peer,base,grants,q,maintenance,run,input,attempt,receipt,reader,close} +} +function refreshCase(name,fn){test(name,async t=>{const f=await fixture(t);try{await fn(f,t)}finally{await f.close()}})} +refreshCase('refresh authority requires its independent explicit markers and creates no runtime DDL',async f=>{ + await assert.rejects(f.attempt(async(_current,{operation})=>{await f.grants.issue(f.input(operation));return{finished:true,value:{ok:false}}}),{code:'node_refresh_authority_unavailable'}) + assert.equal((await f.peer.query('SELECT to_regclass($1) AS relation',[`${f.base.schema}.refresh_grants`])).rows[0].relation,null) + await f.grants.initialize() + assert.equal((await f.peer.query(`SELECT refresh_protocol_version FROM ${f.q}.meta`)).rows[0].refresh_protocol_version,1) + await f.peer.query(`UPDATE ${quoteIdentifier(f.maintenance.schema)}.node_authority_features SET refresh_protocol_version=NULL`) + await assert.rejects(f.grants.state(),{code:'node_refresh_authority_unavailable'}) +}) +refreshCase('refresh grant requires exact live BG/VM/account scope and a canonical launch chain',async f=>{ + await f.grants.initialize() + await f.run('credential_refresh',async operation=>{await assert.rejects(f.grants.issue(f.input(operation)),{code:'node_refresh_scope_required'})}) + const reader=await f.reader();let command + const result=await f.attempt(async(_current,{operation})=>{ + await assert.rejects(f.grants.issue(f.input(operation,{accountId:'other-account'})),{code:'node_refresh_scope_required'}) + const g=await f.grants.issue(f.input(operation));command=g.intent.command_id + assert.equal((await reader.query(`SELECT command_id FROM ${f.q}.authorized_refreshes`)).rows[0].command_id,command) + await assert.rejects(reader.query(`SELECT * FROM ${f.q}.refresh_grants`),{code:'42501'}) + const stored=await f.grants.recordReceipt(g,f.receipt(g));assert.equal(stored.credential.generation,'2') + return{finished:true,value:{ok:true,generation:'2'}} + }) + assert.equal(result.unknown,false) + assert.equal((await reader.query(`SELECT count(*)::int AS n FROM ${f.q}.authorized_refreshes`)).rows[0].n,0) + const row=(await f.peer.query(`SELECT receipt FROM ${f.q}.refresh_grants WHERE command_id=$1`,[command])).rows[0] + assert.equal(row.receipt.credential.generation,'2');assert.equal(JSON.stringify(row).includes('access_token'),false) +}) +refreshCase('unknown refresh cannot be reissued by another connection or a different command ID',async f=>{ + await f.grants.initialize();let id + const result=await f.attempt(async(_current,{operation})=>{const g=await f.grants.issue(f.input(operation));id=g.intent.command_id;return{finished:false,value:{ok:false}}}) + assert.equal(result.unknown,true) + const row=(await f.peer.query(`SELECT receipt,revoked_at FROM ${f.q}.refresh_grants WHERE command_id=$1`,[id])).rows[0] + assert.equal(row.receipt,null) + assert.ok((await f.peer.query('SELECT uncertain_at FROM vm_operations WHERE kind=$1',['credential_refresh'])).rows[0].uncertain_at) + const gate=await new SharedVmOperations(f.peer).tryAcquire({vmId,accountId,kind:'credential_refresh'}) + assert.equal(gate.ok,false);assert.equal(gate.code,'vm_operation_active') +}) From 600e39c5cb0fb93bd3a06030d37d8606ce3b2923 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:33:12 +0800 Subject: [PATCH 422/473] feat: authorize typed node-owned credential refresh with durable claims --- .../2026-10-22-node-credential-refresh.md | 17 + .../node-credential-refresh-grants.mjs | 204 +++++++++++ src/lib/core/node-credential-refresh.mjs | 160 ++++++++ .../db/node-credential-refresh-authority.mjs | 86 +++++ src/lib/db/node-management-authority.mjs | 9 + src/lib/oauth/shared-credential-refresh.mjs | 24 +- src/lib/transport/go-worker-client.mjs | 39 +- src/lib/transport/slot-oauth.mjs | 119 +++++- .../transport/slotgate-management-client.mjs | 37 +- .../node-credential-refresh-wire.test.mjs | 272 ++++++++++++++ test/unit/node-credential-refresh.test.mjs | 345 +++++++++++++++--- worker/internal/slotgate/ksg2.go | 17 +- .../slotgate/node_private_credential.go | 4 +- worker/internal/slotgate/node_refresh.go | 3 +- .../slotgate/node_refresh_authorizer.go | 78 ++++ worker/internal/slotgate/node_refresh_test.go | 61 +++- worker/internal/slotgate/slotgate.go | 4 + 17 files changed, 1398 insertions(+), 81 deletions(-) create mode 100644 docs/plans/2026-10-22-node-credential-refresh.md create mode 100644 src/lib/cluster/node-credential-refresh-grants.mjs create mode 100644 src/lib/core/node-credential-refresh.mjs create mode 100644 src/lib/db/node-credential-refresh-authority.mjs create mode 100644 test/unit/node-credential-refresh-wire.test.mjs create mode 100644 worker/internal/slotgate/node_refresh_authorizer.go diff --git a/docs/plans/2026-10-22-node-credential-refresh.md b/docs/plans/2026-10-22-node-credential-refresh.md new file mode 100644 index 000000000..821c4c3dd --- /dev/null +++ b/docs/plans/2026-10-22-node-credential-refresh.md @@ -0,0 +1,17 @@ +# Node-owned credential refresh + +本批只覆盖新建普通 Claude Docker 节点实例。standalone 的本地 worker 刷新路径保留。所有验证使用合成凭据、fake Engine、httptest provider 和本任务 PostgreSQL。没有向真实 provider 或现有节点发请求。 + +新的 initial publication 在 AuthorityDir/credentials///credentials.json 持久保存 canonical 凭据。该路径不在任何 guest mount 内。guest 的既有 credentials.json 格式只投影 AT、generation、expiry、identity，不包含 RT。manifest 的 credential_authority=node-private-v1 与 private digest 绑定此布局。旧的已暴露 RT manifest 不自动接管；typed refresh 明确拒绝。guest 文件不能提供权威 generation；health 从 canonical 读取，并核对 guest 投影摘要、字节与固定 .credentials.json 链接。guest 替换同账号文件也会拒绝。 + +NodeCredentialRefreshGrants.initialize() 是显式可信初始化。private meta 与 coordination marker 的 refresh_protocol_version=1 必须同时存在。运行时不执行 DDL。readerRoleSql 仅授予 authorized_refreshes 和 refresh_current 的读取/执行权限；既有 reader_identity 仍校验该节点的独立角色。 + +KSG2 独立操作 credential-refresh 使用 refresh 字段，不使用 management/allocation/inference grant。RefreshIntent 固定字段为 action,node_id,vm_id,allocation_id,operation_id,owner_id,command_id,account_id,bg_task_key,bg_execution_id,bg_owner_id,epoch,fence,expected_credential_generation,manifest_revision,manifest_digest,preset_id,policy_digest,force。epoch/generation/revision 使用十进制字符串；Fence generation 沿用数字 wire 并无损解析。fence 必须是原 ACTIVE container 身份，node boot ID 必须等于当前节点。preset 固定 claude-docker-v1，provider 固定 https://platform.claude.com/v1/oauth/token，禁止 redirect。RPC 不接收 URL、argv、env、path 或 token。 + +intent_digest 为 SHA256，NUL 顺序为 slotgate-refresh-v1、action 到 epoch 的前12个字段、fence generation/incarnation、executor kind/node_boot_id/container_id/docker_daemon_id/started_at、expected_credential_generation、manifest_revision、manifest_digest、preset_id、policy_digest、force(0/1)。PG 只存公开 intent 和 receipt。grant 要求当前 SharedCredentialRefresh 回调的 live scope、exact VM operation/account 和当前 BG execution/owner。仅持有别人的 claim ID 不够。锁顺序为 BG row share → VM operation row → node/VM advisory lock → control row share；节点的共享授权锁只跨 journal commit，不跨 provider 请求。 + +节点先持久化 pending，再使用现有 Refresher。只有完整 provider 结果、canonical 保存、guest 投影 fsync、第二次 exact 授权和 journal commit 全部确定，才返回 finished receipt。完整 HTTP 拒绝可结束本次 attempt，后续显式 attempt 仍可重试。响应截断、timeout、保存失败、投影失败、journal/PG receipt 提交未知均保留占位；同 command 不重发，未完成 generation 不能重新 issue。容器结束证明不能清除节点上未知的 refresh mutation。 + +receipt 仅包含 intent、intent_digest、finished、ok、status、refreshed、可选固定 error_code 和公开 credential summary(generation/expiry/identity/presence)。Node 在任何 Pod 凭据读取前选择 typed 路径；PG receipt 发布后才完成共享 BG/VM claim。不通过 SFTP、Pod 凭据文件或 token hash 推断 freshness。未实现的 OAuth helper 保持明确拒绝，不隐式裸 Ensure。 + +验证边界：真实 PG + pinned TLS + Go 节点的组合场景使用未过期凭据、force=false，因此不请求 provider。实际 RT 轮换成功、截断及部分投影失败由 Go httptest provider 验证。未对真实闭源 payload 进行业务推理验收；guest 对新 AT 的运行时重新加载行为未验证。本批没有提供未知 attempt 自动恢复或旧 manifest 升级。 diff --git a/src/lib/cluster/node-credential-refresh-grants.mjs b/src/lib/cluster/node-credential-refresh-grants.mjs new file mode 100644 index 000000000..dfd0fc535 --- /dev/null +++ b/src/lib/cluster/node-credential-refresh-grants.mjs @@ -0,0 +1,204 @@ +import { refreshNodeSlotCredential } from '../transport/slotgate-management-client.mjs' +import { withTransaction, quoteIdentifier } from '../db/database.mjs' +import { coordinationSchemaFor } from '../db/global-maintenance.mjs' +import { NodeManagementGrants } from './node-management-grants.mjs' +import { nodeAuthoritySchemaFor, lockNodeGrant } from '../db/node-management-authority.mjs' +import { initializeNodeRefreshAuthority, nodeRefreshAuthorityState } from '../db/node-credential-refresh-authority.mjs' +import { currentCredentialRefreshAttempt } from '../oauth/shared-credential-refresh.mjs' +import { currentVmOperationIds } from '../vm/shared-vm-operations.mjs' +import { + normalizeRefreshIntent, + refreshIntentDigest, + normalizeRefreshGrant, + normalizeRefreshReceipt, + refreshFailure, +} from '../core/node-credential-refresh.mjs' +import { isDeepStrictEqual } from 'node:util' +export class NodeCredentialRefreshGrants { + constructor(db) { + this.db = db + this.schema = nodeAuthoritySchemaFor(db.schema) + this.base = new NodeManagementGrants(db) + } + initialize() { + return initializeNodeRefreshAuthority(this.db) + } + state() { + return nodeRefreshAuthorityState(this.db) + } + readerRoleSql({ role }) { + if (typeof role !== 'string' || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(role)) throw refreshFailure() + const q = quoteIdentifier(this.schema), + r = quoteIdentifier(role) + return `GRANT USAGE ON SCHEMA ${q} TO ${r};GRANT SELECT ON ${q}.authorized_refreshes TO ${r};GRANT EXECUTE ON FUNCTION ${q}.refresh_current(text) TO ${r};` + } + async _context(input) { + const scope = currentCredentialRefreshAttempt(this.db) + if ( + !scope || + scope.operation !== input.operation || + scope.vmId !== input.vmId || + scope.accountId !== input.accountId || + !currentVmOperationIds(this.db).has(input.operation?.operationId) + ) + throw refreshFailure('node_refresh_scope_required') + const bg = await this.db + .prepare('SELECT * FROM background_task_runs WHERE task_key=$1 AND execution_id=$2 AND owner_id=$3 FOR SHARE') + .get(scope.claim.taskKey, scope.claim.executionId, scope.claim.ownerId) + if (!bg || bg.finished_at || bg.unknown_at) throw refreshFailure('node_refresh_scope_required') + const op = await this.db + .prepare('SELECT * FROM vm_operations WHERE id=$1 AND owner_id=$2 FOR UPDATE') + .get(scope.operation.operationId, scope.operation.ownerId) + if ( + !op || + op.vm_id !== input.vmId || + op.account_id !== input.accountId || + op.kind !== 'credential_refresh' || + op.released_at || + op.uncertain_at + ) + throw refreshFailure('node_refresh_scope_required') + return scope + } + async _state(input) { + const scope = await this._context(input), + state = await this.state() + if (!state.enabled) throw refreshFailure('node_refresh_authority_unavailable') + await lockNodeGrant(this.db, this.schema, input.nodeId, input.vmId) + const control = ( + await this.db.query( + `SELECT epoch::text,phase FROM ${quoteIdentifier(coordinationSchemaFor(this.db.schema))}.control WHERE id=1 FOR SHARE`, + ) + ).rows[0] + if (control?.phase !== 'open') throw refreshFailure('node_refresh_epoch_stale') + const launched = await this.base.readLaunchReceipt({ nodeId: input.nodeId, vmId: input.vmId }) + if (!launched || !isDeepStrictEqual(launched.fence, input.capturedFence)) + throw refreshFailure('node_refresh_launch_required') + return { scope, state, epoch: control.epoch, launched } + } + async issue(input) { + if (this.db.isTransaction) throw refreshFailure('node_refresh_transaction_active') + return withTransaction(this.db, async () => { + const { scope, state, epoch, launched } = await this._state(input) + const intent = normalizeRefreshIntent({ + action: 'credential-refresh', + node_id: input.nodeId, + vm_id: input.vmId, + allocation_id: launched.intent.allocation_id, + operation_id: scope.operation.operationId, + owner_id: scope.operation.ownerId, + command_id: input.commandId, + account_id: input.accountId, + bg_task_key: scope.claim.taskKey, + bg_execution_id: scope.claim.executionId, + bg_owner_id: scope.claim.ownerId, + epoch, + fence: input.capturedFence, + expected_credential_generation: input.expectedGeneration, + manifest_revision: launched.manifest_revision, + manifest_digest: launched.manifest_digest, + preset_id: launched.intent.preset_id, + policy_digest: launched.intent.policy_digest, + force: input.force === true, + }) + const latest = ( + await this.db.query( + `SELECT receipt FROM ${state.q}.refresh_grants WHERE node_id=$1 AND vm_id=$2 AND epoch=$3 AND receipt IS NOT NULL ORDER BY created_at DESC,command_id DESC LIMIT 1`, + [intent.node_id, intent.vm_id, intent.epoch], + ) + ).rows[0] + if ((latest?.receipt?.credential?.generation ?? '1') !== intent.expected_credential_generation) + throw refreshFailure('node_refresh_generation_stale') + const intentDigest = refreshIntentDigest(intent) + const inserted = await this.db.query( + `INSERT INTO ${state.q}.refresh_grants(command_id,node_id,vm_id,allocation_id,operation_id,owner_id,account_id,bg_task_key,bg_execution_id,bg_owner_id,epoch,expected_generation,intent_digest,intent) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14::jsonb) ON CONFLICT DO NOTHING RETURNING command_id`, + [ + intent.command_id, + intent.node_id, + intent.vm_id, + intent.allocation_id, + intent.operation_id, + intent.owner_id, + intent.account_id, + intent.bg_task_key, + intent.bg_execution_id, + intent.bg_owner_id, + intent.epoch, + intent.expected_credential_generation, + intentDigest, + JSON.stringify(intent), + ], + ) + if (inserted.rowCount !== 1) throw refreshFailure('node_refresh_conflict') + return { intent, intentDigest } + }) + } + async refresh(input, link) { + let grant + try { + grant = await this.issue(input) + } catch (error) { + if ( + [ + 'node_refresh_authority_unavailable', + 'node_refresh_scope_required', + 'node_refresh_generation_stale', + 'node_refresh_launch_required', + 'node_refresh_epoch_stale', + ].includes(error.code) + ) { + error.vmOperationNotStarted = true + error.credentialRefreshNotStarted = true + } + throw error + } + let receipt + try { + receipt = await refreshNodeSlotCredential({ link, grant }) + } catch (error) { + if (error.vmOperationNotStarted === true) error.credentialRefreshNotStarted = true + throw error + } + try { + return await this.recordReceipt(grant, receipt) + } catch (error) { + error.vmOperationUncertain = true + error.credentialRefreshUnknown = true + throw error + } + } + async recordReceipt(grantInput, value) { + const grant = normalizeRefreshGrant(grantInput), + receipt = normalizeRefreshReceipt(grant, value), + i = grant.intent + const scope = currentCredentialRefreshAttempt(this.db) + const input = { + operation: scope?.operation, + nodeId: i.node_id, + vmId: i.vm_id, + accountId: i.account_id, + capturedFence: i.fence, + } + return withTransaction(this.db, async () => { + const { state, epoch } = await this._state(input) + if (epoch !== i.epoch) throw refreshFailure('node_refresh_epoch_stale') + const row = ( + await this.db.query(`SELECT * FROM ${state.q}.refresh_grants WHERE command_id=$1 FOR UPDATE`, [i.command_id]) + ).rows[0] + if ( + !row || + row.revoked_at || + row.intent_digest !== grant.intentDigest || + !isDeepStrictEqual(normalizeRefreshIntent(row.intent), i) + ) + throw refreshFailure('node_refresh_conflict') + if (row.receipt && !isDeepStrictEqual(normalizeRefreshReceipt(grant, row.receipt), receipt)) + throw refreshFailure('node_refresh_conflict') + await this.db.query(`UPDATE ${state.q}.refresh_grants SET receipt=$2::jsonb WHERE command_id=$1`, [ + i.command_id, + JSON.stringify(receipt), + ]) + return receipt + }) + } +} diff --git a/src/lib/core/node-credential-refresh.mjs b/src/lib/core/node-credential-refresh.mjs new file mode 100644 index 000000000..e1fbe620c --- /dev/null +++ b/src/lib/core/node-credential-refresh.mjs @@ -0,0 +1,160 @@ +import crypto from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' +import { normalizeNodeFence } from './node-execution-proof.mjs' +export function refreshFailure(code = 'node_refresh_invalid') { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} +const keys = [ + 'action', + 'node_id', + 'vm_id', + 'allocation_id', + 'operation_id', + 'owner_id', + 'command_id', + 'account_id', + 'bg_task_key', + 'bg_execution_id', + 'bg_owner_id', + 'epoch', + 'fence', + 'expected_credential_generation', + 'manifest_revision', + 'manifest_digest', + 'preset_id', + 'policy_digest', + 'force', +] +const digest = (v) => typeof v === 'string' && /^[a-f0-9]{64}$/.test(v) +function exact(v, required, optional = []) { + if ( + !v || + typeof v !== 'object' || + Array.isArray(v) || + required.some((k) => !Object.hasOwn(v, k)) || + Object.keys(v).some((k) => !required.includes(k) && !optional.includes(k)) + ) + throw refreshFailure() +} +function decimal(v, positive = true) { + if ( + typeof v !== 'string' || + !/^(0|[1-9][0-9]{0,18})$/.test(v) || + BigInt(v) > 9223372036854775807n || + (positive && v === '0') + ) + throw refreshFailure() + return v +} +export function normalizeRefreshIntent(v) { + exact(v, keys) + for (const k of [ + 'node_id', + 'vm_id', + 'allocation_id', + 'operation_id', + 'owner_id', + 'command_id', + 'account_id', + 'bg_execution_id', + 'bg_owner_id', + ]) + if (typeof v[k] !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(v[k])) throw refreshFailure() + if ( + v.action !== 'credential-refresh' || + v.bg_task_key !== `oauth-refresh:claude:${v.account_id}` || + v.manifest_revision !== '1' || + v.preset_id !== 'claude-docker-v1' || + !digest(v.manifest_digest) || + !digest(v.policy_digest) || + typeof v.force !== 'boolean' + ) + throw refreshFailure() + decimal(v.epoch) + decimal(v.expected_credential_generation) + if (BigInt(v.expected_credential_generation) === 9223372036854775807n) throw refreshFailure() + const fence = normalizeNodeFence(v.fence) + if (fence.executor.kind !== 'container') throw refreshFailure() + return { ...v, fence } +} +export function refreshIntentDigest(v) { + const i = normalizeRefreshIntent(v), + e = i.fence.executor + return crypto + .createHash('sha256') + .update( + [ + 'slotgate-refresh-v1', + ...keys.slice(0, 12).map((k) => i[k]), + i.fence.generation, + i.fence.incarnation, + e.kind, + e.node_boot_id, + e.container_id, + e.docker_daemon_id, + e.started_at, + i.expected_credential_generation, + i.manifest_revision, + i.manifest_digest, + i.preset_id, + i.policy_digest, + i.force ? '1' : '0', + ].join('\0'), + ) + .digest('hex') +} +export function normalizeRefreshGrant(g) { + exact(g, ['intent', 'intentDigest']) + const intent = normalizeRefreshIntent(g.intent) + if (g.intentDigest !== refreshIntentDigest(intent)) throw refreshFailure() + return { intent, intentDigest: g.intentDigest } +} +export function normalizeRefreshReceipt(grant, value) { + const g = normalizeRefreshGrant(grant) + exact(value, ['intent', 'intent_digest', 'finished', 'ok', 'status', 'refreshed', 'credential'], ['error_code']) + const intent = normalizeRefreshIntent(value.intent) + if ( + !isDeepStrictEqual(intent, g.intent) || + value.intent_digest !== g.intentDigest || + value.finished !== true || + typeof value.ok !== 'boolean' || + typeof value.refreshed !== 'boolean' || + !Number.isInteger(value.status) + ) + throw refreshFailure() + const c = value.credential + exact( + c, + ['generation', 'expires_at', 'account_uuid', 'has_access', 'has_refresh', 'needs_refresh'], + ['org_uuid', 'email'], + ) + decimal(c.generation) + decimal(c.expires_at, false) + if ( + c.account_uuid !== intent.account_id || + ['has_access', 'has_refresh', 'needs_refresh'].some((k) => typeof c[k] !== 'boolean') + ) + throw refreshFailure() + for (const k of ['org_uuid', 'email']) + if (c[k] !== undefined && (typeof c[k] !== 'string' || c[k].length > 512 || /[\x00-\x1f\x7f]/.test(c[k]))) + throw refreshFailure() + if (value.ok) { + if ( + value.status !== 200 || + value.error_code !== undefined || + !c.has_access || + (value.refreshed + ? BigInt(c.generation) <= BigInt(intent.expected_credential_generation) + : c.generation !== intent.expected_credential_generation) + ) + throw refreshFailure() + } else if ( + value.status < 400 || + value.status > 599 || + value.error_code !== 'credential_refresh_rejected' || + value.refreshed || + c.generation !== intent.expected_credential_generation + ) + throw refreshFailure() + return { ...value, intent, credential: { ...c } } +} diff --git a/src/lib/db/node-credential-refresh-authority.mjs b/src/lib/db/node-credential-refresh-authority.mjs new file mode 100644 index 000000000..a5ccefa93 --- /dev/null +++ b/src/lib/db/node-credential-refresh-authority.mjs @@ -0,0 +1,86 @@ +import { quoteIdentifier, withTransaction } from './database.mjs' +import { coordinationSchemaFor } from './global-maintenance.mjs' +import { nodeCommandAuthorityState, nodeAuthorityError } from './node-management-authority.mjs' +export async function nodeRefreshAuthorityState(db) { + const base = await nodeCommandAuthorityState(db) + if (!base.enabled) return { enabled: false, schema: base.schema } + const coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + try { + const r = ( + await db.query( + `SELECT EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass($1) AND attname='refresh_protocol_version' AND NOT attisdropped) AS meta,EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass($2) AND attname='refresh_protocol_version' AND NOT attisdropped) AS marker,to_regclass($3) IS NOT NULL AS grants,to_regclass($4) IS NOT NULL AS view,to_regprocedure($5) IS NOT NULL AS current_function`, + [ + `${base.q}.meta`, + `${coord}.node_authority_features`, + `${base.q}.refresh_grants`, + `${base.q}.authorized_refreshes`, + `${base.q}.refresh_current(text)`, + ], + ) + ).rows[0] + if (Object.values(r).every((v) => v === false)) return { enabled: false, schema: base.schema } + if (!Object.values(r).every((v) => v === true)) throw new Error() + const v = ( + await db.query( + `SELECT m.refresh_protocol_version AS local,f.refresh_protocol_version AS marker FROM ${base.q}.meta m JOIN ${coord}.node_authority_features f ON f.application_schema=m.application_schema WHERE m.id=1 AND f.authority_schema=$1`, + [base.schema], + ) + ).rows[0] + if (v?.local !== 1 || v.marker !== 1) throw new Error() + return base + } catch { + throw nodeAuthorityError('node_refresh_authority_unavailable') + } +} +export async function initializeNodeRefreshAuthority(db) { + return withTransaction(db, async () => { + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:node-authority-bootstrap:${db.schema}`, + ]) + const base = await nodeCommandAuthorityState(db) + if (!base.enabled) throw nodeAuthorityError('node_refresh_authority_unavailable') + const existing = await nodeRefreshAuthorityState(db) + if (existing.enabled) return existing + const { q, schema } = base, + coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + const phase = (await db.query(`SELECT phase FROM ${coord}.control WHERE id=1 FOR SHARE`)).rows[0] + if (phase?.phase !== 'open') throw nodeAuthorityError('node_refresh_authority_unavailable') + await db.query(`ALTER TABLE ${q}.meta ADD COLUMN refresh_protocol_version integer DEFAULT 1 CHECK(refresh_protocol_version=1); + ALTER TABLE ${coord}.node_authority_features ADD COLUMN refresh_protocol_version integer DEFAULT 1 CHECK(refresh_protocol_version=1); + CREATE TABLE ${q}.refresh_grants(command_id text PRIMARY KEY,node_id text NOT NULL,vm_id text NOT NULL,allocation_id text NOT NULL REFERENCES ${q}.allocations(allocation_id),operation_id text NOT NULL,owner_id text NOT NULL,account_id text NOT NULL,bg_task_key text NOT NULL,bg_execution_id text NOT NULL,bg_owner_id text NOT NULL,epoch bigint NOT NULL CHECK(epoch>0),expected_generation text NOT NULL,intent_digest text NOT NULL,intent jsonb NOT NULL,receipt jsonb,created_at timestamptz NOT NULL DEFAULT clock_timestamp(),revoked_at timestamptz); + CREATE UNIQUE INDEX refresh_pending_generation ON ${q}.refresh_grants(node_id,vm_id,epoch,expected_generation) WHERE receipt IS NULL; + CREATE INDEX refresh_operation ON ${q}.refresh_grants(operation_id,owner_id) WHERE revoked_at IS NULL; + REVOKE ALL ON ${q}.refresh_grants FROM PUBLIC; + CREATE FUNCTION ${q}.refresh_current(p_command text) RETURNS boolean LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $body$ + DECLARE g record;allowed boolean; + BEGIN + IF NOT EXISTS(SELECT 1 FROM ${q}.reader_identity()) OR has_table_privilege(session_user,'${schema}.refresh_grants','INSERT,UPDATE,DELETE,TRUNCATE') THEN RETURN false;END IF; + SELECT a.*,m.application_schema INTO g FROM ${q}.refresh_grants a CROSS JOIN ${q}.meta m + JOIN ${q}.reader_roles r ON r.role_name=session_user + JOIN ${coord}.control c ON c.id=1 + JOIN ${coord}.node_authority_features f ON f.application_schema=m.application_schema + WHERE a.command_id=p_command AND a.node_id=r.node_id AND a.revoked_at IS NULL AND a.receipt IS NULL + AND m.id=1 AND m.protocol_version=1 AND m.command_protocol_version=1 AND f.protocol_version=1 AND f.command_protocol_version=1 AND m.refresh_protocol_version=1 AND f.refresh_protocol_version=1 AND f.authority_schema='${schema}' + AND c.application_schema=m.application_schema AND c.phase='open' AND c.epoch=a.epoch + AND a.intent->>'action'='credential-refresh' AND a.intent->>'epoch'=a.epoch::text + AND a.intent->>'node_id'=a.node_id AND a.intent->>'vm_id'=a.vm_id AND a.intent->>'account_id'=a.account_id + AND a.intent->>'operation_id'=a.operation_id AND a.intent->>'owner_id'=a.owner_id + AND a.intent->>'command_id'=a.command_id AND a.intent->>'allocation_id'=a.allocation_id + AND a.intent->>'bg_task_key'=a.bg_task_key AND a.intent->>'bg_execution_id'=a.bg_execution_id AND a.intent->>'bg_owner_id'=a.bg_owner_id + AND a.intent->>'expected_credential_generation'=a.expected_generation + AND EXISTS(SELECT 1 FROM ${q}.command_grants l WHERE l.allocation_id=a.allocation_id AND l.action='launch' AND l.epoch=a.epoch AND l.node_id=a.node_id AND l.vm_id=a.vm_id + AND l.receipt->>'intent_digest'=l.intent_digest AND l.receipt->'fence'=a.intent->'fence' + AND l.receipt->>'manifest_digest'=a.intent->>'manifest_digest' + AND l.intent->>'policy_digest'=a.intent->>'policy_digest'); + IF NOT FOUND THEN RETURN false;END IF; + EXECUTE format('SELECT EXISTS(SELECT 1 FROM %I.vm_operations WHERE id=$1 AND owner_id=$2 AND vm_id=$3 AND account_id=$4 AND kind=''credential_refresh'' AND released_at IS NULL AND uncertain_at IS NULL) AND EXISTS(SELECT 1 FROM %I.background_task_runs WHERE task_key=$5 AND execution_id=$6 AND owner_id=$7 AND finished_at IS NULL AND unknown_at IS NULL)',g.application_schema,g.application_schema) + INTO allowed USING g.operation_id,g.owner_id,g.vm_id,g.account_id,g.bg_task_key,g.bg_execution_id,g.bg_owner_id; + RETURN allowed; + EXCEPTION WHEN undefined_table OR invalid_schema_name THEN RETURN false; + END $body$; + REVOKE ALL ON FUNCTION ${q}.refresh_current(text) FROM PUBLIC; + CREATE VIEW ${q}.authorized_refreshes WITH(security_barrier=true) AS SELECT command_id,node_id,vm_id,allocation_id,operation_id,owner_id,account_id,bg_task_key,bg_execution_id,bg_owner_id,epoch,expected_generation,intent_digest FROM ${q}.refresh_grants WHERE ${q}.refresh_current(command_id); + REVOKE ALL ON ${q}.authorized_refreshes FROM PUBLIC`) + return nodeRefreshAuthorityState(db) + }) +} diff --git a/src/lib/db/node-management-authority.mjs b/src/lib/db/node-management-authority.mjs index 574346d48..5a85d0dea 100644 --- a/src/lib/db/node-management-authority.mjs +++ b/src/lib/db/node-management-authority.mjs @@ -1,6 +1,7 @@ import crypto from 'node:crypto' import { quoteIdentifier, withTransaction } from './database.mjs' import { coordinationSchemaFor } from './global-maintenance.mjs' +import { nodeRefreshAuthorityState } from './node-credential-refresh-authority.mjs' export function nodeAuthoritySchemaFor(applicationSchema) { if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(applicationSchema)) throw new Error('invalid database schema') @@ -275,11 +276,13 @@ export async function revokeNodeOperationGrants(db, { operationId, ownerId }) { const state = await nodeAuthorityState(db) if (!state.enabled) return const commands = await nodeCommandAuthorityState(db) + const refreshes = await nodeRefreshAuthorityState(db) const rows = ( await db.query( `SELECT DISTINCT node_id,vm_id FROM ( SELECT node_id,vm_id FROM ${state.q}.allocations WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL ${commands.enabled ? `UNION ALL SELECT node_id,vm_id FROM ${state.q}.command_grants WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL` : ''} + ${refreshes.enabled ? `UNION ALL SELECT node_id,vm_id FROM ${state.q}.refresh_grants WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL` : ''} ) grants ORDER BY node_id,vm_id`, [operationId, ownerId], ) @@ -296,4 +299,10 @@ export async function revokeNodeOperationGrants(db, { operationId, ownerId }) { WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL`, [operationId, ownerId], ) + if (refreshes.enabled) + await db.query( + `UPDATE ${state.q}.refresh_grants SET revoked_at=clock_timestamp() + WHERE operation_id=$1 AND owner_id=$2 AND revoked_at IS NULL`, + [operationId, ownerId], + ) } diff --git a/src/lib/oauth/shared-credential-refresh.mjs b/src/lib/oauth/shared-credential-refresh.mjs index cc493c81e..356c9f463 100644 --- a/src/lib/oauth/shared-credential-refresh.mjs +++ b/src/lib/oauth/shared-credential-refresh.mjs @@ -5,6 +5,18 @@ import { getSharedBackgroundTasks } from '../core/shared-background-tasks.mjs' import { SharedVmOperations, currentVmOperationIds } from '../vm/shared-vm-operations.mjs' const services = new WeakMap() +const refreshAttempts = new AsyncLocalStorage() +export function currentCredentialRefreshAttempt(db) { + const context = refreshAttempts.getStore() + if ( + !context?.active || + context.operation.uncertain || + context.db.schema !== db.schema || + context.db.connectionString !== db.connectionString + ) + return null + return context +} const admissionContext = new AsyncLocalStorage() export async function withRefreshAdmission(db, admission, callback) { @@ -149,7 +161,12 @@ export class SharedCredentialRefresh { let outcome = { finished: true, value: { ok: true } } let next = fresh if (fresh.generation === expectedGeneration) { - outcome = await refresh(fresh) + const attempt = { db: this.db, operation, claim, accountId, vmId, active: true } + try { + outcome = await refreshAttempts.run(attempt, () => refresh(fresh, { operation, claim })) + } finally { + attempt.active = false + } if (outcome?.finished !== true) { state.unknown = true return { @@ -190,7 +207,10 @@ export class SharedCredentialRefresh { throw failure('credential_refresh_completion_unconfirmed') return result } catch (error) { - if (acquired) { + if (acquired && error.vmOperationNotStarted === true && error.credentialRefreshNotStarted === true) { + if (!(await claim.complete({ sharedResult: { ok: false, code: 'refresh_not_started' } })).completed) + throw failure('credential_refresh_completion_unconfirmed') + } else if (acquired) { state.unknown = true await claim.markUnknown() } else { diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index f6ff9aab2..641633113 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -1141,14 +1141,41 @@ export async function workerHealth(exec, { timeoutMs = 3000, signal } = {}) { export async function ensureWorkerCredential(exec, { force = false, ...refreshOptions } = {}) { requireNodeCapability(exec) - if (exec?.slotgate != null) + if (exec?.slotgate != null) { + const health = await workerHealth(exec) + const credential = health?.credential + if ( + health.ok !== true || + !credential || + typeof credential.generation !== 'string' || + credential.account_uuid !== (exec.accountId || exec.nodeIdentity?.credential?.account_uuid) + ) + return { + ok: false, + status: 503, + refreshed: false, + credentialRefreshNotStarted: true, + error: { code: 'slotgate_route_unavailable', message: 'Node credential refresh is unavailable' }, + } + if (!force && credential.has_access === true && credential.needs_refresh === false) + return { ok: true, status: 200, refreshed: false, refresh_class: 'already_fresh', credential } + const value = await runSlotOauth(exec, 'refresh', { + ...refreshOptions, + force, + expectedGeneration: + refreshOptions.expectedGeneration === undefined ? credential.generation : refreshOptions.expectedGeneration, + }) return { - ok: false, - status: 503, - refreshed: false, - credentialRefreshNotStarted: true, - error: { code: 'slotgate_route_unavailable', message: 'Node credential refresh is not available' }, + ok: value.ok === true, + status: value.status, + refreshed: value.body?.refreshed === true, + credential: value.body?.credential, + ...(value.body?.error ? { error: value.body.error } : {}), + ...(value.credentialRefreshUnknown ? { credentialRefreshUnknown: true } : {}), + ...(value.credentialRefreshNotStarted ? { credentialRefreshNotStarted: true } : {}), + attemptFinished: value.attemptFinished, } + } if (isCrsMock()) { return { ok: true, diff --git a/src/lib/transport/slot-oauth.mjs b/src/lib/transport/slot-oauth.mjs index 679c95ea5..32f303983 100644 --- a/src/lib/transport/slot-oauth.mjs +++ b/src/lib/transport/slot-oauth.mjs @@ -1,9 +1,12 @@ +import { randomUUID } from 'node:crypto' +import { SlotgateClient } from './slotgate-client.mjs' +import { NodeCredentialRefreshGrants } from '../cluster/node-credential-refresh-grants.mjs' import { spawn } from 'node:child_process' import path from 'node:path' import { slotHost } from '../vm/slot-host.mjs' import { slotExecArgv } from '../vm/slot-exec.mjs' import { getDb, getActivityCoordinator } from '../db/database.mjs' -import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { isKubernetesMode, isKubernetesHaMode } from '../core/control-plane-mode.mjs' import { getSharedBackgroundTasks } from '../core/shared-background-tasks.mjs' import { SharedVmOperations } from '../vm/shared-vm-operations.mjs' import { flushVmDbSync } from '../vm/vm-db-sync.mjs' @@ -101,11 +104,125 @@ function lastJsonLine(stdout) { } export function runSlotOauth(exec, op, options = {}) { + if (exec?.slotgate != null) { + const db = options.db || getDb() + return getSharedBackgroundTasks(db).track(runNodeSlotOauth(db, exec, op, options)) + } + if (isKubernetesHaMode()) + throw Object.assign(new Error('Node credential capability required'), { + code: 'slotgate_capability_required', + credentialRefreshNotStarted: true, + vmOperationNotStarted: true, + }) if (!isKubernetesMode() || process.env.KIN_CRS_MOCK === '1') return runSlotOauthRaw(exec, op, options) const db = options.db || getDb() return getSharedBackgroundTasks(db).track(runSharedSlotOauth(db, exec, op, options)) } +async function runNodeSlotOauth(db, exec, op, options) { + if (op !== 'refresh') + return { + ok: false, + status: 503, + body: { error: { code: 'slotgate_route_unavailable' } }, + headers: {}, + attemptFinished: true, + credentialRefreshNotStarted: true, + via: 'slotgate', + } + const client = new SlotgateClient(exec.slotgate) + const vmId = client.nodeBinding.vmId + const accountId = + options.accountId || options.admission?.accountId || exec.accountId || exec.nodeIdentity?.credential?.account_uuid + const readCurrent = async () => { + const status = await client.health() + const credential = status.credential + if ( + status.ok !== true || + !credential || + credential.account_uuid !== accountId || + typeof credential.generation !== 'string' + ) + throw Object.assign(new Error('Node credential identity unavailable'), { + code: 'slotgate_identity_unavailable', + credentialRefreshNotStarted: true, + vmOperationNotStarted: true, + }) + return { credential, generation: credential.generation } + } + const current = await readCurrent() + const grants = new NodeCredentialRefreshGrants(db) + const outcome = await getSharedCredentialRefresh(db).run( + { + platform: 'claude', + vmId, + accountId, + expectedGeneration: options.expectedGeneration === undefined ? current.generation : options.expectedGeneration, + admission: options.admission || currentRefreshAdmission(db), + }, + { + readCurrent, + refresh: async (_old, { operation }) => { + const receipt = await grants.refresh( + { + operation, + nodeId: client.nodeBinding.nodeId, + vmId, + accountId, + commandId: randomUUID(), + capturedFence: client.nodeBinding.fence, + expectedGeneration: _old.generation, + force: options.force === true, + }, + client.link, + ) + return { + finished: true, + value: { + ok: receipt.ok, + status: receipt.status, + body: { + refreshed: receipt.refreshed, + credential: receipt.credential, + ...(!receipt.ok ? { error: { code: receipt.error_code } } : {}), + }, + headers: {}, + attemptFinished: true, + via: 'slotgate', + }, + } + }, + publish: async (_old, value) => { + const next = await readCurrent() + if (next.generation !== value.body.credential.generation) + throw Object.assign(new Error('Node credential publication unconfirmed'), { + code: 'credential_projection_stale', + }) + return next + }, + }, + ) + if (outcome.unknown) + return { + ...(outcome.value || failure('credential_refresh_unknown')), + attemptFinished: false, + credentialRefreshUnknown: true, + } + if (!outcome.shared) return outcome.value + return { + ok: outcome.value?.ok === true, + status: outcome.value?.ok ? 200 : 400, + body: { + refreshed: false, + credential: outcome.current?.credential, + ...(!outcome.value?.ok ? { error: { code: outcome.value?.error || 'credential_refresh_rejected' } } : {}), + }, + headers: {}, + attemptFinished: true, + via: 'slotgate', + } +} + async function runSharedSlotOauth(db, exec, op, options) { const readCurrent = async () => { const credential = readWorkerCredentialFile(exec.homeDir) diff --git a/src/lib/transport/slotgate-management-client.mjs b/src/lib/transport/slotgate-management-client.mjs index 96a383425..f090f3a75 100644 --- a/src/lib/transport/slotgate-management-client.mjs +++ b/src/lib/transport/slotgate-management-client.mjs @@ -1,4 +1,5 @@ import tls from 'node:tls' +import { normalizeRefreshGrant, normalizeRefreshReceipt } from '../core/node-credential-refresh.mjs' import net from 'node:net' import { isDeepStrictEqual } from 'node:util' import crypto from 'node:crypto' @@ -91,7 +92,9 @@ export function allocateNodeSlot({ link, grant }) { socket.once('secureConnect', () => { const pin = socket.getPeerCertificate()?.fingerprint256?.replaceAll(':', '').toLowerCase() if (pin !== link.certSha256) return finish(denied()) - socket.write(`KSG2 ${link.token}\n${JSON.stringify(request)}\n`) + socket.write( + `KSG2 ${link.token}\n${JSON.stringify(request, (key, value) => (key === 'generation' ? JSON.rawJSON(String(value)) : value))}\n`, + ) }) socket.on('data', (chunk) => { buffer = Buffer.concat([buffer, chunk]) @@ -120,7 +123,12 @@ function managementInvalid() { function managementFailure(action, sent) { return Object.assign(new Error('node management acknowledgement unavailable'), { - code: action === 'publish-initial' ? 'node_publication_unconfirmed' : 'node_launch_unconfirmed', + code: + action === 'credential-refresh' + ? 'node_refresh_unconfirmed' + : action === 'publish-initial' + ? 'node_publication_unconfirmed' + : 'node_launch_unconfirmed', ...(sent ? { vmOperationUncertain: true } : { vmOperationNotStarted: true }), }) } @@ -152,7 +160,8 @@ async function commandNodeSlot({ link: linkInput, grant: grantInput, payload: pa throw managementInvalid() payload = Buffer.from(payloadInput) } - grant = normalizeNodeManagementGrant(grantInput) + grant = + action === 'credential-refresh' ? normalizeRefreshGrant(grantInput) : normalizeNodeManagementGrant(grantInput) if ( grant.intent.action !== action || (payload && crypto.createHash('sha256').update(payload).digest('hex') !== grant.intent.payload_digest) @@ -165,7 +174,7 @@ async function commandNodeSlot({ link: linkInput, grant: grantInput, payload: pa const request = { op: action, vm_id: grant.intent.vm_id, - management: grant.intent, + [action === 'credential-refresh' ? 'refresh' : 'management']: grant.intent, ...(payload ? { content_length: payload.length } : {}), } return new Promise((resolve, reject) => { @@ -187,14 +196,16 @@ async function commandNodeSlot({ link: linkInput, grant: grantInput, payload: pa else resolve(value) } const failed = () => finish(managementFailure(action, sent)) - const timer = setTimeout(failed, 8000) + const timer = setTimeout(failed, action === 'credential-refresh' ? 45000 : 8000) socket.once('error', failed) socket.once('close', failed) socket.once('secureConnect', () => { const pin = socket.getPeerCertificate()?.fingerprint256?.replaceAll(':', '').toLowerCase() if (pin !== link.certSha256) return failed() sent = true - socket.write(`KSG2 ${link.token}\n${JSON.stringify(request)}\n`) + socket.write( + `KSG2 ${link.token}\n${JSON.stringify(request, (key, value) => (key === 'generation' ? JSON.rawJSON(String(value)) : value))}\n`, + ) if (payload) socket.write(payload) }) socket.on('data', (chunk) => { @@ -206,10 +217,16 @@ async function commandNodeSlot({ link: linkInput, grant: grantInput, payload: pa const reply = JSON.parse(buffer.subarray(0, newline).toString(), (key, value, context) => key === 'generation' && typeof value === 'number' ? context.source : value, ) - const field = action === 'publish-initial' ? 'publication' : 'launch' + const field = + action === 'credential-refresh' ? 'refresh' : action === 'publish-initial' ? 'publication' : 'launch' exact(reply, ['ok', field]) if (reply.ok !== true) return failed() - finish(null, normalizeNodeManagementReceipt(grant, reply[field])) + finish( + null, + action === 'credential-refresh' + ? normalizeRefreshReceipt(grant, reply[field]) + : normalizeNodeManagementReceipt(grant, reply[field]), + ) } catch { failed() } @@ -223,3 +240,7 @@ export function publishNodeSlotInitial(input) { export function launchNodeSlot(input) { return commandNodeSlot(input, 'launch') } + +export function refreshNodeSlotCredential(input) { + return commandNodeSlot(input, 'credential-refresh') +} diff --git a/test/unit/node-credential-refresh-wire.test.mjs b/test/unit/node-credential-refresh-wire.test.mjs new file mode 100644 index 000000000..440aaa311 --- /dev/null +++ b/test/unit/node-credential-refresh-wire.test.mjs @@ -0,0 +1,272 @@ +import { registerActivityCoordinator } from '../../src/lib/db/database.mjs' +import { withDatabaseActivities } from '../maintenance-fixture.mjs' +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import tls from 'node:tls' +import pg from 'pg' +import { NodeCredentialRefreshGrants } from '../../src/lib/cluster/node-credential-refresh-grants.mjs' +import { SharedCredentialRefresh } from '../../src/lib/oauth/shared-credential-refresh.mjs' +import { runSlotOauth } from '../../src/lib/transport/slot-oauth.mjs' +import * as management from '../../src/lib/transport/slotgate-management-client.mjs' +import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' + +const nodeId = 'command-node' +const vmId = 'command-vm' +const allocationId = 'command-allocation' +const payload = Buffer.from( + JSON.stringify({ + credential: { + type: 'oauth', + access_token: 'synthetic-command-access', + refresh_token: 'synthetic-command-refresh', + account_uuid: 'synthetic-account', + expires_at: Date.now() + 3600000, + org_uuid: 'synthetic-org', + email: 'synthetic@example.invalid', + }, + }), +) + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const coordinator = new GlobalMaintenanceCoordinator(db) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'nmc-')) + let node + let readerRole + let engine + let closed = false + const service = new NodeManagementGrants(db, { resolveNodeLink: async () => node.link }) + const close = async () => { + if (closed) return + closed = true + await node?.close() + if (engine) { + engine.closeAllConnections() + await new Promise((resolve) => engine.close(resolve)) + } + await coordinator.stop() + await peer.close() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + if (readerRole) + await cleanup.query(`DROP OWNED BY ${quoteIdentifier(readerRole)}; DROP ROLE ${quoteIdentifier(readerRole)}`) + await cleanup.query( + `DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE; DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`, + ) + } finally { + await cleanup.end() + fs.rmSync(root, { recursive: true, force: true }) + } + } + t.after(close) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + await service.initialize() + await service.initializeCommands() + const refreshGrants = new NodeCredentialRefreshGrants(db) + await refreshGrants.initialize() + readerRole = 'command_wire_' + crypto.randomBytes(8).toString('hex') + const password = crypto.randomBytes(24).toString('hex') + await db.pool.query( + `CREATE ROLE ${quoteIdentifier(readerRole)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, + ) + await db.pool.query(service.readerRoleSql({ role: readerRole, nodeId })) + await db.pool.query(service.commandReaderRoleSql({ role: readerRole })) + await db.pool.query(refreshGrants.readerRoleSql({ role: readerRole })) + const url = new URL(db.connectionString) + url.username = readerRole + url.password = password + const policy = { + image_id: 'sha256:' + 'a'.repeat(64), + network_id: 'none', + uid: process.getuid() || 1000, + gid: process.getgid() || 1000, + } + const policyDigest = crypto + .createHash('sha256') + .update( + [ + 'slotgate-policy-v1', + 'claude-docker-v1', + policy.image_id, + policy.network_id, + String(policy.uid), + String(policy.gid), + ].join('\0'), + ) + .digest('hex') + const calls = [] + let spec + const containerId = 'b'.repeat(64) + engine = http.createServer(async (req, res) => { + calls.push(`${req.method} ${req.url}`) + const parsed = new URL(req.url, 'http://synthetic-engine') + const send = (body, status = 200) => { + res.writeHead(status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(body)) + } + if (parsed.pathname === '/info') return send({ ID: 'synthetic-daemon' }) + if (parsed.pathname.startsWith('/images/')) + return send({ Id: policy.image_id, Config: { Labels: { 'kin.slot.full': '1' } } }) + if (parsed.pathname === '/containers/create') { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + spec = JSON.parse(Buffer.concat(chunks).toString()) + return send({ Id: containerId }, 201) + } + if (parsed.pathname === `/containers/${containerId}/start`) return res.writeHead(204).end() + if (parsed.pathname === `/containers/${containerId}/json`) + return send({ + Id: containerId, + Image: policy.image_id, + Config: { Labels: spec.Labels }, + State: { Running: true, Restarting: false, StartedAt: '2026-10-22T00:00:00.000000001Z' }, + }) + res.writeHead(404).end() + }) + const socket = path.join(root, 'engine.sock') + await new Promise((resolve, reject) => { + engine.once('error', reject) + engine.listen(socket, resolve) + }) + node = await realAllocationGate( + t, + { connection_string: url.toString(), authority_schema: service.schema, node_id: nodeId }, + { launch: policy, dockerSocket: socket }, + ) + const run = (kind, callback) => new SharedVmOperations(db).run(vmId, callback, { kind }) + const allocated = await run('node_allocate', (operation) => + service.allocate({ operation, nodeId, vmId, allocationId }), + ) + const input = (operation, patch = {}) => ({ + operation, + nodeId, + vmId, + allocationId, + commandId: 'wire-publish', + capturedFence: allocated.fence, + presetId: 'claude-docker-v1', + policyDigest, + payload, + ...patch, + }) + return { + db, + peer, + coordinator, + service, + run, + input, + node, + calls, + root, + close, + allocated, + q: quoteIdentifier(service.schema), + refreshGrants, + } +} + +function nodeCase(name, callback) { + test(name, async (t) => { + const fx = await fixture(t) + try { + registerActivityCoordinator(fx.db, fx.coordinator) + await withDatabaseActivities([fx.db], () => callback(fx, t)) + } finally { + await fx.close() + } + }) +} + +nodeCase('real PG reader and pinned TLS authorize node-owned refresh without Pod credential files', async (fx, t) => { + const published = await fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))) + const launched = await fx.run('node_launch', (operation) => + fx.service.launch(fx.input(operation, { commandId: 'refresh-launch', manifestDigest: published.manifest_digest })), + ) + const socket = path.join( + fx.node.root, + 'runtime', + 'vms', + vmId, + 'incarnations', + launched.fence.incarnation, + 'run', + 'kernel.sock', + ) + const kernel = http.createServer((req, res) => { + assert.equal(req.url, '/internal/health') + res.writeHead(200, { 'content-type': 'application/json' }) + res.end('{"ok":true}') + }) + await new Promise((resolve, reject) => { + kernel.once('error', reject) + kernel.listen(socket, resolve) + }) + try { + const mode = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + try { + const slotgate = { + trusted: true, + pgBindingReady: true, + nodeId, + vmId, + fence: launched.fence, + link: fx.node.link, + manifest: { revision: '1', digest: published.manifest_digest }, + } + const value = await runSlotOauth( + { vmId, accountId: 'synthetic-account', homeDir: '/must-not-read-pod-credentials', slotgate }, + 'refresh', + { db: fx.db, force: false }, + ) + assert.equal(value.ok, true) + assert.equal(value.attemptFinished, true) + assert.equal(value.body.refreshed, false) + assert.equal(value.body.credential.generation, '1') + assert.equal(value.body.credential.has_refresh, true) + const rows = (await fx.peer.query(`SELECT receipt,revoked_at FROM ${fx.q}.refresh_grants`)).rows + assert.equal(rows.length, 1) + assert.equal(rows[0].receipt.finished, true) + assert.ok(rows[0].revoked_at) + const guest = fs.readFileSync( + path.join( + fx.node.root, + 'runtime', + 'vms', + vmId, + 'incarnations', + launched.fence.incarnation, + 'claude', + 'credentials.json', + ), + 'utf8', + ) + assert.equal(guest.includes('synthetic-command-refresh'), false) + const canonical = fs.readFileSync( + path.join(fx.node.root, 'authority', 'credentials', vmId, launched.fence.incarnation, 'credentials.json'), + 'utf8', + ) + assert.equal(canonical.includes('synthetic-command-refresh'), true) + assert.equal(JSON.stringify(rows).includes('synthetic-command-access'), false) + } finally { + if (mode === undefined) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = mode + } + } finally { + kernel.closeAllConnections() + await new Promise((resolve) => kernel.close(resolve)) + } +}) diff --git a/test/unit/node-credential-refresh.test.mjs b/test/unit/node-credential-refresh.test.mjs index 7a426a2a4..cbea21319 100644 --- a/test/unit/node-credential-refresh.test.mjs +++ b/test/unit/node-credential-refresh.test.mjs @@ -2,69 +2,294 @@ import test from 'node:test' import assert from 'node:assert/strict' import crypto from 'node:crypto' import pg from 'pg' -import {getDb,createDatabase,quoteIdentifier} from '../../src/lib/db/database.mjs' -import {GlobalMaintenanceCoordinator} from '../../src/lib/db/global-maintenance.mjs' -import {NodeManagementGrants} from '../../src/lib/cluster/node-management-grants.mjs' -import {SharedVmOperations} from '../../src/lib/vm/shared-vm-operations.mjs' -import {SharedCredentialRefresh} from '../../src/lib/oauth/shared-credential-refresh.mjs' +import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' +import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' +import { SharedCredentialRefresh } from '../../src/lib/oauth/shared-credential-refresh.mjs' import * as refreshModule from '../../src/lib/cluster/node-credential-refresh-grants.mjs' -const nodeId='refresh-node',vmId='refresh-vm',accountId='refresh-account',allocationId='refresh-allocation' -const vacant={generation:'1',incarnation:'a'.repeat(32),executor:{kind:'',node_boot_id:''}} -const fence={generation:'1',incarnation:vacant.incarnation,executor:{kind:'container',node_boot_id:'synthetic-boot',container_id:'b'.repeat(64),docker_daemon_id:'synthetic-daemon',started_at:'2026-10-22T00:00:00.000000001Z'}} -const summary=(generation)=>({generation,expires_at:'2000000000000',account_uuid:accountId,has_access:true,has_refresh:true,needs_refresh:false}) -async function fixture(t){ - const db=getDb(),peer=await createDatabase({connectionString:db.connectionString,schema:db.schema}) - const maintenance=new GlobalMaintenanceCoordinator(db),base=new NodeManagementGrants(db) - const grants=new refreshModule.NodeCredentialRefreshGrants(db),q=quoteIdentifier(base.schema) - const readers=[],roles=[];let closed=false - const close=async()=>{if(closed)return;closed=true;await maintenance.stop();await peer.close();for(const c of readers)await c.end();const c=new pg.Client({connectionString:db.connectionString});await c.connect();try{for(const role of roles)await c.query(`DROP OWNED BY ${quoteIdentifier(role)};DROP ROLE ${quoteIdentifier(role)}`);await c.query(`DROP SCHEMA IF EXISTS ${q} CASCADE;DROP SCHEMA IF EXISTS ${quoteIdentifier(maintenance.schema)} CASCADE`)}finally{await c.end()}} - t.after(close);await maintenance.start();await maintenance.acknowledgeEpoch(1);await base.initialize();await base.initializeCommands() - const run=(kind,fn)=>new SharedVmOperations(db).run(vmId,fn,{kind,accountId}) - await run('node_allocate',async operation=>{const g=await base.issueAllocation({operation,nodeId,vmId,allocationId});await db.query(`UPDATE ${q}.allocations SET receipt=$2::jsonb WHERE allocation_id=$1`,[allocationId,JSON.stringify({node_id:nodeId,vm_id:vmId,allocation_id:allocationId,operation_id:g.operationId,owner_id:g.ownerId,epoch:g.epoch,intent_digest:g.intentDigest,fence:vacant})])}) - await run('node_publish_initial',async operation=>{const g=await base.issueInitialPublish({operation,nodeId,vmId,allocationId,commandId:'initial-publish',capturedFence:vacant,expectedManifestRevision:'0',revision:'1',payloadDigest:'c'.repeat(64),presetId:'claude-docker-v1',policyDigest:'d'.repeat(64)});await base.recordInitialPublishReceipt(g,{intent:g.intent,intent_digest:g.intentDigest,manifest_revision:'1',manifest_digest:'e'.repeat(64)})}) - await run('node_launch',async operation=>{const g=await base.issueLaunch({operation,nodeId,vmId,allocationId,commandId:'initial-launch',capturedFence:vacant,expectedManifestRevision:'1',revision:'1',payloadDigest:'',presetId:'claude-docker-v1',policyDigest:'d'.repeat(64),manifestDigest:'e'.repeat(64)});await base.recordLaunchReceipt(g,{intent:g.intent,intent_digest:g.intentDigest,manifest_revision:'1',manifest_digest:'e'.repeat(64),fence})}) - let current={generation:'1',credential:summary('1')} - const refresh=new SharedCredentialRefresh(db) - const input=(operation,patch={})=>({operation,nodeId,vmId,accountId,commandId:crypto.randomUUID(),capturedFence:fence,expectedGeneration:current.generation,force:true,...patch}) - const attempt=(callback)=>refresh.run({platform:'claude',vmId,accountId,expectedGeneration:current.generation},{readCurrent:async()=>current,refresh:callback,publish:async(_old,value)=>{current={generation:value.generation,credential:summary(value.generation)};return current}}) - const receipt=(g,generation='2')=>({intent:g.intent,intent_digest:g.intentDigest,finished:true,ok:true,status:200,refreshed:true,credential:summary(generation)}) - const reader=async()=>{const role='refresh_reader_'+crypto.randomBytes(8).toString('hex'),password=crypto.randomBytes(20).toString('hex');roles.push(role);await db.pool.query(`CREATE ROLE ${quoteIdentifier(role)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`);await db.pool.query(base.readerRoleSql({role,nodeId}));await db.pool.query(grants.readerRoleSql({role}));const url=new URL(db.connectionString);url.username=role;url.password=password;const c=new pg.Client({connectionString:url.toString()});await c.connect();readers.push(c);return c} - return{db,peer,base,grants,q,maintenance,run,input,attempt,receipt,reader,close} +const nodeId = 'refresh-node', + vmId = 'refresh-vm', + accountId = 'refresh-account', + allocationId = 'refresh-allocation' +const vacant = { generation: '1', incarnation: 'a'.repeat(32), executor: { kind: '', node_boot_id: '' } } +const fence = { + generation: '1', + incarnation: vacant.incarnation, + executor: { + kind: 'container', + node_boot_id: 'synthetic-boot', + container_id: 'b'.repeat(64), + docker_daemon_id: 'synthetic-daemon', + started_at: '2026-10-22T00:00:00.000000001Z', + }, } -function refreshCase(name,fn){test(name,async t=>{const f=await fixture(t);try{await fn(f,t)}finally{await f.close()}})} -refreshCase('refresh authority requires its independent explicit markers and creates no runtime DDL',async f=>{ - await assert.rejects(f.attempt(async(_current,{operation})=>{await f.grants.issue(f.input(operation));return{finished:true,value:{ok:false}}}),{code:'node_refresh_authority_unavailable'}) - assert.equal((await f.peer.query('SELECT to_regclass($1) AS relation',[`${f.base.schema}.refresh_grants`])).rows[0].relation,null) - await f.grants.initialize() - assert.equal((await f.peer.query(`SELECT refresh_protocol_version FROM ${f.q}.meta`)).rows[0].refresh_protocol_version,1) - await f.peer.query(`UPDATE ${quoteIdentifier(f.maintenance.schema)}.node_authority_features SET refresh_protocol_version=NULL`) - await assert.rejects(f.grants.state(),{code:'node_refresh_authority_unavailable'}) +const summary = (generation) => ({ + generation, + expires_at: '2000000000000', + account_uuid: accountId, + has_access: true, + has_refresh: true, + needs_refresh: false, }) -refreshCase('refresh grant requires exact live BG/VM/account scope and a canonical launch chain',async f=>{ - await f.grants.initialize() - await f.run('credential_refresh',async operation=>{await assert.rejects(f.grants.issue(f.input(operation)),{code:'node_refresh_scope_required'})}) - const reader=await f.reader();let command - const result=await f.attempt(async(_current,{operation})=>{ - await assert.rejects(f.grants.issue(f.input(operation,{accountId:'other-account'})),{code:'node_refresh_scope_required'}) - const g=await f.grants.issue(f.input(operation));command=g.intent.command_id - assert.equal((await reader.query(`SELECT command_id FROM ${f.q}.authorized_refreshes`)).rows[0].command_id,command) - await assert.rejects(reader.query(`SELECT * FROM ${f.q}.refresh_grants`),{code:'42501'}) - const stored=await f.grants.recordReceipt(g,f.receipt(g));assert.equal(stored.credential.generation,'2') - return{finished:true,value:{ok:true,generation:'2'}} - }) - assert.equal(result.unknown,false) - assert.equal((await reader.query(`SELECT count(*)::int AS n FROM ${f.q}.authorized_refreshes`)).rows[0].n,0) - const row=(await f.peer.query(`SELECT receipt FROM ${f.q}.refresh_grants WHERE command_id=$1`,[command])).rows[0] - assert.equal(row.receipt.credential.generation,'2');assert.equal(JSON.stringify(row).includes('access_token'),false) +async function fixture(t) { + const db = getDb(), + peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const maintenance = new GlobalMaintenanceCoordinator(db), + base = new NodeManagementGrants(db) + const grants = new refreshModule.NodeCredentialRefreshGrants(db), + q = quoteIdentifier(base.schema) + const readers = [], + roles = [] + let closed = false + const close = async () => { + if (closed) return + closed = true + await maintenance.stop() + await peer.close() + for (const c of readers) await c.end() + const c = new pg.Client({ connectionString: db.connectionString }) + await c.connect() + try { + for (const role of roles) + await c.query(`DROP OWNED BY ${quoteIdentifier(role)};DROP ROLE ${quoteIdentifier(role)}`) + await c.query( + `DROP SCHEMA IF EXISTS ${q} CASCADE;DROP SCHEMA IF EXISTS ${quoteIdentifier(maintenance.schema)} CASCADE`, + ) + } finally { + await c.end() + } + } + t.after(close) + await maintenance.start() + await maintenance.acknowledgeEpoch(1) + await base.initialize() + await base.initializeCommands() + const run = (kind, fn) => new SharedVmOperations(db).run(vmId, fn, { kind, accountId }) + await run('node_allocate', async (operation) => { + const g = await base.issueAllocation({ operation, nodeId, vmId, allocationId }) + await db.query(`UPDATE ${q}.allocations SET receipt=$2::jsonb WHERE allocation_id=$1`, [ + allocationId, + JSON.stringify({ + node_id: nodeId, + vm_id: vmId, + allocation_id: allocationId, + operation_id: g.operationId, + owner_id: g.ownerId, + epoch: g.epoch, + intent_digest: g.intentDigest, + fence: vacant, + }), + ]) + }) + await run('node_publish_initial', async (operation) => { + const g = await base.issueInitialPublish({ + operation, + nodeId, + vmId, + allocationId, + commandId: 'initial-publish', + capturedFence: vacant, + expectedManifestRevision: '0', + revision: '1', + payloadDigest: 'c'.repeat(64), + presetId: 'claude-docker-v1', + policyDigest: 'd'.repeat(64), + }) + await base.recordInitialPublishReceipt(g, { + intent: g.intent, + intent_digest: g.intentDigest, + manifest_revision: '1', + manifest_digest: 'e'.repeat(64), + }) + }) + await run('node_launch', async (operation) => { + const g = await base.issueLaunch({ + operation, + nodeId, + vmId, + allocationId, + commandId: 'initial-launch', + capturedFence: vacant, + expectedManifestRevision: '1', + revision: '1', + payloadDigest: '', + presetId: 'claude-docker-v1', + policyDigest: 'd'.repeat(64), + manifestDigest: 'e'.repeat(64), + }) + await base.recordLaunchReceipt(g, { + intent: g.intent, + intent_digest: g.intentDigest, + manifest_revision: '1', + manifest_digest: 'e'.repeat(64), + fence, + }) + }) + let current = { generation: '1', credential: summary('1') } + const refresh = new SharedCredentialRefresh(db) + const input = (operation, patch = {}) => ({ + operation, + nodeId, + vmId, + accountId, + commandId: crypto.randomUUID(), + capturedFence: fence, + expectedGeneration: current.generation, + force: true, + ...patch, + }) + const attempt = (callback) => + refresh.run( + { platform: 'claude', vmId, accountId, expectedGeneration: current.generation }, + { + readCurrent: async () => current, + refresh: callback, + publish: async (_old, value) => { + current = { generation: value.generation, credential: summary(value.generation) } + return current + }, + }, + ) + const receipt = (g, generation = '2') => ({ + intent: g.intent, + intent_digest: g.intentDigest, + finished: true, + ok: true, + status: 200, + refreshed: true, + credential: summary(generation), + }) + const reader = async () => { + const role = 'refresh_reader_' + crypto.randomBytes(8).toString('hex'), + password = crypto.randomBytes(20).toString('hex') + roles.push(role) + await db.pool.query( + `CREATE ROLE ${quoteIdentifier(role)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, + ) + await db.pool.query(base.readerRoleSql({ role, nodeId })) + await db.pool.query(grants.readerRoleSql({ role })) + const url = new URL(db.connectionString) + url.username = role + url.password = password + const c = new pg.Client({ connectionString: url.toString() }) + await c.connect() + readers.push(c) + return c + } + return { db, peer, base, grants, q, maintenance, run, input, attempt, receipt, reader, close } +} +function refreshCase(name, fn) { + test(name, async (t) => { + const f = await fixture(t) + try { + await fn(f, t) + } finally { + await f.close() + } + }) +} +refreshCase('refresh authority requires its independent explicit markers and creates no runtime DDL', async (f) => { + await assert.rejects( + f.attempt(async (_current, { operation }) => { + await f.grants.issue(f.input(operation)) + return { finished: true, value: { ok: false } } + }), + { code: 'node_refresh_authority_unavailable' }, + ) + assert.equal( + (await f.peer.query('SELECT to_regclass($1) AS relation', [`${f.base.schema}.refresh_grants`])).rows[0].relation, + null, + ) + await f.grants.initialize() + assert.equal( + (await f.peer.query(`SELECT refresh_protocol_version FROM ${f.q}.meta`)).rows[0].refresh_protocol_version, + 1, + ) + await f.peer.query( + `UPDATE ${quoteIdentifier(f.maintenance.schema)}.node_authority_features SET refresh_protocol_version=NULL`, + ) + await assert.rejects(f.grants.state(), { code: 'node_refresh_authority_unavailable' }) +}) +refreshCase('refresh grant requires exact live BG/VM/account scope and a canonical launch chain', async (f) => { + await f.grants.initialize() + await f.run('credential_refresh', async (operation) => { + await assert.rejects(f.grants.issue(f.input(operation)), { code: 'node_refresh_scope_required' }) + }) + const reader = await f.reader() + let command + const result = await f.attempt(async (_current, { operation }) => { + await assert.rejects(f.grants.issue(f.input(operation, { accountId: 'other-account' })), { + code: 'node_refresh_scope_required', + }) + const g = await f.grants.issue(f.input(operation)) + command = g.intent.command_id + await assert.rejects(new refreshModule.NodeCredentialRefreshGrants(f.peer).issue(f.input(operation)), { + code: 'node_refresh_conflict', + }) + await assert.rejects(f.grants.recordReceipt(g, { ...f.receipt(g), access_token: 'synthetic-must-never-persist' }), { + code: 'node_refresh_invalid', + }) + assert.equal((await reader.query(`SELECT command_id FROM ${f.q}.authorized_refreshes`)).rows[0].command_id, command) + await assert.rejects(reader.query(`SELECT * FROM ${f.q}.refresh_grants`), { code: '42501' }) + const stored = await f.grants.recordReceipt(g, f.receipt(g)) + assert.equal(stored.credential.generation, '2') + await assert.rejects(f.grants.issue(f.input(operation)), { code: 'node_refresh_generation_stale' }) + return { finished: true, value: { ok: true, generation: '2' } } + }) + assert.equal(result.unknown, false) + assert.equal((await reader.query(`SELECT count(*)::int AS n FROM ${f.q}.authorized_refreshes`)).rows[0].n, 0) + const row = (await f.peer.query(`SELECT receipt FROM ${f.q}.refresh_grants WHERE command_id=$1`, [command])).rows[0] + assert.equal(row.receipt.credential.generation, '2') + assert.equal(JSON.stringify(row).includes('access_token'), false) }) -refreshCase('unknown refresh cannot be reissued by another connection or a different command ID',async f=>{ - await f.grants.initialize();let id - const result=await f.attempt(async(_current,{operation})=>{const g=await f.grants.issue(f.input(operation));id=g.intent.command_id;return{finished:false,value:{ok:false}}}) - assert.equal(result.unknown,true) - const row=(await f.peer.query(`SELECT receipt,revoked_at FROM ${f.q}.refresh_grants WHERE command_id=$1`,[id])).rows[0] - assert.equal(row.receipt,null) - assert.ok((await f.peer.query('SELECT uncertain_at FROM vm_operations WHERE kind=$1',['credential_refresh'])).rows[0].uncertain_at) - const gate=await new SharedVmOperations(f.peer).tryAcquire({vmId,accountId,kind:'credential_refresh'}) - assert.equal(gate.ok,false);assert.equal(gate.code,'vm_operation_active') +refreshCase('unknown refresh cannot be reissued by another connection or a different command ID', async (f) => { + await f.grants.initialize() + let id + const result = await f.attempt(async (_current, { operation }) => { + const g = await f.grants.issue(f.input(operation)) + id = g.intent.command_id + return { finished: false, value: { ok: false } } + }) + assert.equal(result.unknown, true) + const row = (await f.peer.query(`SELECT receipt,revoked_at FROM ${f.q}.refresh_grants WHERE command_id=$1`, [id])) + .rows[0] + assert.equal(row.receipt, null) + assert.ok( + (await f.peer.query('SELECT uncertain_at FROM vm_operations WHERE kind=$1', ['credential_refresh'])).rows[0] + .uncertain_at, + ) + const gate = await new SharedVmOperations(f.peer).tryAcquire({ vmId, accountId, kind: 'credential_refresh' }) + assert.equal(gate.ok, false) + assert.equal(gate.code, 'vm_operation_active') +}) + +refreshCase('a failed PG receipt commit retains the exact background and VM claims', async (f) => { + await f.grants.initialize() + await f.db.query( + `CREATE FUNCTION reject_refresh_receipt() RETURNS trigger LANGUAGE plpgsql AS $body$ BEGIN RAISE EXCEPTION 'synthetic receipt commit failure' USING ERRCODE='P648R';END $body$;CREATE TRIGGER reject_refresh_receipt BEFORE UPDATE OF receipt ON ${f.q}.refresh_grants FOR EACH ROW EXECUTE FUNCTION reject_refresh_receipt()`, + ) + await assert.rejects( + f.attempt(async (_current, { operation }) => { + const g = await f.grants.issue(f.input(operation)) + await f.grants.recordReceipt(g, f.receipt(g)) + return { finished: true, value: { ok: true, generation: '2' } } + }), + { code: 'P648R' }, + ) + const op = (await f.peer.query("SELECT uncertain_at,released_at FROM vm_operations WHERE kind='credential_refresh'")) + .rows[0] + assert.ok(op.uncertain_at) + assert.equal(op.released_at, null) + const bg = ( + await f.peer.query( + "SELECT unknown_at,finished_at FROM background_task_runs WHERE task_key='oauth-refresh:claude:refresh-account'", + ) + ).rows[0] + assert.ok(bg.unknown_at) + assert.equal(bg.finished_at, null) + const g = (await f.peer.query(`SELECT receipt,revoked_at FROM ${f.q}.refresh_grants`)).rows[0] + assert.equal(g.receipt, null) + assert.ok(g.revoked_at) }) diff --git a/worker/internal/slotgate/ksg2.go b/worker/internal/slotgate/ksg2.go index 994873daa..9459596e4 100644 --- a/worker/internal/slotgate/ksg2.go +++ b/worker/internal/slotgate/ksg2.go @@ -28,6 +28,7 @@ var errGateTicketInvalid = errors.New("ticket_invalid") // commands continue with one parsed HTTP/1 request and an HTTP response after // OK\n. KSG1 raw splicing is never accepted when fenced mode is enabled. type gate2Request struct { + Refresh *RefreshIntent `json:"refresh,omitempty"` Op string `json:"op"` VMID string `json:"vm_id"` Fence Fence `json:"fence"` @@ -110,6 +111,10 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. io.WriteString(conn, "ERR bad_request\n") return } + if request.Op != "credential-refresh" && request.Refresh != nil { + io.WriteString(conn, "ERR bad_request\n") + return + } if request.Op != "publish-initial" && request.Op != "launch" && (request.Management != nil || request.ContentLength != 0) { io.WriteString(conn, "ERR bad_request\n") return @@ -128,6 +133,16 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply := map[string]any{"ok": false} var state AuthorityState switch request.Op { + case "credential-refresh": + if s.refresher == nil || request.Refresh == nil || request.Management != nil || request.Fence != (Fence{}) || request.CommandID != "" || request.Kind != "" || request.Ticket != "" || request.AllocationID != "" || request.OperationID != "" || request.OwnerID != "" || request.ContentLength != 0 || request.Refresh.VMID != request.VMID { + err = ErrManagementDenied + break + } + var receipt RefreshReceipt + receipt, err = s.refresher.Refresh(ctx, *request.Refresh) + if err == nil { + reply["refresh"] = receipt + } case "health": if err = s.currentBootFence(request.Fence); err != nil { break @@ -271,7 +286,7 @@ func (s *Server) handleFenced(ctx context.Context, conn net.Conn, reader *bufio. reply["error"] = gate2Code(err) } else { reply["ok"] = true - if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" && request.Op != "health" && request.Op != "recovery-status" && request.Op != "seal-not-started" { + if request.Op != "prepare" && request.Op != "command" && request.Op != "allocate" && request.Op != "publish-initial" && request.Op != "launch" && request.Op != "health" && request.Op != "recovery-status" && request.Op != "seal-not-started" && request.Op != "credential-refresh" { reply["state"] = publicGateState(state) } } diff --git a/worker/internal/slotgate/node_private_credential.go b/worker/internal/slotgate/node_private_credential.go index a371980db..a8f53067a 100644 --- a/worker/internal/slotgate/node_private_credential.go +++ b/worker/internal/slotgate/node_private_credential.go @@ -3,6 +3,7 @@ package slotgate import ( "bytes" "encoding/json" + "strconv" ) const nodePrivateCredentialAuthority = "node-private-v1" @@ -18,7 +19,8 @@ func validNodeCredentialState(s *nodeCredentialState) bool { return false } gen, err := credentialScalar(json.RawMessage(s.Generation)) - return err == nil && gen == s.Generation && gen != "0" + n, parseErr := strconv.ParseInt(gen, 10, 64) + return err == nil && parseErr == nil && n > 0 && gen == s.Generation } // Only the private canonical document contains the rotating token. The guest diff --git a/worker/internal/slotgate/node_refresh.go b/worker/internal/slotgate/node_refresh.go index cf267a2dc..bf0702008 100644 --- a/worker/internal/slotgate/node_refresh.go +++ b/worker/internal/slotgate/node_refresh.go @@ -75,6 +75,7 @@ type RefreshReceipt struct { } type RefreshAuthorizer func(context.Context, RefreshIntent, func() error) error type nodeCredentialRefresher struct { + boot string authority *Authority root string nodeID string @@ -97,7 +98,7 @@ func publicCredential(data []byte, identity credentialIdentity) (PublicCredentia return result, nil } func (r *nodeCredentialRefresher) Refresh(ctx context.Context, i RefreshIntent) (RefreshReceipt, error) { - if r.authorize == nil || i.NodeID != r.nodeID || !validRefreshIntent(i) || i.PolicyDigest != r.policy.digest() { + if r.authorize == nil || i.NodeID != r.nodeID || i.Fence.Executor.NodeBootID != r.boot || !validRefreshIntent(i) || i.PolicyDigest != r.policy.digest() { return RefreshReceipt{}, ErrManagementDenied } var state AuthorityState diff --git a/worker/internal/slotgate/node_refresh_authorizer.go b/worker/internal/slotgate/node_refresh_authorizer.go new file mode 100644 index 000000000..4d12ec420 --- /dev/null +++ b/worker/internal/slotgate/node_refresh_authorizer.go @@ -0,0 +1,78 @@ +package slotgate + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "github.com/jackc/pgx/v5" + "time" +) + +// Refresh authorization is independently installed; no allocation/launch grant is accepted. +func (a *pgAllocationAuthorizer) AuthorizeRefresh(ctx context.Context, intent RefreshIntent, dispatch func() error) error { + if intent.NodeID != a.nodeID || !validRefreshIntent(intent) { + return ErrManagementDenied + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + connection, err := pgx.ConnectConfig(ctx, a.config.Copy()) + if err != nil { + return ErrManagementUnavailable + } + defer func() { + closed, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + connection.Close(closed) + }() + var directRole bool + if err = connection.QueryRow(ctx, "SELECT current_user=session_user").Scan(&directRole); err != nil || !directRole { + return ErrManagementDenied + } + q := pgx.Identifier{a.schema}.Sanitize() + var nodeID, applicationSchema string + var version int + if err = connection.QueryRow(ctx, "SELECT node_id,application_schema,protocol_version FROM "+q+".reader_identity()").Scan(&nodeID, &applicationSchema, &version); err != nil { + return ErrManagementUnavailable + } + expectedSchema := sha256.Sum256([]byte(applicationSchema)) + if version != 1 || nodeID != a.nodeID || a.schema != "vm2api_node_"+hex.EncodeToString(expectedSchema[:16]) { + return ErrManagementDenied + } + tx, err := connection.BeginTx(ctx, pgx.TxOptions{AccessMode: pgx.ReadOnly}) + if err != nil { + return ErrManagementUnavailable + } + defer func() { + rollback, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + tx.Rollback(rollback) + }() + if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock_shared(hashtext($1),hashtext($2))", + "vm2api:node-management:"+a.schema, "node:"+a.nodeID+":vm:"+intent.VMID); err != nil { + return ErrManagementUnavailable + } + var epoch, digest string + err = tx.QueryRow(ctx, "SELECT epoch::text,intent_digest FROM "+q+".authorized_refreshes WHERE allocation_id=$1 AND node_id=$2 AND vm_id=$3 AND operation_id=$4 AND owner_id=$5 AND command_id=$6 AND account_id=$7 AND bg_task_key=$8 AND bg_execution_id=$9 AND bg_owner_id=$10 AND expected_generation=$11", + intent.AllocationID, intent.NodeID, intent.VMID, intent.OperationID, intent.OwnerID, intent.CommandID, intent.AccountID, intent.BGTaskKey, intent.BGExecutionID, intent.BGOwnerID, intent.ExpectedGeneration).Scan(&epoch, &digest) + if err == pgx.ErrNoRows { + return ErrManagementDenied + } + if err != nil { + return ErrManagementUnavailable + } + if epoch != intent.Epoch || digest != intent.digest() { + return ErrManagementDenied + } + if ctx.Err() != nil { + return ErrManagementUnavailable + } + // Only bounded local journal I/O belongs in dispatch. Never Docker or an + // outbound RPC. A failed COMMIT cannot acknowledge a local receipt. + if err = dispatch(); err != nil { + return err + } + if err = tx.Commit(ctx); err != nil { + return ErrManagementUnavailable + } + return nil +} diff --git a/worker/internal/slotgate/node_refresh_test.go b/worker/internal/slotgate/node_refresh_test.go index 18e6fe289..06de22590 100644 --- a/worker/internal/slotgate/node_refresh_test.go +++ b/worker/internal/slotgate/node_refresh_test.go @@ -27,7 +27,7 @@ func refreshFixture(t *testing.T, handler http.Handler) (*publicationFixture, *n calls := new(atomic.Int64) provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); handler.ServeHTTP(w, r) })) t.Cleanup(provider.Close) - r := &nodeCredentialRefresher{authority: f.a, root: f.root, nodeID: f.intent.NodeID, policy: f.l.policy, authorize: func(_ context.Context, _ RefreshIntent, dispatch func() error) error { return dispatch() }, newRefresher: func(store *credential.Store) *kinoauth.Refresher { + r := &nodeCredentialRefresher{boot: f.l.boot, authority: f.a, root: f.root, nodeID: f.intent.NodeID, policy: f.l.policy, authorize: func(_ context.Context, _ RefreshIntent, dispatch func() error) error { return dispatch() }, newRefresher: func(store *credential.Store) *kinoauth.Refresher { return &kinoauth.Refresher{Store: store, Client: provider.Client(), TokenURL: provider.URL, MaxTries: 1} }} i := RefreshIntent{Action: "credential-refresh", NodeID: f.intent.NodeID, VMID: "v", AllocationID: f.intent.AllocationID, OperationID: "refresh-operation", OwnerID: "refresh-owner", CommandID: "refresh-command", AccountID: "synthetic-account", BGTaskKey: "oauth-refresh:claude:synthetic-account", BGExecutionID: "refresh-execution", BGOwnerID: "refresh-bg-owner", Epoch: "1", Fence: state.Fence, ExpectedGeneration: "1", ManifestRevision: "1", ManifestDigest: state.InitialManifest.Receipt.ManifestDigest, PresetID: initialPresetID, PolicyDigest: f.l.policy.digest(), Force: true} @@ -109,3 +109,62 @@ func TestNodeRefreshRequiresPrivateOwnershipAndOriginalActiveFence(t *testing.T) t.Fatal("denied refresh reached provider") } } + +func TestNodeRefreshProjectionFailuresRemainUnknown(t *testing.T) { + for _, point := range []string{"before-projection", "after-projection"} { + t.Run(point, func(t *testing.T) { + f, r, i, calls := refreshFixture(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"access_token":"synthetic-next","expires_in":3600}`) + })) + r.fault = func(stage string) error { + if stage == point { + return errors.New("synthetic projection failure") + } + return nil + } + if _, err := r.Refresh(context.Background(), i); err == nil { + t.Fatal("partial publication completed") + } + i.CommandID = "after-partial-publication" + if _, err := r.Refresh(context.Background(), i); err == nil { + t.Fatal("partial publication allowed another refresh") + } + state, _ := f.a.State("v") + if calls.Load() != 1 || !hasPendingMutation(state) { + t.Fatal("partial publication lost its pending attempt") + } + if _, err := f.a.Revoke("v", i.Fence); err != nil { + t.Fatal(err) + } + if _, err := f.a.Advance(context.Background(), "v", i.Fence); !errors.Is(err, ErrPendingMutation) { + t.Fatal("container termination was allowed to clear a node refresh") + } + }) + } +} +func TestNodeRefreshCompleteRejectionAllowsLaterOperatorAttempt(t *testing.T) { + _, r, i, calls := refreshFixture(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(401) + fmt.Fprint(w, `{"error":"invalid_grant"}`) + })) + for _, id := range []string{"first-rejection", "later-operator-attempt"} { + i.CommandID = id + receipt, err := r.Refresh(context.Background(), i) + if err != nil || receipt.OK || !receipt.Finished || receipt.Credential.Generation != "1" { + t.Fatalf("complete rejection: %+v %v", receipt, err) + } + } + if calls.Load() != 2 { + t.Fatal("a previous complete rejection permanently suppressed an operator attempt") + } +} +func TestNodePrivateCredentialStateIsNotMutableThroughSnapshot(t *testing.T) { + f := publishingFixture(t) + f.publish(t) + state, _ := f.a.State("v") + state.Credential.Generation = "999" + persisted, _ := f.a.State("v") + if persisted.Credential.Generation != "1" { + t.Fatal("snapshot mutated private authority") + } +} diff --git a/worker/internal/slotgate/slotgate.go b/worker/internal/slotgate/slotgate.go index 1e0b5efef..30b182494 100644 --- a/worker/internal/slotgate/slotgate.go +++ b/worker/internal/slotgate/slotgate.go @@ -54,6 +54,7 @@ var ( ) type Server struct { + refresher *nodeCredentialRefresher cfg Config token [sha256.Size]byte tls *tls.Config @@ -140,6 +141,9 @@ func New(cfg Config) (*Server, error) { srv.allocator = &allocationManager{authority: srv.authority, nodeID: cfg.Management.NodeID, authorize: authorizer.Authorize} srv.managementNodeID = cfg.Management.NodeID srv.authorizeCommand = authorizer.AuthorizeCommand + if srv.launcher != nil { + srv.refresher = &nodeCredentialRefresher{boot: boot, authority: srv.authority, root: cfg.Root, nodeID: cfg.Management.NodeID, policy: *cfg.Launch, authorize: authorizer.AuthorizeRefresh} + } } } return srv, nil From 1b31fdd5847b9129de8f1b5c0267e6ec9b58c11a Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:36:15 +0800 Subject: [PATCH 423/473] test: require file-free HA credential refresh producers --- .../ha-credential-refresh-producers.mjs | 27 ++++++++++ .../ha-credential-refresh-producers.test.mjs | 50 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 test/fixtures/ha-credential-refresh-producers.mjs create mode 100644 test/unit/ha-credential-refresh-producers.test.mjs diff --git a/test/fixtures/ha-credential-refresh-producers.mjs b/test/fixtures/ha-credential-refresh-producers.mjs new file mode 100644 index 000000000..c52d79924 --- /dev/null +++ b/test/fixtures/ha-credential-refresh-producers.mjs @@ -0,0 +1,27 @@ +import test from 'node:test' +import fs from 'node:fs' +const scenario=process.argv[2] +let ensureCalls=0,podReads=0,marked=0,captures=0,legacy=0 +const credential={has_access:true,has_refresh:true,needs_refresh:false,generation:'2',expires_at:'2000000000000'} +const outcome=scenario.endsWith('unknown')?{ok:false,status:503,attemptFinished:false,credentialRefreshUnknown:true}:scenario.endsWith('refused')?{ok:false,status:503,credentialRefreshNotStarted:true}:{ok:true,status:200,refreshed:true,attemptFinished:true,credential} +const ensure=async exec=>{ensureCalls++;if(!exec.slotgate)throw new Error('missing typed node capability');return outcome} +let result,error +if(scenario.startsWith('runner')){ + const worker=await import('../../src/lib/transport/go-worker-client.mjs') + test.mock.module(new URL('../../src/lib/transport/go-worker-client.mjs',import.meta.url).href,{namedExports:{...worker,ensureWorkerCredential:ensure}}) + const credentials=await import('../../src/lib/oauth/oauth-credentials.mjs') + test.mock.module(new URL('../../src/lib/oauth/oauth-credentials.mjs',import.meta.url).href,{namedExports:{...credentials,readWorkerCredentialFile:()=>{podReads++;throw new Error('Pod credential read')}}}) + const{FailoverRunner}=await import('../../src/lib/pool/failover-runner.mjs') + const runner=new FailoverRunner({scheduler:{}}) + const selected={vmId:'ha-vm',accountId:'ha-account',exec:{vmId:'ha-vm',accountId:'ha-account',slotgate:{trusted:true}},bindExecution:async()=>{},markUncertain:async()=>{marked++}} + try{result=await runner.recoverCredential(selected,{reason:'oauth_refresh_required',action:'switch',retrySameAccount:false},{executionEnded:true})}catch(e){error={code:e.code,executionUncertain:e.executionUncertain}} +}else{ + const source=fs.readFileSync(new URL('../../src/server.mjs',import.meta.url),'utf8') + const start=source.indexOf('async function refreshWorkerCredentialForVm('),end=source.indexOf('\n}\n',start)+2 + if(start<0||end{},get:()=>current},captureNodeExecution:async input=>{captures++;if(input.nodeId!=='pg-node'||input.accountId!=='ha-account')throw new Error('stale VM metadata was trusted');return{vmId:'ha-vm',accountId:'ha-account',slotgate:{trusted:true},workerStatus:{ok:true,credential:{account_uuid:'ha-account'}}}},ensureWorkerCredential:ensure,publicCredentialRefreshResult:(vmId,value)=>({vm_id:vmId,...value}),mirrorWorkerCredentialsToVm:()=>{legacy++},flushVmDbSync:async()=>{legacy++},invalidateLiveCredentialCache:()=>{legacy++},markVmRefreshError:()=>{legacy++},accountQuota:{clearGrantRevokeLeftover:()=>{legacy++}},poolScheduler:{clearAuthCooldownFor:()=>{legacy++},clearGrantRevokeCooldownFor:()=>{legacy++}}} + const refresh=Function('deps',`with(deps){return (${source.slice(start,end)})}`)(deps) + try{result=await refresh({vmId:'ha-vm',homeDir:'/must-not-read',vmPath:'/must-not-write',vm:{id:'ha-vm',node_id:'stale-node'},force:true})}catch(e){error={code:e.code,message:e.message}} +} +process.stdout.write(JSON.stringify({ensureCalls,podReads,marked,captures,legacy,result,error})) diff --git a/test/unit/ha-credential-refresh-producers.test.mjs b/test/unit/ha-credential-refresh-producers.test.mjs new file mode 100644 index 000000000..601458d39 --- /dev/null +++ b/test/unit/ha-credential-refresh-producers.test.mjs @@ -0,0 +1,50 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +const run = (scenario) => { + const env = { ...process.env, VM2API_CONTROL_PLANE_MODE: 'standalone' } + delete env.NODE_TEST_CONTEXT + return JSON.parse( + execFileSync( + process.execPath, + [ + '--experimental-test-module-mocks', + new URL('../fixtures/ha-credential-refresh-producers.mjs', import.meta.url).pathname, + scenario, + ], + { encoding: 'utf8', env, stdio: ['ignore', 'pipe', 'pipe'] }, + ), + ) +} +test('HA retry recovery uses typed refresh with no homeDir and no Pod credential comparison', () => { + const r = run('runner-success') + assert.equal(r.ensureCalls, 1) + assert.equal(r.podReads, 0) + assert.equal(r.result.retrySameAccount, true) + assert.equal(r.result.action, 'continue') +}) +test('HA retry recovery preserves unknown and handles only an explicit known refusal', () => { + const unknown = run('runner-unknown') + assert.equal(unknown.marked, 1) + assert.equal(unknown.error.executionUncertain, true) + const refused = run('runner-refused') + assert.equal(refused.marked, 0) + assert.equal(refused.result.reason, 'oauth_refresh_not_started') + assert.equal(refused.result.action, 'stop') +}) +test('server HA refresh captures authoritative PG metadata and avoids all file writers', () => { + const r = run('server-success') + assert.equal(r.error, undefined) + assert.equal(r.captures, 1) + assert.equal(r.ensureCalls, 1) + assert.equal(r.legacy, 0) + assert.equal(r.result.ok, true) + assert.equal(r.result.taskTerminal, true) +}) +test('server HA refresh retains typed uncertainty for the background task wrapper', () => { + const r = run('server-unknown') + assert.equal(r.captures, 1) + assert.equal(r.legacy, 0) + assert.equal(r.result.credentialRefreshUnknown, true) + assert.notEqual(r.result.taskTerminal, true) +}) From c1c0a781ac478dcf983912675ac69e7cc925f8a2 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:37:56 +0800 Subject: [PATCH 424/473] fix: use typed node credentials in HA refresh producers --- src/lib/pool/failover-runner.mjs | 26 ++++++++++++++++--- src/server.mjs | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index bdd7a0117..0f4cd99cf 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -517,8 +517,9 @@ export class FailoverRunner { async recoverCredential(selected, policy, { executionEnded = false } = {}) { if (policy?.reason !== 'oauth_refresh_required') return policy const exec = selected?.exec - if (!exec?.homeDir) return permanentOAuthRevoke(policy) - const before = readWorkerCredentialFile(exec.homeDir) + const nodeOwned = exec?.slotgate != null + if (!nodeOwned && !exec?.homeDir) return permanentOAuthRevoke(policy) + const before = nodeOwned ? null : readWorkerCredentialFile(exec.homeDir) const beforeRefresh = String(before?.refresh_token || '') let refreshed try { @@ -540,14 +541,31 @@ export class FailoverRunner { } refreshed = { ok: false, error: { message: String(error?.message || error) } } } - if (selected.bindExecution && refreshed?.credentialRefreshUnknown) { + if ( + nodeOwned && + selected.bindExecution && + executionEnded === true && + refreshed?.credentialRefreshNotStarted === true + ) + return { + ...policy, + retrySameAccount: false, + action: 'stop', + cooldownUntil: null, + reason: 'oauth_refresh_not_started', + credentialRefreshNotStarted: true, + } + if ( + selected.bindExecution && + (refreshed?.credentialRefreshUnknown || (nodeOwned && refreshed?.attemptFinished !== true)) + ) { await selected.markUncertain() throw Object.assign(new Error('credential refresh termination is unknown'), { code: 'credential_refresh_unknown', executionUncertain: true, }) } - const after = readWorkerCredentialFile(exec.homeDir) + const after = nodeOwned ? null : readWorkerCredentialFile(exec.homeDir) const afterRefresh = String(after?.refresh_token || '') const blob = `${refreshed?.error?.code || ''} ${refreshed?.error?.message || ''} ${refreshed?.refresh_class || ''}` const grantDead = /invalid_grant|token has been revoked|oauth_revoked/i.test(blob) diff --git a/src/server.mjs b/src/server.mjs index 7698e1f5d..065101808 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -794,6 +794,49 @@ function publicCredentialRefreshResult(vmId, result, { force = false } = {}) { } async function refreshWorkerCredentialForVm({ vmId, vmPath, homeDir, vm, force = false }) { + if (haMode) { + let captured + let current + try { + await haCatalog.refresh() + current = haCatalog.get(vmId) + if (!current?.node_id || !current?.claude?.account_uuid) + throw Object.assign(new Error('HA credential metadata unavailable'), { code: 'node_capability_unavailable' }) + captured = await captureNodeExecution({ + db, + trustedNodes, + vmId, + accountId: current.claude.account_uuid, + nodeId: current.node_id, + }) + } catch (error) { + error.credentialRefreshNotStarted = true + error.vmOperationNotStarted = true + error.taskNotStarted = true + throw error + } + const result = await ensureWorkerCredential( + { ...captured, vm: current, nodeIdentity: captured.workerStatus }, + { force: !!force, db }, + ) + const published = publicCredentialRefreshResult(vmId, result, { force }) + return { + ...published, + refresh_owner: 'go-node-worker', + credential: { + ...published.credential, + account_uuid: captured.accountId, + ...(result.credential?.org_uuid ? { org_uuid: result.credential.org_uuid } : {}), + ...(result.credential?.email ? { email: result.credential.email } : {}), + }, + taskTerminal: result.credentialRefreshUnknown !== true && (result.attemptFinished === true || result.ok === true), + ...(result.credentialRefreshUnknown === true ? { credentialRefreshUnknown: true } : {}), + ...(result.credentialRefreshNotStarted === true + ? { credentialRefreshNotStarted: true, taskNotStarted: true } + : {}), + } + } + // Host RefreshIfNeeded is the credential writer. Slot workers only read AT. const result = await ensureWorkerCredential( { vmId, vm, homeDir }, From 3ad659a86c023d98754bdd7e3a6b2f2c2db1caf3 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:39:45 +0800 Subject: [PATCH 425/473] fix: require exact node account before credential health reads --- src/lib/transport/go-worker-client.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index 641633113..3785978f2 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -1142,6 +1142,14 @@ export async function workerHealth(exec, { timeoutMs = 3000, signal } = {}) { export async function ensureWorkerCredential(exec, { force = false, ...refreshOptions } = {}) { requireNodeCapability(exec) if (exec?.slotgate != null) { + if (!exec.accountId && !exec.nodeIdentity?.credential?.account_uuid) + return { + ok: false, + status: 503, + refreshed: false, + credentialRefreshNotStarted: true, + error: { code: 'slotgate_route_unavailable', message: 'Node credential identity is required' }, + } const health = await workerHealth(exec) const credential = health?.credential if ( From 43ff50196e855c90cbf4e7e47fd63b41b709d446 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:17:32 +0800 Subject: [PATCH 426/473] test: specify immutable node recovery status and strict usage receipts --- test/fixtures/fake-slotgate.mjs | 4 + test/unit/node-recovery-receipt.test.mjs | 97 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 test/unit/node-recovery-receipt.test.mjs diff --git a/test/fixtures/fake-slotgate.mjs b/test/fixtures/fake-slotgate.mjs index 61741d3c1..354e20936 100644 --- a/test/fixtures/fake-slotgate.mjs +++ b/test/fixtures/fake-slotgate.mjs @@ -56,6 +56,10 @@ export async function fakeSlotgate(t, options = {}) { key === 'generation' && typeof value === 'number' ? context.source : value) buffer = buffer.subarray(second + 1) requests.push(command) + if (command.op === 'recovery-status') { + const reply = typeof options.recoveryStatus === 'function' ? options.recoveryStatus(command) : options.recoveryStatus + return socket.end(wire(reply || { ok: true, found: false, unknown: true }) + '\n') + } if (command.op === 'health') { const reply = typeof options.health === 'function' ? options.health(command) : options.health if (options.holdHealth) return diff --git a/test/unit/node-recovery-receipt.test.mjs b/test/unit/node-recovery-receipt.test.mjs new file mode 100644 index 000000000..8826d741f --- /dev/null +++ b/test/unit/node-recovery-receipt.test.mjs @@ -0,0 +1,97 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { SlotgateClient } from '../../src/lib/transport/slotgate-client.mjs' +import { fakeSlotgate, syntheticFence } from '../fixtures/fake-slotgate.mjs' + +const receipt = () => ({ + id: 'a'.repeat(32), + model: 'claude-sonnet-4-5', + usage_complete: true, + usage: { + input_tokens: '9007199254740993', + output_tokens: '7', + cache_read_input_tokens: '3', + cache_creation_input_tokens: '5', + cache_creation: { ephemeral_5m_input_tokens: '2', ephemeral_1h_input_tokens: '3' }, + server_tool_use: { web_search_requests: '1' }, + }, + speed: 'fast', + service_tier: 'priority', + stop_reason: 'end_turn', +}) +const status = (command, value = receipt()) => ({ + ok: true, + found: true, + command: { id: command.command_id, fence: command.fence, status: 'completed', dispatched: true }, + ...(value ? { receipt: value } : {}), +}) + +test('history reads original command and Fence without a dead ticket, manifest, prepare or latest State', async (t) => { + const fx = await fakeSlotgate(t, { recoveryStatus: status }) + fx.config.fence = syntheticFence('18446744073709551615') + const client = new SlotgateClient(fx.config) + const result = await client.recoveryStatus('original-command') + assert.equal(result.found, true) + assert.equal(result.command.fence.generation, '18446744073709551615') + assert.deepEqual(result.receipt, receipt()) + assert.equal(typeof result.receipt.usage.input_tokens, 'string') + assert.deepEqual(fx.requests, [ + { op: 'recovery-status', vm_id: fx.config.vmId, fence: fx.config.fence, command_id: 'original-command' }, + ]) +}) + +test('notfound remains unknown and an incomplete receipt preserves missing fields', async (t) => { + const fx = await fakeSlotgate(t, { + recoveryStatus: (command) => + command.command_id === 'notfound' + ? { ok: true, found: false, unknown: true } + : status(command, { id: 'b'.repeat(32), usage_complete: false, usage: { output_tokens: '7' } }), + }) + const client = new SlotgateClient(fx.config) + assert.deepEqual(await client.recoveryStatus('notfound'), { found: false, unknown: true }) + const incomplete = await client.recoveryStatus('incomplete') + assert.equal(incomplete.receipt.usage_complete, false) + assert.deepEqual(incomplete.receipt.usage, { output_tokens: '7' }) + assert.equal(incomplete.receipt.model, undefined) +}) + +for (const malformed of [ + 'numeric-count', + 'leading-zero', + 'overflow', + 'negative', + 'credential-field', + 'nested-provider', + 'missing-model', + 'missing-count', + 'wrong-command', + 'wrong-fence', + 'pending-receipt', + 'undispatched-receipt', +]) { + test(`history rejects ${malformed} without fallback or writes`, async (t) => { + const fx = await fakeSlotgate(t, { + recoveryStatus: (command) => { + const value = status(command) + if (malformed === 'numeric-count') value.receipt.usage.input_tokens = 12 + if (malformed === 'leading-zero') value.receipt.usage.input_tokens = '01' + if (malformed === 'overflow') value.receipt.usage.input_tokens = '9223372036854775808' + if (malformed === 'negative') value.receipt.usage.input_tokens = '-1' + if (malformed === 'credential-field') value.receipt.credentials = { refresh_token: 'synthetic-private-value' } + if (malformed === 'nested-provider') value.receipt.usage.provider = { extra: true } + if (malformed === 'missing-model') delete value.receipt.model + if (malformed === 'missing-count') delete value.receipt.usage.output_tokens + if (malformed === 'wrong-command') value.command.id = 'new-command' + if (malformed === 'wrong-fence') value.command.fence = syntheticFence('2') + if (malformed === 'pending-receipt') value.command.status = 'pending' + if (malformed === 'undispatched-receipt') value.command.dispatched = false + return value + }, + }) + await assert.rejects(new SlotgateClient(fx.config).recoveryStatus('original-command'), { + code: 'slotgate_recovery_unconfirmed', + }) + assert.equal(fx.requests.length, 1) + assert.equal(fx.requests[0].op, 'recovery-status') + }) +} From 4420b3dee69b1ddc2432a45040b13a03bff28e82 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:18:05 +0800 Subject: [PATCH 427/473] feat: read original node completion with exact public usage receipts --- src/lib/core/node-execution-proof.mjs | 51 +++++++++++++++++++++++++++ src/lib/transport/slotgate-client.mjs | 35 ++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/lib/core/node-execution-proof.mjs b/src/lib/core/node-execution-proof.mjs index c20b8632b..2787cdb07 100644 --- a/src/lib/core/node-execution-proof.mjs +++ b/src/lib/core/node-execution-proof.mjs @@ -106,3 +106,54 @@ export function normalizeNodeCompletion(value, binding) { if (command.status !== 'completed' || command.dispatched !== true) invalid() return command } + +/** Public node journal metadata only. Counts remain exact decimal strings. */ +export function normalizeNodeReceipt(value) { + const object = (item, required, optional = []) => { + if ( + !item || + typeof item !== 'object' || + Array.isArray(item) || + ![Object.prototype, null].includes(Object.getPrototypeOf(item)) || + required.some((key) => !Object.hasOwn(item, key)) || + Object.keys(item).some((key) => !required.includes(key) && !optional.includes(key)) + ) + invalid() + } + const count = (item) => { + if (typeof item !== 'string' || !/^(0|[1-9][0-9]{0,18})$/.test(item) || BigInt(item) > 9223372036854775807n) + invalid() + return item + } + object(value, ['id', 'usage_complete', 'usage'], ['model', 'speed', 'service_tier', 'stop_reason']) + if (typeof value.id !== 'string' || !/^[a-f0-9]{32}$/.test(value.id) || typeof value.usage_complete !== 'boolean') + invalid() + const result = { id: value.id, usage_complete: value.usage_complete, usage: {} } + for (const key of ['model', 'speed', 'service_tier', 'stop_reason']) { + if (!Object.hasOwn(value, key)) continue + const item = value[key] + if ( + typeof item !== 'string' || + !/^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(item) || + item.length > (key === 'model' ? 256 : 64) + ) + invalid() + if (key === 'stop_reason' && !['end_turn', 'tool_use', 'max_tokens', 'stop_sequence'].includes(item)) invalid() + result[key] = item + } + const main = ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'] + object(value.usage, [], [...main, 'cache_creation', 'server_tool_use']) + for (const key of main) if (Object.hasOwn(value.usage, key)) result.usage[key] = count(value.usage[key]) + for (const [key, fields] of [ + ['cache_creation', ['ephemeral_5m_input_tokens', 'ephemeral_1h_input_tokens']], + ['server_tool_use', ['web_search_requests']], + ]) { + if (!Object.hasOwn(value.usage, key)) continue + const nested = value.usage[key] + object(nested, [], fields) + if (!Object.keys(nested).length) invalid() + result.usage[key] = Object.fromEntries(Object.entries(nested).map(([field, item]) => [field, count(item)])) + } + if (value.usage_complete && (!result.model || main.some((key) => !Object.hasOwn(result.usage, key)))) invalid() + return result +} diff --git a/src/lib/transport/slotgate-client.mjs b/src/lib/transport/slotgate-client.mjs index 028a94731..30f65f3c1 100644 --- a/src/lib/transport/slotgate-client.mjs +++ b/src/lib/transport/slotgate-client.mjs @@ -7,6 +7,7 @@ import { normalizeNodeBinding, normalizeNodeCommand, normalizeNodeCompletion, + normalizeNodeReceipt, sameNodeBinding, } from '../core/node-execution-proof.mjs' @@ -349,6 +350,40 @@ export class SlotgateClient { return handle } + /** Query the original journal after controller loss. Notfound never proves no dispatch. */ + async recoveryStatus(requestId, { timeoutMs = 3000, signal } = {}) { + if ( + typeof requestId !== 'string' || + !requestId || + requestId.trim() !== requestId || + requestId.length > 256 || + /[\x00-\x1f\x7f]/.test(requestId) + ) + throw failure('slotgate_request_invalid') + const reply = await this._rpc( + { op: 'recovery-status', vm_id: this.nodeBinding.vmId, fence: this.nodeBinding.fence, command_id: requestId }, + timeoutMs, + signal, + ) + try { + if (reply.found === false && reply.unknown === true && Object.keys(reply).length === 3) + return freeze({ found: false, unknown: true }) + if ( + reply.found !== true || + Object.keys(reply).some((key) => !['ok', 'found', 'command', 'receipt'].includes(key)) + ) + throw failure('slotgate_recovery_unconfirmed') + const command = normalizeNodeCommand(reply.command, { requestId, nodeBinding: this.nodeBinding }) + if (Object.hasOwn(reply, 'receipt')) { + normalizeNodeCompletion(command, { requestId, nodeBinding: this.nodeBinding }) + return freeze({ found: true, command, receipt: normalizeNodeReceipt(reply.receipt) }) + } + return freeze({ found: true, command }) + } catch { + throw failure('slotgate_recovery_unconfirmed') + } + } + async completion(handle) { if (!this.handles.has(handle)) throw failure('slotgate_ticket_invalid') const deadline = Date.now() + 1000 From 00dffaf6f0fb5068bf8540f56b2ad30be8223f8b Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:22:13 +0800 Subject: [PATCH 428/473] test: require dispatch-time HA price snapshots before billing recovery --- test/fixtures/ha-billing.mjs | 71 ++++++++++++++++++++++ test/unit/ha-request-billing.test.mjs | 84 +++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 test/fixtures/ha-billing.mjs create mode 100644 test/unit/ha-request-billing.test.mjs diff --git a/test/fixtures/ha-billing.mjs b/test/fixtures/ha-billing.mjs new file mode 100644 index 000000000..795b932ab --- /dev/null +++ b/test/fixtures/ha-billing.mjs @@ -0,0 +1,71 @@ +import pg from 'pg' +import { getDb, quoteIdentifier, registerActivityCoordinator } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator, coordinationSchemaFor } from '../../src/lib/db/global-maintenance.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { withHaRequestContext, withHaRequestAttempt } from '../../src/lib/core/ha-request-context.mjs' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' +import { syntheticFence } from './fake-slotgate.mjs' + +export const syntheticBillingSnapshot = () => ({ + version: 1, model: 'claude-sonnet-4-5', rate_multiplier: 2, cache_ttl: '1h', requested_speed: null, + resolved: { id: 'claude-sonnet-4-5', platform: 'anthropic', source: 'override', version: 'synthetic-v1', mode_sources: {}, price: { input: 3, output: 15, cache_read: .3, cache_5m: 3.75, cache_1h: 6 } }, +}) +export const syntheticUsageReceipt = () => ({ + id: 'a'.repeat(32), model: 'claude-sonnet-4-5-20250929', usage_complete: true, + usage: { input_tokens: '100', output_tokens: '10', cache_read_input_tokens: '20', cache_creation_input_tokens: '30' }, + stop_reason: 'end_turn', +}) + +export async function haBillingFixture(t) { + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const original = new GlobalMaintenanceCoordinator(db, { ownerId: 'billing-original-owner' }) + const recovery = new GlobalMaintenanceCoordinator(peer, { ownerId: 'billing-recovery-owner' }) + await original.start(); await recovery.start() + registerActivityCoordinator(db, original); registerActivityCoordinator(peer, recovery) + await original.acknowledgeEpoch(1); await recovery.acknowledgeEpoch(1) + const originalGate = await original.admit({ kind: 'request', requestId: 'billing-parent' }) + const recoveryGate = await recovery.admit({ kind: 'recovery' }) + const before = process.env.VM2API_CONTROL_PLANE_MODE + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + t.after(async () => { + if (before == null) delete process.env.VM2API_CONTROL_PLANE_MODE + else process.env.VM2API_CONTROL_PLANE_MODE = before + await Promise.allSettled([original.stop(), recovery.stop()]) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinationSchemaFor(db.schema))} CASCADE`) } + finally { await cleanup.end() } + await peer.close() + }) + return { db, peer, original, recovery, originalGate, recoveryGate } +} + +export async function billingRequest(f, callback = async () => {}, { snapshot = syntheticBillingSnapshot(), jobs = 1 } = {}) { + const nodeBinding = { protocol: 'KSG2', nodeId: 'original-node', vmId: 'vm-fixture', kind: 'kernel', fence: syntheticFence() } + const data = { nodeBinding, commandIds: [] } + await f.originalGate.run(() => withHaRequestContext({ db: f.db, requestId: 'billing-parent', activity: f.originalGate, recoveryKind: 'claude_messages' }, async () => { + data.keys = new ApiKeyStore({ db: f.db }) + data.key = await data.keys.create({ name: 'synthetic-billing-key' }) + data.reservation = await data.keys.reserve(data.key) + data.admission = await new ExecutionAdmissionsRepo(f.db).tryReserve({ accountId: 'fixture-account', vmId: 'vm-fixture', accountLimit: 5, vmLimit: 5, fableLimit: 0, rpmLimit: 0 }) + for (let attempt = 1; attempt <= jobs; attempt++) { + await new RequestAttemptsRepo(f.db).begin({ requestId: 'billing-parent', attemptNo: attempt, vmId: 'vm-fixture', accountId: 'fixture-account', model: snapshot?.model ?? 'unpriced' }) + const commandId = 'billing-command-' + attempt + data.commandIds.push(commandId) + await withHaRequestAttempt(attempt, () => data.admission.bindExecution({ requestId: commandId, nodeBinding, billingSnapshot: snapshot })) + await data.admission.completeExecution({ requestId: commandId, nodeBinding, nodeCompletion: { id: commandId, fence: nodeBinding.fence, status: 'completed', dispatched: true }, started: true, ended: true, reason: 'node_completed' }) + } + await callback(data) + await data.admission.markUncertain() + })) + await f.originalGate.complete({ ended: false, reason: 'synthetic_owner_lost' }) + await f.original.stop() + return data +} + +export function statusFor(data, requestId, receipt = syntheticUsageReceipt()) { + return { found: true, command: { id: requestId, fence: data.nodeBinding.fence, status: 'completed', dispatched: true }, receipt } +} diff --git a/test/unit/ha-request-billing.test.mjs b/test/unit/ha-request-billing.test.mjs new file mode 100644 index 000000000..535c1a2cc --- /dev/null +++ b/test/unit/ha-request-billing.test.mjs @@ -0,0 +1,84 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { + haBillingFixture, + billingRequest, + syntheticBillingSnapshot, + syntheticUsageReceipt, +} from '../fixtures/ha-billing.mjs' +import { calculateCost } from '../../src/lib/admin/pricing.mjs' +import { setPriceCatalogTestHooks, invalidatePriceCatalog } from '../../src/lib/admin/price-catalog.mjs' + +test('HA job binding persists price, group, cache TTL and speed before returning permission to dispatch', async (t) => { + const f = await haBillingFixture(t) + const snapshot = syntheticBillingSnapshot() + await billingRequest( + f, + async (data) => { + const row = ( + await f.peer.pool.query('SELECT billing_snapshot FROM execution_jobs WHERE request_id=$1', [data.commandIds[0]]) + ).rows[0] + assert.deepEqual(row.billing_snapshot, snapshot) + snapshot.rate_multiplier = 100 + assert.equal( + ( + await f.peer.pool.query('SELECT billing_snapshot FROM execution_jobs WHERE request_id=$1', [ + data.commandIds[0], + ]) + ).rows[0].billing_snapshot.rate_multiplier, + 2, + ) + }, + { snapshot }, + ) +}) + +test('frozen price snapshots keep original rates after another Pod changes overrides', async () => { + const { captureHaBillingSnapshot, priceHaReceipt } = await import('../../src/lib/core/ha-request-billing.mjs') + const values = { + model_price_overrides: { + 'anthropic/claude-sonnet-4-5': { + input: 3, + output: 15, + cache_read: 0.3, + cache_5m: 3.75, + cache_1h: 6, + updated_at: 'before', + }, + }, + } + setPriceCatalogTestHooks({ settings: { get: (key, fallback) => values[key] ?? fallback }, backgroundRefresh: false }) + try { + const snapshot = captureHaBillingSnapshot({ + model: 'claude-sonnet-4-5', + rateMultiplier: 2, + cacheTtl: '1h', + requestedSpeed: null, + }) + const receipt = syntheticUsageReceipt() + const expected = calculateCost({ ...receipt.usage, cache_ttl: '1h' }, receipt.model) + values.model_price_overrides['anthropic/claude-sonnet-4-5'].input = 900 + invalidatePriceCatalog() + const actual = priceHaReceipt(receipt, snapshot) + assert.equal(actual.known, true) + assert.equal(actual.pricing.total_cost, expected.total_cost) + assert.equal(actual.actualCost, expected.total_cost * 2) + } finally { + setPriceCatalogTestHooks(null) + } +}) + +for (const absent of ['usage', 'price', 'different-model']) { + test(`HA ${absent} stays unpriced without substituting zero or request model`, async () => { + const { priceHaReceipt } = await import('../../src/lib/core/ha-request-billing.mjs') + const snapshot = syntheticBillingSnapshot() + const receipt = syntheticUsageReceipt() + if (absent === 'usage') { + receipt.usage_complete = false + delete receipt.usage.output_tokens + } + if (absent === 'price') snapshot.resolved = null + if (absent === 'different-model') receipt.model = 'claude-opus-4-6' + assert.equal(priceHaReceipt(receipt, snapshot).known, false) + }) +} From 3699a5e83d19a66ce7e7e13699b1484d017a39ae Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:23:37 +0800 Subject: [PATCH 429/473] feat: persist HA dispatch price snapshots and settlement audit schema --- src/lib/admin/price-catalog.mjs | 4 +- src/lib/admin/pricing.mjs | 7 +- src/lib/core/ha-request-billing.mjs | 95 +++++++++++++++++++ .../021_ha_request_billing.sql | 24 +++++ .../db/repos/execution-admissions-repo.mjs | 6 +- test/unit/database.test.mjs | 7 +- 6 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 src/lib/core/ha-request-billing.mjs create mode 100644 src/lib/db/postgres-migrations/021_ha_request_billing.sql diff --git a/src/lib/admin/price-catalog.mjs b/src/lib/admin/price-catalog.mjs index 0590cab06..8158291df 100644 --- a/src/lib/admin/price-catalog.mjs +++ b/src/lib/admin/price-catalog.mjs @@ -593,7 +593,7 @@ function hasExactLayer(platform, id, catalog) { ) } -export function resolvePrice(rawModel) { +export function resolvePrice(rawModel, { refresh = true } = {}) { const platform = platformOfModel(rawModel) const candidates = catalogIdCandidates(rawModel, platform) const catalog = loadCatalog() @@ -610,7 +610,7 @@ export function resolvePrice(rawModel) { const got = candidates.length ? resolveLayers(p, candidates[0], catalog) : null if (got) return got } - maybeRefreshModelsDev() + if (refresh) maybeRefreshModelsDev() return null } diff --git a/src/lib/admin/pricing.mjs b/src/lib/admin/pricing.mjs index 1869b3592..c2b8724d5 100644 --- a/src/lib/admin/pricing.mjs +++ b/src/lib/admin/pricing.mjs @@ -179,9 +179,14 @@ function selectBand(price, mode, prompt) { * After normalizeUsage, input_tokens is the uncached bucket for every vendor. */ export function calculateCost(usage = {}, model = null) { + const mid = model || usage?.model || usage?.upstream_model || usage?.requested_model || '' + return calculateCostAtPrice(usage, model, resolvePrice(mid)) +} + +/** Same billing bands, using a previously captured price rather than the current catalog. */ +export function calculateCostAtPrice(usage = {}, model = null, resolved = null) { const u = normalizeUsage(usage) const mid = model || u.model || u.upstream_model || u.requested_model || '' - const resolved = resolvePrice(mid) const outputTokens = n(u.output_tokens) const cacheRead = n(u.cache_read_tokens) const { cache5m, cache1h } = cacheSplit(u) diff --git a/src/lib/core/ha-request-billing.mjs b/src/lib/core/ha-request-billing.mjs new file mode 100644 index 000000000..99afa78fc --- /dev/null +++ b/src/lib/core/ha-request-billing.mjs @@ -0,0 +1,95 @@ +import { catalogIdCandidates, resolvePrice, validatePrice } from '../admin/price-catalog.mjs' +import { calculateCostAtPrice } from '../admin/pricing.mjs' +import { applyCacheTtlToUsage } from '../protocol/cache-ttl.mjs' +import { normalizeNodeReceipt } from './node-execution-proof.mjs' + +function invalid() { + throw Object.assign(new Error('invalid HA billing snapshot'), { code: 'ha_billing_snapshot_invalid' }) +} +function text(value) { + if (typeof value !== 'string' || !value || value.length > 256 || /[\x00-\x1f\x7f]/.test(value)) invalid() + return value +} + +export function normalizeHaBillingSnapshot(value) { + if ( + !value || + value.version !== 1 || + Object.keys(value).some( + (key) => !['version', 'model', 'rate_multiplier', 'cache_ttl', 'requested_speed', 'resolved'].includes(key), + ) || + !Number.isFinite(value.rate_multiplier) || + value.rate_multiplier < 0 || + ![null, '5m', '1h'].includes(value.cache_ttl) || + ![null, 'standard', 'fast'].includes(value.requested_speed) + ) + invalid() + const result = { + version: 1, + model: text(value.model), + rate_multiplier: value.rate_multiplier, + cache_ttl: value.cache_ttl, + requested_speed: value.requested_speed, + resolved: null, + } + if (value.resolved !== null) { + const resolved = value.resolved + if ( + !resolved || + resolved.platform !== 'anthropic' || + Object.keys(resolved).some( + (key) => !['id', 'platform', 'source', 'version', 'mode_sources', 'price'].includes(key), + ) + ) + invalid() + const validated = validatePrice(resolved.price, { platform: 'anthropic' }) + if (!validated.ok) invalid() + result.resolved = { + id: text(resolved.id), + platform: 'anthropic', + source: resolved.source == null ? null : text(resolved.source), + version: resolved.version == null ? null : text(resolved.version), + mode_sources: {}, + price: validated.price, + } + } + return result +} + +/** Called synchronously for the actual outbound body, before its job bind permits node submission. */ +export function captureHaBillingSnapshot({ model, rateMultiplier = 1, cacheTtl = null, requestedSpeed = null }) { + return normalizeHaBillingSnapshot({ + version: 1, + model, + rate_multiplier: rateMultiplier, + cache_ttl: cacheTtl, + requested_speed: requestedSpeed, + resolved: resolvePrice(model, { refresh: false }), + }) +} + +export function priceHaReceipt(receipt, snapshot) { + try { + const observed = normalizeNodeReceipt(receipt) + const captured = normalizeHaBillingSnapshot(snapshot) + if ( + !observed.usage_complete || + !captured.resolved || + !catalogIdCandidates(observed.model, 'anthropic').includes(captured.resolved.id) + ) + return { known: false } + const usage = { + ...observed.usage, + ...(observed.speed ? { speed: observed.speed } : {}), + ...(observed.service_tier ? { service_tier: observed.service_tier } : {}), + requested_speed: captured.requested_speed, + } + const effective = captured.cache_ttl ? applyCacheTtlToUsage(usage, captured.cache_ttl) : usage + const pricing = calculateCostAtPrice(effective, observed.model, captured.resolved) + const actualCost = pricing.total_cost * captured.rate_multiplier + if (!pricing.known || !Number.isFinite(pricing.total_cost) || !Number.isFinite(actualCost)) return { known: false } + return { known: true, pricing, actualCost } + } catch { + return { known: false } + } +} diff --git a/src/lib/db/postgres-migrations/021_ha_request_billing.sql b/src/lib/db/postgres-migrations/021_ha_request_billing.sql new file mode 100644 index 000000000..e40a35e83 --- /dev/null +++ b/src/lib/db/postgres-migrations/021_ha_request_billing.sql @@ -0,0 +1,24 @@ +ALTER TABLE execution_jobs ADD COLUMN billing_snapshot jsonb; +ALTER TABLE execution_jobs ADD COLUMN node_receipt jsonb; +ALTER TABLE ha_request_contexts ADD COLUMN billable_command_id text COLLATE "C"; + +-- Keep the audit when a key or short-lived admission/job history is deleted. +-- Their exact IDs and original node evidence are copied while those rows are locked. +CREATE TABLE ha_request_settlements ( + ha_request_id text COLLATE "C" PRIMARY KEY REFERENCES ha_request_contexts(id), + reservation_id text COLLATE "C" NOT NULL UNIQUE, + key_id text COLLATE "C" NOT NULL, + user_id text COLLATE "C", + command_id text COLLATE "C", + node_binding jsonb, + node_receipt jsonb, + billing_snapshot jsonb, + outcome text NOT NULL CHECK (outcome IN ('charged','key_deleted','already_settled','not_dispatched')), + official_cost double precision, + actual_cost double precision, + tokens_in bigint, + tokens_out bigint, + cache_read_tokens bigint, + cache_creation_tokens bigint, + settled_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index a6de91281..4dc2256d4 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -5,6 +5,7 @@ import { readWatchdogState } from '../../transport/kernel-watchdog-state.mjs' import { ExecutionCircuitsRepo } from './execution-circuits-repo.mjs' import { normalizeNodeBinding, normalizeNodeCompletion, sameNodeBinding } from '../../core/node-execution-proof.mjs' import { lockHaRequestContext } from '../../core/ha-request-context.mjs' +import { normalizeHaBillingSnapshot } from '../../core/ha-request-billing.mjs' function invalid(field) { return Object.assign(new Error(`Invalid execution admission ${field}`), { code: 'invalid_execution_admission' }) @@ -212,6 +213,8 @@ export class ExecutionAdmissionsRepo { async bindExecution(handle, execution) { const nodeBinding = execution?.nodeBinding === undefined ? null : normalizeNodeBinding(execution.nodeBinding) + const billingSnapshot = + execution?.billingSnapshot == null ? null : normalizeHaBillingSnapshot(execution.billingSnapshot) return withTransaction(this.db, async () => { const context = await lockHaRequestContext(this.db) const before = await this.db @@ -255,7 +258,7 @@ export class ExecutionAdmissionsRepo { .get(handle.admissionId) if (pending) throw jobError('execution_job_pending') const row = await this.db - .prepare(`INSERT INTO execution_jobs (request_id, admission_id, node_binding, ha_request_id, attempt_no) VALUES ($1, $2, $3::jsonb, $4, $5) + .prepare(`INSERT INTO execution_jobs (request_id, admission_id, node_binding, ha_request_id, attempt_no, billing_snapshot) VALUES ($1, $2, $3::jsonb, $4, $5, $6::jsonb) ON CONFLICT (request_id) DO NOTHING RETURNING request_id`) .get( execution.requestId, @@ -263,6 +266,7 @@ export class ExecutionAdmissionsRepo { nodeBinding ? JSON.stringify(nodeBinding) : null, context?.requestId ?? null, context?.attemptNo ?? null, + context && billingSnapshot ? JSON.stringify(billingSnapshot) : null, ) if (!row) throw jobError('execution_job_exists') return { diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index 2f6bc153f..b46995b07 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 43) + assert.equal(rows.length, 44) for (const name of [ 'settings', 'users', @@ -53,6 +53,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'execution_seats', 'execution_jobs', 'ha_request_contexts', + 'ha_request_settlements', 'session_turns', 'vm_operations', 'background_task_runs', @@ -118,7 +119,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 20) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 21) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -134,7 +135,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 20) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 21) fs.rmSync(dir, { recursive: true, force: true }) }) From 753e92f464e2cd3260634186c068ccedf0ac5040 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:24:50 +0800 Subject: [PATCH 430/473] test: specify idempotent HA key settlement from original node receipts --- test/fixtures/ha-billing.mjs | 4 +- test/unit/ha-request-billing.test.mjs | 235 ++++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 1 deletion(-) diff --git a/test/fixtures/ha-billing.mjs b/test/fixtures/ha-billing.mjs index 795b932ab..9e99dc5bc 100644 --- a/test/fixtures/ha-billing.mjs +++ b/test/fixtures/ha-billing.mjs @@ -48,7 +48,9 @@ export async function billingRequest(f, callback = async () => {}, { snapshot = const data = { nodeBinding, commandIds: [] } await f.originalGate.run(() => withHaRequestContext({ db: f.db, requestId: 'billing-parent', activity: f.originalGate, recoveryKind: 'claude_messages' }, async () => { data.keys = new ApiKeyStore({ db: f.db }) - data.key = await data.keys.create({ name: 'synthetic-billing-key' }) + const now = new Date().toISOString() + data.user = await data.keys.users.insert({ id: 'synthetic-billing-user', username: 'synthetic-billing-user', password_hash: 'synthetic-test-only', role: 'user', balance: 123, concurrency: 4, created_at: now, updated_at: now }) + data.key = await data.keys.create({ name: 'synthetic-billing-key', user_id: data.user.id }) data.reservation = await data.keys.reserve(data.key) data.admission = await new ExecutionAdmissionsRepo(f.db).tryReserve({ accountId: 'fixture-account', vmId: 'vm-fixture', accountLimit: 5, vmLimit: 5, fableLimit: 0, rpmLimit: 0 }) for (let attempt = 1; attempt <= jobs; attempt++) { diff --git a/test/unit/ha-request-billing.test.mjs b/test/unit/ha-request-billing.test.mjs index 535c1a2cc..8b7e84326 100644 --- a/test/unit/ha-request-billing.test.mjs +++ b/test/unit/ha-request-billing.test.mjs @@ -5,7 +5,9 @@ import { billingRequest, syntheticBillingSnapshot, syntheticUsageReceipt, + statusFor, } from '../fixtures/ha-billing.mjs' +import { HaRequestRecovery } from '../../src/lib/core/ha-request-recovery.mjs' import { calculateCost } from '../../src/lib/admin/pricing.mjs' import { setPriceCatalogTestHooks, invalidatePriceCatalog } from '../../src/lib/admin/price-catalog.mjs' @@ -82,3 +84,236 @@ for (const absent of ['usage', 'price', 'different-model']) { assert.equal(priceHaReceipt(receipt, snapshot).known, false) }) } + +async function reconcile(f, data, receipt = syntheticUsageReceipt()) { + const recovery = new HaRequestRecovery(f.peer, { coordinator: f.recovery }) + return f.recoveryGate.run(() => + recovery.recover('billing-parent', { + readRecoveryStatus: async ({ requestId }) => statusFor(data, requestId, receipt), + }), + ) +} + +test('two PG peers settle a recovered single billable command once from its original frozen rates', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f) + const first = await reconcile(f, data) + assert.equal(first.recovered, true) + assert.equal(first.billingHeld, false) + await reconcile(f, data) + await f.recoveryGate.run(async () => { + const key = await new (await import('../../src/lib/admin/api-keys.mjs')).ApiKeyStore({ db: f.peer }).getById( + data.key.id, + ) + assert.equal(key.requests, 1) + assert.equal(key.tokens_in, 100) + assert.ok(Math.abs(key.quota_used - 0.001272) < 1e-12) + const ledger = await f.peer + .prepare('SELECT * FROM ha_request_settlements WHERE ha_request_id=$1') + .get('billing-parent') + assert.equal(ledger.outcome, 'charged') + assert.equal(ledger.command_id, data.commandIds[0]) + assert.equal(ledger.reservation_id, data.reservation.reservationId) + assert.deepEqual(ledger.node_receipt, syntheticUsageReceipt()) + assert.ok( + ( + await f.peer + .prepare('SELECT settled_at,released_at FROM api_key_admissions WHERE id=$1') + .get(data.reservation.reservationId) + ).released_at, + ) + assert.equal((await f.peer.prepare('SELECT balance FROM users WHERE id=$1').get(data.user.id)).balance, 123) + assert.equal( + ( + await f.peer + .prepare('SELECT count(*) AS n FROM api_key_admissions WHERE user_id=$1 AND released_at IS NULL') + .get(data.user.id) + ).n, + 0, + ) + }) +}) + +test('normal HA handler settlement and later orphan retry share one immutable ledger', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async (current) => { + const outcome = await current.keys.recordHaUsage({ + reservation: current.reservation, + commandId: current.commandIds[0], + nodeStatus: statusFor(current, current.commandIds[0]), + }) + assert.equal(outcome.settled, true) + assert.equal(current.reservation.settled, true) + assert.equal( + ( + await f.db + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(current.reservation.reservationId) + ).released_at, + null, + ) + }) + const recovered = await reconcile(f, data) + assert.equal(recovered.billingHeld, false) + await f.recoveryGate.run(async () => { + assert.equal((await f.peer.prepare('SELECT requests FROM api_keys WHERE id=$1').get(data.key.id)).requests, 1) + assert.equal((await f.peer.prepare('SELECT count(*) AS n FROM ha_request_settlements').get()).n, 1) + }) +}) + +test('a deleted key still gets a durable settlement audit and releases only its original reservation', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async (current) => { + await current.keys.remove(current.key.id) + }) + assert.equal((await reconcile(f, data)).billingHeld, false) + assert.equal((await reconcile(f, data)).billingHeld, false) + await f.recoveryGate.run(async () => { + const ledger = await f.peer + .prepare('SELECT * FROM ha_request_settlements WHERE ha_request_id=$1') + .get('billing-parent') + assert.equal(ledger.outcome, 'key_deleted') + assert.equal(ledger.key_id, data.key.id) + assert.ok(ledger.actual_cost > 0) + assert.equal(ledger.tokens_in, 100) + assert.equal((await f.peer.prepare('SELECT balance FROM users WHERE id=$1').get(data.user.id)).balance, 123) + assert.ok( + ( + await f.peer + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(data.reservation.reservationId) + ).released_at, + ) + }) +}) + +test('a legacy settled reservation is audited and released without a second key increment', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async (current) => { + await current.keys.recordUsage( + current.key, + { input_tokens: 33 }, + { officialCost: 9, reservation: current.reservation }, + ) + }) + assert.equal((await reconcile(f, data)).billingHeld, false) + await f.recoveryGate.run(async () => { + const key = await f.peer.prepare('SELECT requests,tokens_in,quota_used FROM api_keys WHERE id=$1').get(data.key.id) + assert.equal(key.requests, 1) + assert.equal(key.tokens_in, 33) + assert.equal(key.quota_used, 9) + const ledger = await f.peer + .prepare('SELECT * FROM ha_request_settlements WHERE ha_request_id=$1') + .get('billing-parent') + assert.equal(ledger.outcome, 'already_settled') + assert.equal(ledger.actual_cost, null, 'prior aggregate key quota cannot reconstruct this reservation charge') + }) +}) + +test('bigint node counts reach key and ledger counters without Number rounding', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f) + const receipt = syntheticUsageReceipt() + receipt.usage.input_tokens = '9007199254740993' + assert.equal((await reconcile(f, data, receipt)).billingHeld, false) + await f.recoveryGate.run(async () => { + assert.equal( + (await f.peer.prepare('SELECT tokens_in FROM api_keys WHERE id=$1').get(data.key.id)).tokens_in, + '9007199254740993', + ) + assert.equal( + ( + await f.peer + .prepare('SELECT tokens_in FROM ha_request_settlements WHERE ha_request_id=$1') + .get('billing-parent') + ).tokens_in, + '9007199254740993', + ) + }) +}) + +for (const absent of ['usage', 'price', 'model']) { + test(`orphan ${absent} evidence is held after execution ends and cannot create a zero settlement`, async (t) => { + const f = await haBillingFixture(t) + const snapshot = syntheticBillingSnapshot() + if (absent === 'price') snapshot.resolved = null + const data = await billingRequest(f, async () => {}, { snapshot }) + const receipt = syntheticUsageReceipt() + if (absent === 'usage') { + receipt.usage_complete = false + delete receipt.usage.output_tokens + } + if (absent === 'model') receipt.model = 'claude-opus-4-6' + const result = await reconcile(f, data, receipt) + assert.equal(result.recovered, true) + assert.equal(result.billingHeld, true) + await f.recoveryGate.run(async () => { + assert.equal((await f.peer.prepare('SELECT count(*) AS n FROM ha_request_settlements').get()).n, 0) + assert.equal((await f.peer.prepare('SELECT requests FROM api_keys WHERE id=$1').get(data.key.id)).requests, 0) + assert.equal( + ( + await f.peer + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(data.reservation.reservationId) + ).released_at, + null, + ) + }) + }) +} + +test('multiple commands without a durable billable choice stay held rather than summing or selecting latest', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async () => {}, { jobs: 2 }) + assert.equal((await reconcile(f, data)).billingHeld, true) + await f.recoveryGate.run(async () => { + assert.equal((await f.peer.prepare('SELECT requests FROM api_keys WHERE id=$1').get(data.key.id)).requests, 0) + assert.equal((await f.peer.prepare('SELECT count(*) AS n FROM ha_request_settlements').get()).n, 0) + }) +}) + +test('an explicit final command choice bills only that original command in a multiattempt request', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest( + f, + async (current) => { + const { selectHaBillableCommand } = await import('../../src/lib/core/ha-request-billing.mjs') + await selectHaBillableCommand(f.db, current.commandIds[1]) + }, + { jobs: 2 }, + ) + assert.equal((await reconcile(f, data)).billingHeld, false) + await f.recoveryGate.run(async () => { + const ledger = await f.peer + .prepare('SELECT * FROM ha_request_settlements WHERE ha_request_id=$1') + .get('billing-parent') + assert.equal(ledger.command_id, data.commandIds[1]) + assert.equal(ledger.tokens_in, 100) + assert.equal((await f.peer.prepare('SELECT requests FROM api_keys WHERE id=$1').get(data.key.id)).requests, 1) + }) +}) + +test('a sealed request with no bound jobs records not_dispatched and releases unused quota without a charge', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async () => {}, { jobs: 0 }) + assert.equal((await reconcile(f, data)).billingHeld, false) + await f.recoveryGate.run(async () => { + const ledger = await f.peer + .prepare('SELECT * FROM ha_request_settlements WHERE ha_request_id=$1') + .get('billing-parent') + assert.equal(ledger.outcome, 'not_dispatched') + assert.equal(ledger.command_id, null) + assert.equal(ledger.actual_cost, null) + assert.equal( + (await f.peer.prepare('SELECT requests,quota_requests_used FROM api_keys WHERE id=$1').get(data.key.id)).requests, + 0, + ) + assert.ok( + ( + await f.peer + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(data.reservation.reservationId) + ).released_at, + ) + }) +}) From 769a384006f431c5e4e26ff79823298ca504f988 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:29:29 +0800 Subject: [PATCH 431/473] feat: reconcile HA key quotas once from exact command receipts --- src/lib/admin/api-keys.mjs | 5 + src/lib/core/ha-request-billing.mjs | 227 +++++++++++++++++++++++++ src/lib/core/ha-request-recovery.mjs | 25 ++- test/unit/ha-request-billing.test.mjs | 23 +++ test/unit/ha-request-recovery.test.mjs | 2 +- 5 files changed, 276 insertions(+), 6 deletions(-) diff --git a/src/lib/admin/api-keys.mjs b/src/lib/admin/api-keys.mjs index 5acb50b47..201e082b9 100644 --- a/src/lib/admin/api-keys.mjs +++ b/src/lib/admin/api-keys.mjs @@ -26,6 +26,7 @@ import { calculateCost, normalizeUsage } from './pricing.mjs' import { UsersRepo } from '../db/repos/users-repo.mjs' import { ApiKeyAdmissionsRepo } from '../db/repos/api-key-admissions-repo.mjs' import { isKubernetesMode } from '../core/control-plane-mode.mjs' +import { recordHaRequestUsage } from '../core/ha-request-billing.mjs' const KEY_PREFIX = 'sk-vm-' const HASH_MARKER = 'hmac:' @@ -588,6 +589,10 @@ export class ApiKeyStore { return write() } + recordHaUsage(options) { + return recordHaRequestUsage(this.db, options) + } + async snapshot() { const keys = await this.list({ reveal: false }) return { diff --git a/src/lib/core/ha-request-billing.mjs b/src/lib/core/ha-request-billing.mjs index 99afa78fc..9463cedce 100644 --- a/src/lib/core/ha-request-billing.mjs +++ b/src/lib/core/ha-request-billing.mjs @@ -2,6 +2,11 @@ import { catalogIdCandidates, resolvePrice, validatePrice } from '../admin/price import { calculateCostAtPrice } from '../admin/pricing.mjs' import { applyCacheTtlToUsage } from '../protocol/cache-ttl.mjs' import { normalizeNodeReceipt } from './node-execution-proof.mjs' +import { normalizeNodeCompletion } from './node-execution-proof.mjs' +import { getActivityCoordinator, getDb, withTransaction } from '../db/database.mjs' +import { lockHaRequestContext, currentHaRequestContext } from './ha-request-context.mjs' +import { ApiKeysRepo } from '../db/repos/api-keys-repo.mjs' +import { isDeepStrictEqual } from 'node:util' function invalid() { throw Object.assign(new Error('invalid HA billing snapshot'), { code: 'ha_billing_snapshot_invalid' }) @@ -93,3 +98,225 @@ export function priceHaReceipt(receipt, snapshot) { return { known: false } } } + +function failure(code) { + return Object.assign(new Error(code.replaceAll('_', ' ')), { code }) +} + +/** A final handler decision, not an ordering heuristic over failed attempts. */ +export async function selectHaBillableCommand(db, commandId) { + return withTransaction(db, async () => { + const context = await lockHaRequestContext(db) + if (!context) throw failure('ha_request_context_required') + const job = await db + .prepare(`SELECT j.request_id FROM execution_jobs j + JOIN execution_admissions a ON a.id=j.admission_id AND a.ha_request_id=j.ha_request_id + JOIN request_attempts t ON t.ha_request_id=j.ha_request_id AND t.attempt_no=j.attempt_no + AND t.vm_id=a.vm_id AND t.account_id=a.account_id + WHERE j.request_id=$1 AND j.ha_request_id=$2`) + .get(commandId, context.requestId) + if (!job) throw failure('ha_billable_command_mismatch') + const row = await db + .prepare(`UPDATE ha_request_contexts SET billable_command_id=$2 + WHERE id=$1 AND (billable_command_id IS NULL OR billable_command_id=$2) RETURNING id`) + .get(context.requestId, commandId) + if (!row) throw failure('ha_billable_command_mismatch') + return { selected: true } + }) +} + +/** One ledger row and one counter increment, under the same parent/key/reservation transaction. */ +export class HaRequestBilling { + constructor(db = getDb()) { + this.db = db + this.keys = new ApiKeysRepo(db) + } + + async reconcile(requestId, { readRecoveryStatus } = {}) { + let result = await this.settle(requestId, { source: 'recovery' }) + if (!result.query || typeof readRecoveryStatus !== 'function') return result + let nodeStatus + try { + nodeStatus = await readRecoveryStatus(result.query) + } catch { + return { settled: false, reason: 'node_receipt_unavailable' } + } + result = await this.settle(requestId, { source: 'recovery', commandId: result.query.requestId, nodeStatus }) + return result + } + + async settle(requestId, { source, commandId, nodeStatus, reservation: handle } = {}) { + return withTransaction(this.db, async () => { + if (source === 'handler') { + const context = await lockHaRequestContext(this.db) + if (context?.requestId !== requestId) throw failure('ha_request_context_mismatch') + } else if (source !== 'recovery') throw failure('ha_billing_source_invalid') + const request = await this.db.prepare('SELECT * FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get(requestId) + if (!request || request.recovery_kind !== 'claude_messages') + return { settled: false, reason: 'request_kind_unproven' } + if (source === 'recovery') { + if (!request.recovery_sealed_at || !request.execution_recovered_at || request.accepting_children) + return { settled: false, reason: 'ha_execution_unproven' } + const coordinator = getActivityCoordinator(this.db) + if (!coordinator) throw failure('ha_recovery_coordinator_required') + const absent = await coordinator.lockAbsentOwner(request.activity_owner_id) + if (!absent.absent) return { settled: false, reason: absent.reason } + } + const prior = await this.db.prepare('SELECT * FROM ha_request_settlements WHERE ha_request_id=$1').get(requestId) + if (prior) { + if (handle && (prior.reservation_id !== handle.reservationId || prior.command_id !== commandId)) + throw failure('invalid_api_key_reservation') + if (handle) { + const original = await this.db + .prepare('SELECT owner_id FROM api_key_admissions WHERE id=$1 AND ha_request_id=$2') + .get(prior.reservation_id, requestId) + if (original?.owner_id !== handle.ownerId) throw failure('invalid_api_key_reservation') + } + if (source === 'recovery') + await this.db + .prepare( + 'UPDATE api_key_admissions SET released_at=COALESCE(released_at,clock_timestamp()) WHERE id=$1 AND ha_request_id=$2 AND key_id=$3', + ) + .run(prior.reservation_id, requestId, prior.key_id) + const priced = + prior.node_receipt && prior.billing_snapshot + ? priceHaReceipt(prior.node_receipt, prior.billing_snapshot) + : null + return { + settled: true, + outcome: prior.outcome, + pricing: priced?.known ? priced.pricing : null, + record: await this.keys.getById(prior.key_id), + } + } + const before = await this.db.prepare('SELECT * FROM api_key_admissions WHERE ha_request_id=$1').get(requestId) + if (!before) return { settled: true, outcome: 'no_managed_reservation' } + // Same key -> reservation order as the legacy settle path. The parent is always first. + await this.db.prepare('SELECT id FROM api_keys WHERE id=$1 FOR UPDATE').get(before.key_id) + const reservation = await this.db + .prepare('SELECT * FROM api_key_admissions WHERE id=$1 AND ha_request_id=$2 FOR UPDATE') + .get(before.id, requestId) + if ( + !reservation || + reservation.key_id !== before.key_id || + (handle && (handle.reservationId !== reservation.id || handle.ownerId !== reservation.owner_id)) + ) + throw failure('invalid_api_key_reservation') + const jobs = await this.db + .prepare('SELECT * FROM execution_jobs WHERE ha_request_id=$1 ORDER BY request_id') + .all(requestId) + let chosen = request.billable_command_id + let outcome, + priced = null, + job = null, + observed = null + if (reservation.settled_at) outcome = 'already_settled' + else if (!jobs.length && source === 'recovery') outcome = 'not_dispatched' + else { + if (reservation.released_at) return { settled: false, reason: 'api_key_reservation_released' } + if (!chosen && source === 'recovery' && jobs.length === 1) { + chosen = jobs[0].request_id + await this.db + .prepare('UPDATE ha_request_contexts SET billable_command_id=$2 WHERE id=$1') + .run(requestId, chosen) + } + if (!chosen || (commandId && chosen !== commandId)) + return { settled: false, reason: 'ha_billable_command_unproven' } + job = jobs.find((row) => row.request_id === chosen) + if (!job?.ended_at || !job.node_binding || !job.node_completion) + return { settled: false, reason: 'node_completion_unproven' } + const linked = await this.db + .prepare(`SELECT a.id FROM execution_admissions a JOIN request_attempts t + ON t.ha_request_id=a.ha_request_id AND t.attempt_no=$3 AND t.vm_id=a.vm_id AND t.account_id=a.account_id + WHERE a.id=$1 AND a.ha_request_id=$2`) + .get(job.admission_id, requestId, job.attempt_no) + if (!linked) return { settled: false, reason: 'request_job_binding_unproven' } + try { + normalizeNodeCompletion(job.node_completion, { requestId: chosen, nodeBinding: job.node_binding }) + if (nodeStatus) { + if (nodeStatus.found !== true) return { settled: false, reason: 'node_receipt_unavailable' } + const completion = normalizeNodeCompletion(nodeStatus.command, { + requestId: chosen, + nodeBinding: job.node_binding, + }) + if (!isDeepStrictEqual(completion, job.node_completion)) + return { settled: false, reason: 'node_completion_unproven' } + observed = normalizeNodeReceipt(nodeStatus.receipt) + if (job.node_receipt && !isDeepStrictEqual(observed, job.node_receipt)) + return { settled: false, reason: 'node_receipt_mismatch' } + } else if (job.node_receipt) observed = normalizeNodeReceipt(job.node_receipt) + else + return { + settled: false, + reason: 'node_receipt_required', + query: { requestId: chosen, nodeBinding: structuredClone(job.node_binding) }, + } + } catch { + return { settled: false, reason: 'node_receipt_unproven' } + } + if (nodeStatus) + await this.db + .prepare('UPDATE execution_jobs SET node_receipt=$2::jsonb WHERE request_id=$1') + .run(chosen, JSON.stringify(observed)) + priced = priceHaReceipt(observed, job.billing_snapshot) + if (!priced.known) return { settled: false, reason: 'ha_usage_unpriced' } + const record = await this.keys.recordUsage(reservation.key_id, { + tokens_in: observed.usage.input_tokens, + tokens_out: observed.usage.output_tokens, + cache_read_tokens: observed.usage.cache_read_input_tokens, + cache_creation_tokens: observed.usage.cache_creation_input_tokens, + cost: priced.actualCost, + }) + outcome = record ? 'charged' : 'key_deleted' + } + await this.db + .prepare(`INSERT INTO ha_request_settlements(ha_request_id,reservation_id,key_id,user_id,command_id,node_binding,node_receipt,billing_snapshot,outcome,official_cost,actual_cost,tokens_in,tokens_out,cache_read_tokens,cache_creation_tokens) + VALUES($1,$2,$3,$4,$5,$6::jsonb,$7::jsonb,$8::jsonb,$9,$10,$11,$12,$13,$14,$15)`) + .run( + requestId, + reservation.id, + reservation.key_id, + reservation.user_id, + job?.request_id ?? chosen ?? null, + job?.node_binding ? JSON.stringify(job.node_binding) : null, + observed ? JSON.stringify(observed) : null, + job?.billing_snapshot ? JSON.stringify(job.billing_snapshot) : null, + outcome, + priced?.known ? priced.pricing.total_cost : null, + priced?.known ? priced.actualCost : null, + observed?.usage.input_tokens ?? null, + observed?.usage.output_tokens ?? null, + observed?.usage.cache_read_input_tokens ?? null, + observed?.usage.cache_creation_input_tokens ?? null, + ) + await this.db + .prepare(`UPDATE api_key_admissions SET + settled_at=CASE WHEN $3='not_dispatched' THEN settled_at ELSE COALESCE(settled_at,clock_timestamp()) END, + released_at=CASE WHEN $4 THEN COALESCE(released_at,clock_timestamp()) ELSE released_at END + WHERE id=$1 AND ha_request_id=$2`) + .run(reservation.id, requestId, outcome, source === 'recovery') + return { + settled: true, + outcome, + pricing: priced?.known ? priced.pricing : null, + record: await this.keys.getById(reservation.key_id), + } + }) + } +} + +export async function recordHaRequestUsage(db, { reservation, commandId, nodeStatus }) { + if (reservation?.shared !== true) throw failure('invalid_api_key_reservation') + const context = currentHaRequestContext(db) + if (!context) throw failure('ha_request_context_required') + await selectHaBillableCommand(db, commandId) + const result = await new HaRequestBilling(db).settle(context.requestId, { + source: 'handler', + commandId, + nodeStatus, + reservation, + }) + if (result.settled) reservation?._markSettled?.() + else if (reservation) reservation.billingFailed = true + return result +} diff --git a/src/lib/core/ha-request-recovery.mjs b/src/lib/core/ha-request-recovery.mjs index f805430e7..6eafc9069 100644 --- a/src/lib/core/ha-request-recovery.mjs +++ b/src/lib/core/ha-request-recovery.mjs @@ -3,6 +3,7 @@ import { isDeepStrictEqual } from 'node:util' import { normalizeNodeBinding, normalizeNodeCompletion } from './node-execution-proof.mjs' import { SharedExecutionAdmissions } from '../pool/shared-execution-admissions.mjs' import { SharedSessionTurns } from '../pool/shared-session-turns.mjs' +import { HaRequestBilling } from './ha-request-billing.mjs' function unchangedGraph(graph) { return { @@ -20,10 +21,11 @@ function unchangedGraph(graph) { /** Explicit recovery only. Closing a parent proves no late child can be admitted, not that a node ended. */ export class HaRequestRecovery { - constructor(db = getDb(), { coordinator = getActivityCoordinator(db) } = {}) { + constructor(db = getDb(), { coordinator = getActivityCoordinator(db), readNodeRecoveryStatus = null } = {}) { if (!coordinator || coordinator.db !== db) throw new Error('HA recovery requires its activity coordinator') this.db = db this.coordinator = coordinator + this.readNodeRecoveryStatus = readNodeRecoveryStatus } async seal(requestId) { @@ -117,7 +119,7 @@ export class HaRequestRecovery { } /** readCompletion authenticates the original node; it must never substitute current State or a new command. */ - async recover(requestId, { readCompletion } = {}) { + async recover(requestId, { readCompletion, readRecoveryStatus = this.readNodeRecoveryStatus } = {}) { const snapshot = await withTransaction(this.db, async () => { const sealed = await this._seal(requestId) if (!sealed.sealed) return { reason: sealed.reason } @@ -126,14 +128,21 @@ export class HaRequestRecovery { return this._validGraph(requestId, graph) ? { graph } : { reason: 'request_job_binding_unproven' } }) if (snapshot.reason) return { recovered: false, reason: snapshot.reason } - if (snapshot.already) return { recovered: true, billingHeld: await this._billingHeld(requestId) } + if (snapshot.already) return this._bill(requestId, readRecoveryStatus) const proofs = new Map() for (const job of snapshot.graph.jobs) { try { const value = job.ended_at && job.node_completion ? job.node_completion - : await readCompletion?.({ requestId: job.request_id, nodeBinding: structuredClone(job.node_binding) }) + : readRecoveryStatus + ? ( + await readRecoveryStatus({ + requestId: job.request_id, + nodeBinding: structuredClone(job.node_binding), + }) + )?.command + : await readCompletion?.({ requestId: job.request_id, nodeBinding: structuredClone(job.node_binding) }) proofs.set( job.request_id, normalizeNodeCompletion(value, { requestId: job.request_id, nodeBinding: job.node_binding }), @@ -142,7 +151,7 @@ export class HaRequestRecovery { return { recovered: false, reason: 'node_completion_unproven' } } } - return withTransaction(this.db, async () => { + const result = await withTransaction(this.db, async () => { const sealed = await this._seal(requestId) if (!sealed.sealed) return { recovered: false, reason: sealed.reason } if (sealed.request.execution_recovered_at) @@ -219,5 +228,11 @@ export class HaRequestRecovery { .run(requestId) return { recovered: true, billingHeld: await this._billingHeld(requestId) } }) + return result.recovered ? this._bill(requestId, readRecoveryStatus) : result + } + + async _bill(requestId, readRecoveryStatus) { + const billing = await new HaRequestBilling(this.db).reconcile(requestId, { readRecoveryStatus }) + return { recovered: true, billingHeld: !billing.settled, billing } } } diff --git a/test/unit/ha-request-billing.test.mjs b/test/unit/ha-request-billing.test.mjs index 8b7e84326..355ee5ecf 100644 --- a/test/unit/ha-request-billing.test.mjs +++ b/test/unit/ha-request-billing.test.mjs @@ -317,3 +317,26 @@ test('a sealed request with no bound jobs records not_dispatched and releases un ) }) }) + +test('a PostgreSQL counter overflow rolls back settlement and keeps the exact reservation held', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async (current) => { + await f.db.prepare('UPDATE api_keys SET tokens_in=$2 WHERE id=$1').run(current.key.id, '9223372036854775807') + }) + await assert.rejects(reconcile(f, data), { code: '22003' }) + await f.recoveryGate.run(async () => { + assert.equal((await f.peer.prepare('SELECT count(*) AS n FROM ha_request_settlements').get()).n, 0) + assert.equal( + ( + await f.peer + .prepare('SELECT released_at,settled_at FROM api_key_admissions WHERE id=$1') + .get(data.reservation.reservationId) + ).released_at, + null, + ) + assert.equal( + (await f.peer.prepare('SELECT tokens_in FROM api_keys WHERE id=$1').get(data.key.id)).tokens_in, + '9223372036854775807', + ) + }) +}) diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs index dd0ff5ee3..c3928c5a7 100644 --- a/test/unit/ha-request-recovery.test.mjs +++ b/test/unit/ha-request-recovery.test.mjs @@ -341,7 +341,7 @@ test('sealed supported requests with no bound node jobs can release execution an readCompletion: async () => assert.fail('no original node job'), }) assert.equal(outcome.recovered, true) - assert.equal(outcome.billingHeld, true) + assert.equal(outcome.billingHeld, false) assert.ok( ( await f.peer From 00f15d25c06e60398400aa4c5ab8505304cf78d6 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:31:00 +0800 Subject: [PATCH 432/473] test: require HA protocol to capture and settle original dispatch evidence --- test/fixtures/ha-protocol-billing.mjs | 97 ++++++++++++++++++++++++++ test/unit/ha-protocol-billing.test.mjs | 19 +++++ 2 files changed, 116 insertions(+) create mode 100644 test/fixtures/ha-protocol-billing.mjs create mode 100644 test/unit/ha-protocol-billing.test.mjs diff --git a/test/fixtures/ha-protocol-billing.mjs b/test/fixtures/ha-protocol-billing.mjs new file mode 100644 index 000000000..a2f5dfdfd --- /dev/null +++ b/test/fixtures/ha-protocol-billing.mjs @@ -0,0 +1,97 @@ +import { test, mock } from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { EventEmitter } from 'node:events' +import { Readable } from 'node:stream' +import { haBillingFixture, syntheticUsageReceipt } from './ha-billing.mjs' +import { syntheticFence } from './fake-slotgate.mjs' +import { withHaRequestContext, withHaRequestAttempt } from '../../src/lib/core/ha-request-context.mjs' +import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' +import { readBody } from '../../src/lib/http/respond.mjs' + +const kind = process.env.VM2API_HA_BILLING_CASE +const router = await import('../../src/lib/transport/kernel-router.mjs') +const identity = await import('../../src/lib/identity/vm-identity.mjs') +let dispatch +mock.module(new URL('../../src/lib/transport/kernel-router.mjs', import.meta.url), { namedExports: { ...router, dispatchStreamInference: (...args) => dispatch(...args) } }) +mock.module(new URL('../../src/lib/identity/vm-identity.mjs', import.meta.url), { namedExports: { ...identity, loadVmIdentity: () => ({ accountUuid: 'fixture-account', fingerprint: { device_id: 'synthetic-device' } }) } }) +const { createHandleProtocol } = await import('../../src/lib/protocol/handle-protocol.mjs') + +test(`HA protocol captures and settles original billable command: ${kind}`, async (t) => { + const f = await haBillingFixture(t) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vm2api-648-ha-billing-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const routing = { compatibility: { claude_code_only: false, cache_ttl: '1h' } } + const routingFile = path.join(root, 'routing.json') + fs.writeFileSync(routingFile, JSON.stringify(routing)) + const keys = new ApiKeyStore({ db: f.db }) + const key = await f.originalGate.run(() => keys.create({ name: 'synthetic-protocol-key' })) + const nodeBinding = { protocol: 'KSG2', nodeId: 'original-node', vmId: 'vm-fixture', kind: 'kernel', fence: syntheticFence() } + const commandId = 'protocol-original-command' + const execution = { requestId: commandId, nodeBinding, started: true, ended: true, reason: 'node_completed', nodeCompletion: { id: commandId, fence: nodeBinding.fence, status: 'completed', dispatched: true } } + const assistant = { id: 'synthetic-message', type: 'message', role: 'assistant', model: 'claude-sonnet-4-5-20250929', content: [{ type: 'text', text: 'synthetic' }], stop_reason: 'end_turn', usage: { input_tokens: 100, output_tokens: 10, cache_read_input_tokens: 20, cache_creation_input_tokens: 30 } } + let rateReads = 0, receiptReads = 0 + dispatch = async (opts) => { + await opts.onExecutionStart({ requestId: commandId, nodeBinding }) + const job = (await f.peer.pool.query('SELECT * FROM execution_jobs WHERE request_id=$1', [commandId])).rows[0] + assert.equal(job.billing_snapshot?.rate_multiplier, 2, 'dispatch permission requires the original PG-priced group snapshot') + assert.equal(job.billing_snapshot.cache_ttl, '1h') + assert.equal(job.billing_snapshot.model, opts.body.model) + if (kind === 'deleted-key') await keys.remove(key.id) + await opts.onExecutionEnd(execution) + return { ok: true, status: 200, body: structuredClone(assistant), usage: assistant.usage, model: assistant.model, execution } + } + const handler = createHandleProtocol({ + cfg: { limits: { max_body_bytes: 1 << 20, upstream_timeout_ms: 1000 }, rewrite: { enabled: false }, intercept: { rules: [] }, distill: { enabled: false }, paths: { project: root, data: root } }, + json(res, status, body) { res.statusCode = status; res.body = body }, readBody, + requireAuth(req) { req.apiKeyKind = 'managed'; req.apiKeyRecord = key; return true }, + requestLog: { start: () => ({ request_id: 'ignored-client-id', t0: Date.now() }) }, + stats: { errors: 0, requests: 0, by_route: {} }, + groupsRepo: { async rateMultiplier() { return ++rateReads === 1 ? 2 : 99 } }, + apiKeyStore: keys, stickyRouter: { resolve: () => null }, accountQuota: { ingestHeaders() {} }, routingConfig: routing, routingConfigPath: routingFile, + async readNodeRecoveryStatus(input) { + receiptReads++ + assert.equal(input.requestId, commandId) + assert.deepEqual(input.nodeBinding, nodeBinding) + const receipt = syntheticUsageReceipt() + if (kind === 'no-usage') { receipt.usage_complete = false; delete receipt.usage.output_tokens } + return { found: true, command: execution.nodeCompletion, receipt } + }, + failoverRunner: { async run(args) { + const admission = await new ExecutionAdmissionsRepo(f.db).tryReserve({ accountId: 'fixture-account', vmId: 'vm-fixture', accountLimit: 5, vmLimit: 5, fableLimit: 0, rpmLimit: 0 }) + const candidate = { ...admission, vm: { id: 'vm-fixture', claude: { has_access: true } }, exec: { vmId: 'vm-fixture', vm: {} } } + await new RequestAttemptsRepo(f.db).begin({ requestId: 'billing-parent', attemptNo: 1, vmId: 'vm-fixture', accountId: 'fixture-account', model: args.canonicalBody.model }) + try { + const prepared = await args.applyAttempt(structuredClone(args.canonicalBody), candidate, { attemptNo: 1, attemptStartedAt: Date.now() }) + const result = await withHaRequestAttempt(1, () => args.callAttempt({ candidate, body: prepared.body, attemptMeta: prepared.meta, onCommit() {} })) + return { ...result, vmId: 'vm-fixture', accountId: 'fixture-account', finalState: 'verified', attemptCount: 1 } + } finally { await admission.release() } + } }, + }) + const req = Readable.from([Buffer.from(JSON.stringify({ model: 'claude-sonnet-4-5', max_tokens: 20, messages: [{ role: 'user', content: 'synthetic static prompt' }] }))]) + req.headers = { 'user-agent': 'synthetic-ha-client' } + const res = new EventEmitter(); res.setHeader = () => {} + await f.originalGate.run(() => withHaRequestContext({ db: f.db, requestId: 'billing-parent', activity: f.originalGate, recoveryKind: 'claude_messages' }, () => handler.handleProtocol(req, res, 'anthropic.messages', '/v1/messages'))) + assert.equal(res.statusCode, 200, JSON.stringify(res.body)) + assert.equal(rateReads, 1, 'group rates must not be reloaded after dispatch for billing') + assert.equal(receiptReads, 1) + await f.originalGate.run(async () => { + const rows = await f.db.prepare('SELECT * FROM ha_request_settlements').all() + const admission = await f.db.prepare('SELECT * FROM api_key_admissions WHERE ha_request_id=$1').get('billing-parent') + if (kind === 'no-usage') { + assert.equal(rows.length, 0) + assert.equal(admission.released_at, null) + assert.equal((await keys.getById(key.id)).requests, 0) + } else { + assert.equal(rows.length, 1) + assert.equal(rows[0].outcome, kind === 'deleted-key' ? 'key_deleted' : 'charged') + assert.ok(admission.released_at) + assert.equal(rows[0].billing_snapshot.rate_multiplier, 2) + assert.ok(Math.abs(rows[0].actual_cost - .001272) < 1e-12) + } + }) +}) diff --git a/test/unit/ha-protocol-billing.test.mjs b/test/unit/ha-protocol-billing.test.mjs new file mode 100644 index 000000000..446b24184 --- /dev/null +++ b/test/unit/ha-protocol-billing.test.mjs @@ -0,0 +1,19 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const fixture = fileURLToPath(new URL('../fixtures/ha-protocol-billing.mjs', import.meta.url)) +for (const kind of ['complete', 'no-usage', 'deleted-key']) { + test(`HA protocol original command billing: ${kind}`, () => { + const env = { ...process.env, VM2API_CONTROL_PLANE_MODE: 'standalone', VM2API_HA_BILLING_CASE: kind } + delete env.NODE_TEST_CONTEXT + const result = spawnSync( + process.execPath, + ['--experimental-test-module-mocks', '--import', './test/pg-setup.mjs', '--test', fixture], + { env, encoding: 'utf8', timeout: 15000 }, + ) + assert.equal(result.status, 0, result.stdout + result.stderr) + assert.match(result.stdout, /# tests 1\b/) + }) +} From b7170644c3820a8485e209c1816f9d90c084f30e Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:34:14 +0800 Subject: [PATCH 433/473] feat: bind HA protocol billing to dispatch snapshots and original node receipts --- src/lib/core/ha-request-context.mjs | 1 + src/lib/core/ha-request-recovery.mjs | 2 +- .../db/repos/execution-admissions-repo.mjs | 6 ++ src/lib/protocol/handle-protocol.mjs | 55 +++++++++++++++++-- test/fixtures/ha-protocol-billing.mjs | 3 +- test/unit/ha-request-recovery.test.mjs | 15 ++++- 6 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/lib/core/ha-request-context.mjs b/src/lib/core/ha-request-context.mjs index b37a0ad9d..c4e577b54 100644 --- a/src/lib/core/ha-request-context.mjs +++ b/src/lib/core/ha-request-context.mjs @@ -73,6 +73,7 @@ export async function withHaRequestContext({ db = getDb(), requestId, activity, activityId: scope.id, activityOwnerId: coordinator.ownerId, activityEpoch: scope.epoch, + recoveryKind, attemptNo: null, isActive: () => active, assertActive: () => { diff --git a/src/lib/core/ha-request-recovery.mjs b/src/lib/core/ha-request-recovery.mjs index 6eafc9069..fad59533f 100644 --- a/src/lib/core/ha-request-recovery.mjs +++ b/src/lib/core/ha-request-recovery.mjs @@ -232,7 +232,7 @@ export class HaRequestRecovery { } async _bill(requestId, readRecoveryStatus) { - const billing = await new HaRequestBilling(this.db).reconcile(requestId, { readRecoveryStatus }) + const { record, ...billing } = await new HaRequestBilling(this.db).reconcile(requestId, { readRecoveryStatus }) return { recovered: true, billingHeld: !billing.settled, billing } } } diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 4dc2256d4..946466e9a 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -239,6 +239,12 @@ export class ExecutionAdmissionsRepo { } if ((admission.ha_request_id ?? null) !== (context?.requestId ?? null)) throw jobError('ha_request_context_mismatch') + if (context?.recoveryKind === 'claude_messages' && !billingSnapshot) { + const managed = await this.db + .prepare('SELECT id FROM api_key_admissions WHERE ha_request_id=$1 LIMIT 1') + .get(context.requestId) + if (managed) throw jobError('ha_billing_snapshot_required') + } if (context) { if (!nodeBinding || !context.attemptNo) throw jobError('ha_request_attempt_missing') const attempt = await this.db diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 416ea0608..ff3196d0c 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -1,5 +1,6 @@ import { trackPendingWrite } from '../db/pending-writes.mjs' import { currentHaRequestContext } from '../core/ha-request-context.mjs' +import { captureHaBillingSnapshot } from '../core/ha-request-billing.mjs' import { isKubernetesHaMode } from '../core/control-plane-mode.mjs' /** * /v1 protocol handler. URLs, auth, and response envelopes stay with the @@ -214,8 +215,42 @@ export function createHandleProtocol(deps) { async function attachPricing( logBag, - { managedKey = null, rateMultiplier = 1, bill = false, reservation = null } = {}, + { managedKey = null, rateMultiplier = 1, bill = false, reservation = null, execution = null } = {}, ) { + if (isKubernetesHaMode() && bill && managedKey && reservation?.shared) { + try { + let nodeStatus + if (execution?.requestId && execution?.nodeBinding) { + try { + nodeStatus = await deps.readNodeRecoveryStatus?.({ + requestId: execution.requestId, + nodeBinding: execution.nodeBinding, + }) + } catch {} + } + const settled = await apiKeyStore.recordHaUsage({ reservation, commandId: execution?.requestId, nodeStatus }) + const price = settled.pricing + Object.assign(logBag, { + input_cost: price?.input_cost ?? null, + output_cost: price?.output_cost ?? null, + cache_read_cost: price?.cache_read_cost ?? null, + cache_creation_cost: price?.cache_creation_cost ?? null, + web_search_cost: price?.web_search_cost ?? null, + total_cost: price?.total_cost ?? null, + pricing_model: price?.pricing_key ?? 'unpriced', + price_source: price?.price_source ?? null, + price_version: price?.price_version ?? null, + service_tier: price?.service_tier ?? null, + speed: price?.speed ?? null, + long_context: price ? (price.long_context ? 1 : 0) : null, + }) + if (!settled.settled) reservation.billingFailed = true + } catch (error) { + reservation.billingFailed = true + throw error + } + return + } const model = logBag.upstream_model || logBag.model const priced = costColumnsFromUsage(logBag.usage || {}, model) Object.assign(logBag, priced) @@ -521,12 +556,12 @@ export function createHandleProtocol(deps) { if (admission.lease?.shared) admission.lease.executionUncertain = admission.pendingExecutions.size > 0 } - function executionHooks(req, admission, candidate = {}) { + function executionHooks(req, admission, candidate = {}, billingSnapshot = null) { if (!deps.onExecutionState && !admission.lease?.shared && !candidate.bindExecution) return {} return { onExecutionStart: async (execution) => { await observeExecution(req, admission, { ...execution, started: true, ended: false }) - return await candidate.bindExecution?.(execution) + return await candidate.bindExecution?.(billingSnapshot ? { ...execution, billingSnapshot } : execution) }, onExecutionEnd: async (execution) => { await candidate.completeExecution?.(execution) @@ -1612,7 +1647,16 @@ export function createHandleProtocol(deps) { } }, callAttempt: async ({ candidate, body, attemptMeta, deliveryMode: attemptDelivery, signal, onCommit }) => { - const hooks = executionHooks(req, admission, candidate) + const billingSnapshot = + isKubernetesHaMode() && managedKey + ? captureHaBillingSnapshot({ + model: body.model, + rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey.group_id ?? 1), + cacheTtl, + requestedSpeed: wantsFastMode(ctx.body) ? 'fast' : null, + }) + : null + const hooks = executionHooks(req, admission, candidate, billingSnapshot) if (!clientStream) { return await streamAndAssembleClaudeMessage({ ...hooks, @@ -1804,8 +1848,9 @@ export function createHandleProtocol(deps) { await attachPricing(logBag, { managedKey, reservation: admission.lease, - rateMultiplier: await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), + rateMultiplier: isKubernetesHaMode() ? 1 : await deps.groupsRepo.rateMultiplier(managedKey?.group_id ?? 1), bill: !!managedKey, + execution: result?.execution, }) } diff --git a/test/fixtures/ha-protocol-billing.mjs b/test/fixtures/ha-protocol-billing.mjs index a2f5dfdfd..81590f288 100644 --- a/test/fixtures/ha-protocol-billing.mjs +++ b/test/fixtures/ha-protocol-billing.mjs @@ -39,7 +39,7 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn await opts.onExecutionStart({ requestId: commandId, nodeBinding }) const job = (await f.peer.pool.query('SELECT * FROM execution_jobs WHERE request_id=$1', [commandId])).rows[0] assert.equal(job.billing_snapshot?.rate_multiplier, 2, 'dispatch permission requires the original PG-priced group snapshot') - assert.equal(job.billing_snapshot.cache_ttl, '1h') + assert.equal(job.billing_snapshot.cache_ttl, null, 'native wire retains provider cache split rather than a menu override') assert.equal(job.billing_snapshot.model, opts.body.model) if (kind === 'deleted-key') await keys.remove(key.id) await opts.onExecutionEnd(execution) @@ -58,6 +58,7 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn assert.equal(input.requestId, commandId) assert.deepEqual(input.nodeBinding, nodeBinding) const receipt = syntheticUsageReceipt() + receipt.usage.cache_creation = { ephemeral_5m_input_tokens: '0', ephemeral_1h_input_tokens: '30' } if (kind === 'no-usage') { receipt.usage_complete = false; delete receipt.usage.output_tokens } return { found: true, command: execution.nodeCompletion, receipt } }, diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs index c3928c5a7..624eaecb6 100644 --- a/test/unit/ha-request-recovery.test.mjs +++ b/test/unit/ha-request-recovery.test.mjs @@ -92,7 +92,20 @@ async function graph({ db, first }, requestId, { withJob = true, graceMs = 30000 accountId: 'fixture-account', model: 'claude-synthetic', }) - await withHaRequestAttempt(1, () => admission.bindExecution({ requestId: 'command-' + requestId, nodeBinding })) + await withHaRequestAttempt(1, () => + admission.bindExecution({ + requestId: 'command-' + requestId, + nodeBinding, + billingSnapshot: { + version: 1, + model: 'claude-synthetic', + rate_multiplier: 1, + cache_ttl: '1h', + requested_speed: null, + resolved: null, + }, + }), + ) } await admission.markUncertain() }), From 6c161f768b1cfa52e4098c1b7162a06769e3d8b0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:35:53 +0800 Subject: [PATCH 434/473] test: retain aged attempt and job evidence until exact HA recovery ends --- test/unit/ha-request-retention.test.mjs | 68 +++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 test/unit/ha-request-retention.test.mjs diff --git a/test/unit/ha-request-retention.test.mjs b/test/unit/ha-request-retention.test.mjs new file mode 100644 index 000000000..ec4c66f53 --- /dev/null +++ b/test/unit/ha-request-retention.test.mjs @@ -0,0 +1,68 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { haBillingFixture, billingRequest, statusFor } from '../fixtures/ha-billing.mjs' +import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' +import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' +import { HaRequestRecovery } from '../../src/lib/core/ha-request-recovery.mjs' +import { withHaRequestContext } from '../../src/lib/core/ha-request-context.mjs' + +for (const state of ['held-key', 'unrecovered-activity', 'normal-finished']) { + test(`aged attempt cleanup retains only required HA evidence: ${state}`, async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async (current) => { + if (state !== 'held-key') { + await current.admission.release() + await current.reservation.release() + } + }) + await f.recoveryGate.run(async () => { + await f.peer + .prepare('UPDATE request_attempts SET started_at=$1 WHERE ha_request_id=$2') + .run('2000-01-01T00:00:00.000Z', 'billing-parent') + if (state === 'normal-finished') + await f.peer.query( + `UPDATE ${f.recovery.qualified}.activities SET completed_at=clock_timestamp() WHERE id=$1 AND owner_id=$2`, + [f.originalGate.activityId, f.original.ownerId], + ) + const deleted = await new RequestAttemptsRepo(f.peer).cleanup(7) + assert.equal(deleted, state === 'normal-finished' ? 1 : 0) + if (state === 'held-key') { + const outcome = await new HaRequestRecovery(f.peer, { coordinator: f.recovery }).recover('billing-parent', { + readRecoveryStatus: async ({ requestId }) => statusFor(data, requestId), + }) + assert.equal(outcome.recovered, true) + assert.equal(outcome.billingHeld, false) + assert.equal((await f.peer.prepare('SELECT requests FROM api_keys WHERE id=$1').get(data.key.id)).requests, 1) + } + }) + }) +} + +test('released admission history keeps its original job while the exact HA activity still needs recovery', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async (current) => { + await current.admission.release() + await current.reservation.release() + }) + await f.recoveryGate.run(async () => { + await f.peer + .prepare("UPDATE execution_admissions SET created_at=clock_timestamp()-interval '2 minutes' WHERE id=$1") + .run(data.admission.admissionId) + await withHaRequestContext({ db: f.peer, activity: f.recoveryGate, requestId: 'new-cleanup-trigger' }, async () => { + const fresh = await new ExecutionAdmissionsRepo(f.peer).tryReserve({ + accountId: 'fixture-account', + vmId: 'vm-fixture', + accountLimit: 5, + vmLimit: 5, + fableLimit: 0, + rpmLimit: 0, + }) + assert.equal(fresh.ok, true) + await fresh.release() + }) + assert.ok( + await f.peer.prepare('SELECT node_binding FROM execution_jobs WHERE request_id=$1').get(data.commandIds[0]), + 'exact original node binding must survive until its activity is recovered', + ) + }) +}) From 103779e1e578924b99b0ef02caf545e1eb166dec Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:39:20 +0800 Subject: [PATCH 435/473] fix: retain exact HA attempts while execution or billing recovery is pending --- .../db/repos/execution-admissions-repo.mjs | 13 ++++++-- src/lib/db/repos/request-attempts-repo.mjs | 31 ++++++++++++++++--- test/unit/ha-request-recovery.test.mjs | 5 +++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/lib/db/repos/execution-admissions-repo.mjs b/src/lib/db/repos/execution-admissions-repo.mjs index 946466e9a..6f8f50af3 100644 --- a/src/lib/db/repos/execution-admissions-repo.mjs +++ b/src/lib/db/repos/execution-admissions-repo.mjs @@ -1,5 +1,5 @@ import crypto from 'node:crypto' -import { withTransaction } from '../database.mjs' +import { withTransaction, getActivityCoordinator } from '../database.mjs' import { physicalContainerName, physicalVmAliases } from '../../vm/vm-physical-target.mjs' import { readWatchdogState } from '../../transport/kernel-watchdog-state.mjs' import { ExecutionCircuitsRepo } from './execution-circuits-repo.mjs' @@ -131,14 +131,21 @@ export class ExecutionAdmissionsRepo { .get(physicalVmAliases(vmId), vmId) if (alias) return denied('vm_physical_target_conflict', 0, vmLimit) const { now } = await this.db.prepare('SELECT clock_timestamp() AS now').get() + const coordinator = getActivityCoordinator(this.db) + const activityEnded = coordinator + ? `EXISTS (SELECT 1 FROM ${coordinator.qualified}.activities x + WHERE x.id=r.activity_id AND x.owner_id=r.activity_owner_id AND x.epoch=r.activity_epoch + AND x.completed_at IS NOT NULL)` + : 'false' await this.db .prepare(`DELETE FROM execution_admissions WHERE account_id = $1 AND released_at IS NOT NULL AND created_at <= $2::timestamptz - interval '60 seconds' AND NOT EXISTS (SELECT 1 FROM api_key_admissions k WHERE k.ha_request_id=execution_admissions.ha_request_id AND k.released_at IS NULL) AND NOT EXISTS (SELECT 1 FROM ha_request_contexts r - WHERE r.id=execution_admissions.ha_request_id AND r.recovery_sealed_at IS NOT NULL - AND r.execution_recovered_at IS NULL)`) + WHERE r.id=execution_admissions.ha_request_id AND (r.accepting_children + OR (r.recovery_sealed_at IS NOT NULL AND r.execution_recovered_at IS NULL) + OR (r.execution_recovered_at IS NULL AND NOT ${activityEnded})))`) .run(accountId, now) const counts = await this._counts(accountId, vmId, now) // Concurrency zero disables execution; RPM/Fable zero adds no sublimit. diff --git a/src/lib/db/repos/request-attempts-repo.mjs b/src/lib/db/repos/request-attempts-repo.mjs index 2699af258..d85efd0f7 100644 --- a/src/lib/db/repos/request-attempts-repo.mjs +++ b/src/lib/db/repos/request-attempts-repo.mjs @@ -1,4 +1,4 @@ -import { getDb, withTransaction } from '../database.mjs' +import { getDb, withTransaction, getActivityCoordinator } from '../database.mjs' import { lockHaRequestContext } from '../../core/ha-request-context.mjs' function parse(value, fallback = null) { @@ -56,9 +56,6 @@ export class RequestAttemptsRepo { this._list = db.prepare(` SELECT * FROM request_attempts WHERE request_id = $1 ORDER BY attempt_no `) - this._removeOld = db.prepare(` - DELETE FROM request_attempts WHERE started_at < $1 - `) } async begin({ requestId, attemptNo, vmId, accountId, model, selectionReason, waitMs = 0 }) { @@ -115,6 +112,30 @@ export class RequestAttemptsRepo { async cleanup(retainDays = 7) { const days = Math.max(1, Number(retainDays) || 7) const cutoff = new Date(Date.now() - days * 86400_000).toISOString() - return (await this._removeOld.run(cutoff)).changes + return withTransaction(this.db, async () => { + const parents = await this.db + .prepare(`SELECT DISTINCT ha_request_id FROM request_attempts + WHERE started_at<$1 AND ha_request_id IS NOT NULL ORDER BY ha_request_id`) + .all(cutoff) + // Recovery reads an exact attempt graph under this same parent lock. + for (const parent of parents) + await this.db.prepare('SELECT id FROM ha_request_contexts WHERE id=$1 FOR UPDATE').get(parent.ha_request_id) + const coordinator = getActivityCoordinator(this.db) + const activityEnded = coordinator + ? `EXISTS (SELECT 1 FROM ${coordinator.qualified}.activities x + WHERE x.id=r.activity_id AND x.owner_id=r.activity_owner_id AND x.epoch=r.activity_epoch + AND x.completed_at IS NOT NULL)` + : 'false' + const removed = await this.db + .prepare(`DELETE FROM request_attempts t WHERE started_at<$1 + AND NOT EXISTS (SELECT 1 FROM api_key_admissions k WHERE k.ha_request_id=t.ha_request_id AND k.released_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM execution_admissions a WHERE a.ha_request_id=t.ha_request_id AND a.released_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM execution_jobs j WHERE j.ha_request_id=t.ha_request_id AND j.ended_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM ha_request_contexts r WHERE r.id=t.ha_request_id AND (r.accepting_children + OR (r.recovery_sealed_at IS NOT NULL AND r.execution_recovered_at IS NULL) + OR (r.execution_recovered_at IS NULL AND NOT ${activityEnded})))`) + .run(cutoff) + return removed.changes + }) } } diff --git a/test/unit/ha-request-recovery.test.mjs b/test/unit/ha-request-recovery.test.mjs index 624eaecb6..9019468f9 100644 --- a/test/unit/ha-request-recovery.test.mjs +++ b/test/unit/ha-request-recovery.test.mjs @@ -462,6 +462,11 @@ for (const heldBy of ['pending-billing', 'sealed-recovery', 'normal-finished']) await f.peer .prepare('UPDATE api_key_admissions SET released_at=clock_timestamp() WHERE id=$1') .run(original.keyLease.reservationId) + if (heldBy === 'normal-finished') + await f.peer.query( + `UPDATE ${f.second.qualified}.activities SET completed_at=clock_timestamp() WHERE id=$1 AND owner_id=$2`, + [original.activity.activityId, f.first.ownerId], + ) } await f.peer .prepare("UPDATE execution_admissions SET created_at=clock_timestamp()-interval '2 minutes' WHERE id=$1") From 17b4c5e441f251658eab4cb7e2bd6ed7f93e2a4d Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:39:54 +0800 Subject: [PATCH 436/473] test: specify bounded HA orphan scan with independent item recovery --- test/unit/ha-recovery-scan.test.mjs | 94 +++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 test/unit/ha-recovery-scan.test.mjs diff --git a/test/unit/ha-recovery-scan.test.mjs b/test/unit/ha-recovery-scan.test.mjs new file mode 100644 index 000000000..5bbec0a90 --- /dev/null +++ b/test/unit/ha-recovery-scan.test.mjs @@ -0,0 +1,94 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { haBillingFixture, billingRequest, statusFor } from '../fixtures/ha-billing.mjs' +import { withHaRequestContext } from '../../src/lib/core/ha-request-context.mjs' +import { HaRequestRecovery } from '../../src/lib/core/ha-request-recovery.mjs' + +async function emptyParent(f, requestId, recoveryKind = 'claude_messages') { + const gate = await f.original.admit({ kind: 'request', requestId }) + await gate.run(() => withHaRequestContext({ db: f.db, activity: gate, requestId, recoveryKind }, async () => {})) + await gate.complete({ ended: false, reason: 'synthetic_orphan' }) +} + +test('scan paginates live-owner candidates and never treats old request timestamps as owner loss', async (t) => { + const f = await haBillingFixture(t) + await emptyParent(f, 'a-live') + await emptyParent(f, 'b-live') + await emptyParent(f, 'legacy-untyped', null) + const recovery = new HaRequestRecovery(f.peer, { coordinator: f.recovery }) + await f.recoveryGate.run(async () => { + const first = await recovery.scan({ limit: 1 }) + assert.equal(first.results.length, 1) + assert.equal(first.results[0].requestId, 'a-live') + assert.equal(first.results[0].reason, 'owner_connected') + assert.equal(first.nextCursor, 'a-live') + const second = await recovery.scan({ limit: 1, afterRequestId: first.nextCursor }) + assert.equal(second.results[0].requestId, 'b-live') + assert.equal(second.nextCursor, null) + assert.equal( + (await f.peer.prepare('SELECT count(*) AS n FROM ha_request_contexts WHERE recovery_sealed_at IS NOT NULL').get()) + .n, + 0, + ) + await f.original.stop() + const dead = await recovery.scan({ limit: 10 }) + assert.equal(dead.results.length, 2) + assert.equal( + dead.results.every((result) => result.recovered === true), + true, + ) + assert.equal((await recovery.scan()).results.length, 0) + }) +}) + +test('scan continues after one exact recovery throws and excludes fully released settled requests', async (t) => { + const f = await haBillingFixture(t) + await emptyParent(f, 'a-unavailable') + const data = await billingRequest(f) + const recovery = new HaRequestRecovery(f.peer, { + coordinator: f.recovery, + readNodeRecoveryStatus: async ({ requestId }) => statusFor(data, requestId), + }) + const recover = recovery.recover.bind(recovery) + recovery.recover = async (requestId) => { + if (requestId === 'a-unavailable') + throw Object.assign(new Error('synthetic database failure'), { code: 'synthetic_unavailable' }) + return recover(requestId) + } + await f.recoveryGate.run(async () => { + const results = await recovery.scan({ limit: 10 }) + assert.equal(results.results.length, 2) + assert.equal(results.results[0].errorCode, 'synthetic_unavailable') + assert.equal(results.results[1].requestId, 'billing-parent') + assert.equal(results.results[1].billingHeld, false) + assert.deepEqual((await recovery.scan({ afterRequestId: 'a-unavailable' })).results, []) + }) +}) + +test('scan includes already paid but unreleased reservations and excludes broken activity/epoch links', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f, async (current) => { + await current.keys.recordUsage( + current.key, + { input_tokens: 33 }, + { officialCost: 9, reservation: current.reservation }, + ) + }) + const recovery = new HaRequestRecovery(f.peer, { + coordinator: f.recovery, + readNodeRecoveryStatus: async ({ requestId }) => statusFor(data, requestId), + }) + await f.recoveryGate.run(async () => { + await f.peer + .prepare( + `INSERT INTO ha_request_contexts(id,activity_id,activity_owner_id,activity_epoch,recovery_kind) VALUES('broken-link','absent','missing-owner',1,'claude_messages')`, + ) + .run() + const result = await recovery.scan() + assert.equal(result.results.length, 1) + assert.equal(result.results[0].requestId, 'billing-parent') + assert.equal(result.results[0].billingHeld, false) + assert.equal((await f.peer.prepare('SELECT requests FROM api_keys WHERE id=$1').get(data.key.id)).requests, 1) + assert.deepEqual((await recovery.scan()).results, []) + }) +}) From 4c1d1a82b799475d7a94938e82d9bd91659479c2 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:41:41 +0800 Subject: [PATCH 437/473] feat: expose bounded HA recovery scan without starting a timer --- docs/plans/2026-10-22-ha-orphan-recovery.md | 16 ++++++--- src/lib/core/ha-request-recovery.mjs | 38 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/plans/2026-10-22-ha-orphan-recovery.md b/docs/plans/2026-10-22-ha-orphan-recovery.md index 26c867fc6..4733d14ac 100644 --- a/docs/plans/2026-10-22-ha-orphan-recovery.md +++ b/docs/plans/2026-10-22-ha-orphan-recovery.md @@ -23,12 +23,20 @@ readCompletion({requestId: originalCommandId, nodeBinding: originalPgBinding}) 原节点正常 completed 仅依赖现有 kernel 正常终态加 EOF 的协议假设。它不证明未知异常的 provider 行为已经结束。未 dispatch 的已绑定 command 仍需要节点持久 tombstone 或后续可靠停止证明，本阶段保留占位。 -## 计费与维护限制 +## API key 对账 -本阶段不写 usage、model、请求成功状态或金额，也不释放 API key reservation。缺少节点用量、实际模型和 PG 定价快照时，不能按 0 结算。API key 的业务 claim 继续阻止破坏性维护。恢复不会释放任何 VM operation、background task 或其他 parent 的资源。 +021 在实际 job bind 的短事务内保存 dispatch 前价格快照，包含 model 的 resolved rates/modes/tiers、group multiplier、实际 cache TTL 和请求 speed。后续改价不改变已提交快照。正常 handler 将最终 result.execution.requestId 持久选为 billable command；服务用原节点 recovery-status 的公开 receipt 结算。派发和结算都不读取 Pod 凭据文件。 -后续用量回执应在节点 completed 的同一 journal 事务持久化，通过原 node binding + command ID 关联 `execution_jobs`。回执仅包含 receipt ID、实际 model、usage_complete 和明确计数白名单；缺字段不得补 0，不保存响应正文、headers 或凭据。控制面按原 pricing snapshot 结算；它不能重发原 command 或重新 mint grant。 +`ApiKeyStore.recordHaUsage({reservation,commandId,nodeStatus})` 与恢复服务共用同一 parent/key/reservation 事务和唯一 `ha_request_settlements` 行。它更新原 key 的请求/token/USD 与 5h/1d/7d 窗口。现有推理链不扣 users.balance，本阶段保持该规则。key 已删除仍写 key_deleted 审计；原 reservation 已 settled 则写 already_settled，不重复增加计数。正常 handler 的 outer finally 负责 release；孤儿执行图已恢复后才原子结算并 release。 -历史清理保留两类原始执行图：仍有未 release 的同 parent API key reservation；已封闭但 execution_recovered_at 仍为空的 parent。正常已结束且无这些待处理关联的历史仍遵循旧清理策略。节点 proof 的长期独立审计存储和完整计费恢复尚未实现。 +恢复仅选择已有持久 billable command，或 sealed parent 的唯一原 job。多 command 缺最终选择、usage_complete=false、缺实际 model/四项主计数、原 snapshot 无价格或模型不匹配时，API key claim 继续保留。不会猜最高 attempt、累加多 attempt 或按 0 结算。无 job 的 sealed/execution_recovered parent 可以写 not_dispatched，释放未使用预留且不增加请求数或金额。 + +receipt 仅允许 ID、实际 model、usage_complete、规范 decimal-string 的主计数、缓存时间拆分、web search 次数与明确 speed/service_tier/stop_reason。计数进入 PG bigint 时保持精确字符串；USD 仍沿用现有 double precision schema 与价格计算规则。账单独立保留原 receipt、binding 和 snapshot；不保存响应正文、headers 或凭据。 + +## 分页发现与保留 + +`scan({limit:20,afterRequestId:null})` 返回 `{results,nextCursor}`。它在当前真实 activity 内只读发现 recovery_kind=claude_messages、原 activity/owner/epoch 一致的候选，随后每条 recover 再证明 owner session lock 已空闲。单条错误不终止其余条目，cursor 防止未知旧条目占满同一页。settled 但未 release 的原预留仍需处理；全部资源和额度已结束才排除。服务本身没有 timer。 + +历史清理保留仍有 active API key/执行资源、parent 尚未封闭、恢复未结束或原 activity 未确认 completed 的原始图。attempt cleanup 在精确 parent 锁下判断，与恢复读取互斥。正常已结束且无这些依赖的历史仍遵循旧清理策略。长期财务审计保留在 settlement 行，不依赖 admission/job 的短期历史。 新 HTTP/异步请求可能继承已关闭的 HA context。新 context 仅在新真实 coordinator gate 的 scope 下允许建立；继承的 active context 仍拒绝嵌套，原 activity 无法借 closed context 重新建立 parent。attempt context 共用父 context 的关闭状态。 diff --git a/src/lib/core/ha-request-recovery.mjs b/src/lib/core/ha-request-recovery.mjs index fad59533f..9e21bcd01 100644 --- a/src/lib/core/ha-request-recovery.mjs +++ b/src/lib/core/ha-request-recovery.mjs @@ -28,6 +28,44 @@ export class HaRequestRecovery { this.readNodeRecoveryStatus = readNodeRecoveryStatus } + /** Candidate discovery is read-only. recover() proves owner loss again for every exact parent. */ + async scan({ limit = 20, afterRequestId = null } = {}) { + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 100 || + (afterRequestId !== null && + (typeof afterRequestId !== 'string' || !afterRequestId || afterRequestId.length > 128)) + ) { + throw Object.assign(new Error('invalid HA recovery scan'), { code: 'ha_recovery_scan_invalid' }) + } + const scope = this.coordinator.assertActivity() + const rows = await this.db + .prepare(`SELECT r.id FROM ha_request_contexts r + JOIN ${this.coordinator.qualified}.activities a ON a.id=r.activity_id + AND a.owner_id=r.activity_owner_id AND a.epoch=r.activity_epoch + JOIN ${this.coordinator.qualified}.control c ON c.id=1 AND c.phase='open' + AND c.epoch=r.activity_epoch AND c.epoch=$3 + WHERE r.recovery_kind='claude_messages' AND ($1::text IS NULL OR r.id>$1) + AND ((r.execution_recovered_at IS NULL AND (a.completed_at IS NULL OR r.recovery_sealed_at IS NOT NULL + OR EXISTS (SELECT 1 FROM execution_admissions e WHERE e.ha_request_id=r.id AND e.released_at IS NULL) + OR EXISTS (SELECT 1 FROM execution_jobs j WHERE j.ha_request_id=r.id AND j.ended_at IS NULL))) + OR EXISTS (SELECT 1 FROM api_key_admissions k WHERE k.ha_request_id=r.id AND k.released_at IS NULL)) + ORDER BY r.id LIMIT $2`) + .all(afterRequestId, limit + 1, scope.epoch) + const page = rows.slice(0, limit) + const results = [] + for (const row of page) { + try { + results.push({ requestId: row.id, ...(await this.recover(row.id)) }) + } catch (error) { + const code = /^[A-Za-z0-9_]{1,80}$/.test(error?.code) ? error.code : 'ha_recovery_failed' + results.push({ requestId: row.id, recovered: false, errorCode: code }) + } + } + return { results, nextCursor: rows.length > limit ? page.at(-1).id : null } + } + async seal(requestId) { return withTransaction(this.db, async () => { const { request, ...result } = await this._seal(requestId) From bcefd8243bc44f352360a1bdd5a34f77709d6562 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:42:03 +0800 Subject: [PATCH 438/473] test: preserve pre-dispatch HA validation failures without charging --- test/fixtures/ha-protocol-billing.mjs | 13 ++++++++++--- test/unit/ha-protocol-billing.test.mjs | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test/fixtures/ha-protocol-billing.mjs b/test/fixtures/ha-protocol-billing.mjs index 81590f288..b517dd8bd 100644 --- a/test/fixtures/ha-protocol-billing.mjs +++ b/test/fixtures/ha-protocol-billing.mjs @@ -36,6 +36,7 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn const assistant = { id: 'synthetic-message', type: 'message', role: 'assistant', model: 'claude-sonnet-4-5-20250929', content: [{ type: 'text', text: 'synthetic' }], stop_reason: 'end_turn', usage: { input_tokens: 100, output_tokens: 10, cache_read_input_tokens: 20, cache_creation_input_tokens: 30 } } let rateReads = 0, receiptReads = 0 dispatch = async (opts) => { + if (kind === 'not-started') return { ok: false, status: 400, body: { type: 'error', error: { type: 'invalid_request_error', code: 'invalid_cache_control', message: 'synthetic validation failure' } }, execution: { requestId: null, started: false, ended: true, reason: 'not_started' } } await opts.onExecutionStart({ requestId: commandId, nodeBinding }) const job = (await f.peer.pool.query('SELECT * FROM execution_jobs WHERE request_id=$1', [commandId])).rows[0] assert.equal(job.billing_snapshot?.rate_multiplier, 2, 'dispatch permission requires the original PG-priced group snapshot') @@ -69,7 +70,7 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn try { const prepared = await args.applyAttempt(structuredClone(args.canonicalBody), candidate, { attemptNo: 1, attemptStartedAt: Date.now() }) const result = await withHaRequestAttempt(1, () => args.callAttempt({ candidate, body: prepared.body, attemptMeta: prepared.meta, onCommit() {} })) - return { ...result, vmId: 'vm-fixture', accountId: 'fixture-account', finalState: 'verified', attemptCount: 1 } + return { ...result, vmId: 'vm-fixture', accountId: 'fixture-account', finalState: result.ok ? 'verified' : 'rejected', attemptCount: 1 } } finally { await admission.release() } } }, }) @@ -77,9 +78,9 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn req.headers = { 'user-agent': 'synthetic-ha-client' } const res = new EventEmitter(); res.setHeader = () => {} await f.originalGate.run(() => withHaRequestContext({ db: f.db, requestId: 'billing-parent', activity: f.originalGate, recoveryKind: 'claude_messages' }, () => handler.handleProtocol(req, res, 'anthropic.messages', '/v1/messages'))) - assert.equal(res.statusCode, 200, JSON.stringify(res.body)) + assert.equal(res.statusCode, kind === 'not-started' ? 400 : 200, JSON.stringify(res.body)) assert.equal(rateReads, 1, 'group rates must not be reloaded after dispatch for billing') - assert.equal(receiptReads, 1) + assert.equal(receiptReads, kind === 'not-started' ? 0 : 1) await f.originalGate.run(async () => { const rows = await f.db.prepare('SELECT * FROM ha_request_settlements').all() const admission = await f.db.prepare('SELECT * FROM api_key_admissions WHERE ha_request_id=$1').get('billing-parent') @@ -87,6 +88,12 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn assert.equal(rows.length, 0) assert.equal(admission.released_at, null) assert.equal((await keys.getById(key.id)).requests, 0) + } else if (kind === 'not-started') { + assert.equal(rows.length, 1) + assert.equal(rows[0].outcome, 'not_dispatched') + assert.equal(rows[0].actual_cost, null) + assert.ok(admission.released_at) + assert.equal((await keys.getById(key.id)).requests, 0) } else { assert.equal(rows.length, 1) assert.equal(rows[0].outcome, kind === 'deleted-key' ? 'key_deleted' : 'charged') diff --git a/test/unit/ha-protocol-billing.test.mjs b/test/unit/ha-protocol-billing.test.mjs index 446b24184..85c7ad976 100644 --- a/test/unit/ha-protocol-billing.test.mjs +++ b/test/unit/ha-protocol-billing.test.mjs @@ -4,7 +4,7 @@ import { spawnSync } from 'node:child_process' import { fileURLToPath } from 'node:url' const fixture = fileURLToPath(new URL('../fixtures/ha-protocol-billing.mjs', import.meta.url)) -for (const kind of ['complete', 'no-usage', 'deleted-key']) { +for (const kind of ['complete', 'no-usage', 'deleted-key', 'not-started']) { test(`HA protocol original command billing: ${kind}`, () => { const env = { ...process.env, VM2API_CONTROL_PLANE_MODE: 'standalone', VM2API_HA_BILLING_CASE: kind } delete env.NODE_TEST_CONTEXT From 809ebb50139ff306dc7a5402f1291d7f5c25ebc2 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:43:05 +0800 Subject: [PATCH 439/473] fix: keep HA pre-dispatch validation errors uncharged and close late child admission --- docs/plans/2026-10-22-ha-orphan-recovery.md | 4 +++- src/lib/core/ha-request-billing.mjs | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/plans/2026-10-22-ha-orphan-recovery.md b/docs/plans/2026-10-22-ha-orphan-recovery.md index 4733d14ac..c32c71393 100644 --- a/docs/plans/2026-10-22-ha-orphan-recovery.md +++ b/docs/plans/2026-10-22-ha-orphan-recovery.md @@ -29,7 +29,7 @@ readCompletion({requestId: originalCommandId, nodeBinding: originalPgBinding}) `ApiKeyStore.recordHaUsage({reservation,commandId,nodeStatus})` 与恢复服务共用同一 parent/key/reservation 事务和唯一 `ha_request_settlements` 行。它更新原 key 的请求/token/USD 与 5h/1d/7d 窗口。现有推理链不扣 users.balance，本阶段保持该规则。key 已删除仍写 key_deleted 审计；原 reservation 已 settled 则写 already_settled，不重复增加计数。正常 handler 的 outer finally 负责 release；孤儿执行图已恢复后才原子结算并 release。 -恢复仅选择已有持久 billable command，或 sealed parent 的唯一原 job。多 command 缺最终选择、usage_complete=false、缺实际 model/四项主计数、原 snapshot 无价格或模型不匹配时，API key claim 继续保留。不会猜最高 attempt、累加多 attempt 或按 0 结算。无 job 的 sealed/execution_recovered parent 可以写 not_dispatched，释放未使用预留且不增加请求数或金额。 +恢复仅选择已有持久 billable command，或 sealed parent 的唯一原 job。多 command 缺最终选择、usage_complete=false、缺实际 model/四项主计数、原 snapshot 无价格或模型不匹配时，API key claim 继续保留。不会猜最高 attempt、累加多 attempt 或按 0 结算。无 job 的 sealed/execution_recovered parent 可以写 not_dispatched，释放未使用预留且不增加请求数或金额。原 handler 的 final 准备失败也可在同一 parent 锁下确认无 job，先关闭 child 准入再记 not_dispatched，保持既有 400；有任何 job 但缺最终 command 仍保留预留。 receipt 仅允许 ID、实际 model、usage_complete、规范 decimal-string 的主计数、缓存时间拆分、web search 次数与明确 speed/service_tier/stop_reason。计数进入 PG bigint 时保持精确字符串；USD 仍沿用现有 double precision schema 与价格计算规则。账单独立保留原 receipt、binding 和 snapshot；不保存响应正文、headers 或凭据。 @@ -40,3 +40,5 @@ receipt 仅允许 ID、实际 model、usage_complete、规范 decimal-string 的 历史清理保留仍有 active API key/执行资源、parent 尚未封闭、恢复未结束或原 activity 未确认 completed 的原始图。attempt cleanup 在精确 parent 锁下判断，与恢复读取互斥。正常已结束且无这些依赖的历史仍遵循旧清理策略。长期财务审计保留在 settlement 行，不依赖 admission/job 的短期历史。 新 HTTP/异步请求可能继承已关闭的 HA context。新 context 仅在新真实 coordinator gate 的 scope 下允许建立；继承的 active context 仍拒绝嵌套，原 activity 无法借 closed context 重新建立 parent。attempt context 共用父 context 的关闭状态。 + +服务接线示例：`new HaRequestRecovery(db,{coordinator,readNodeRecoveryStatus})`。reader 接收 `{requestId,nodeBinding}`，由调用方使用原绑定和固定可信 TLS 创建 SlotgateClient，再调用 `recoveryStatus(requestId,{timeoutMs,signal})`。正常 protocol 使用同名 `deps.readNodeRecoveryStatus`。`scan()` 和 `recover()` 必须位于当前 Pod 的真实 `gate.run()` 内；本服务不自行建立 activity 或启动后台任务。 diff --git a/src/lib/core/ha-request-billing.mjs b/src/lib/core/ha-request-billing.mjs index 9463cedce..061f49da0 100644 --- a/src/lib/core/ha-request-billing.mjs +++ b/src/lib/core/ha-request-billing.mjs @@ -211,8 +211,11 @@ export class HaRequestBilling { job = null, observed = null if (reservation.settled_at) outcome = 'already_settled' - else if (!jobs.length && source === 'recovery') outcome = 'not_dispatched' - else { + else if (!jobs.length && !chosen) { + if (source === 'handler') + await this.db.prepare('UPDATE ha_request_contexts SET accepting_children=false WHERE id=$1').run(requestId) + outcome = 'not_dispatched' + } else { if (reservation.released_at) return { settled: false, reason: 'api_key_reservation_released' } if (!chosen && source === 'recovery' && jobs.length === 1) { chosen = jobs[0].request_id @@ -309,7 +312,7 @@ export async function recordHaRequestUsage(db, { reservation, commandId, nodeSta if (reservation?.shared !== true) throw failure('invalid_api_key_reservation') const context = currentHaRequestContext(db) if (!context) throw failure('ha_request_context_required') - await selectHaBillableCommand(db, commandId) + if (commandId != null) await selectHaBillableCommand(db, commandId) const result = await new HaRequestBilling(db).settle(context.requestId, { source: 'handler', commandId, From 6f5633b0ee6272b56cd518dcc3da6a3cee9121b5 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:40:44 +0800 Subject: [PATCH 440/473] test: require a standalone synthetic node with explicit listener and cleanup --- test/unit/ha-node-launcher.test.mjs | 103 ++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 test/unit/ha-node-launcher.test.mjs diff --git a/test/unit/ha-node-launcher.test.mjs b/test/unit/ha-node-launcher.test.mjs new file mode 100644 index 000000000..2b98d8d38 --- /dev/null +++ b/test/unit/ha-node-launcher.test.mjs @@ -0,0 +1,103 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { getDb } from '../../src/lib/db/database.mjs' +import { nodeManagementFixture } from '../fixtures/node-management-fixture.mjs' +import { pickFreePort } from '../harness.mjs' + +test('the real node fixture honors its exact configured listener instead of replacing it with loopback ephemeral', async (t) => { + const port = await pickFreePort() + const fx = await nodeManagementFixture(t, { listen: `127.0.0.1:${port}` }) + try { + assert.equal(fx.node.link.address, '127.0.0.1') + assert.equal(fx.node.link.port, port) + } finally { + await fx.close() + } +}) + +test('standalone synthetic node exports private parameters, controls response completion, and cleans its exact resources', { timeout: 90000 }, async () => { + const db = getDb() + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ha-launcher-test-')) + const config = path.join(root, 'config.json') + fs.writeFileSync(config, JSON.stringify({ connectionString: db.connectionString, schema: db.schema, listen: '127.0.0.1:0' }), { mode: 0o600 }) + const child = spawn(process.execPath, [new URL('../fixtures/ha-node-launcher.mjs', import.meta.url).pathname, config], { stdio: ['ignore', 'pipe', 'pipe'] }) + const exited = once(child, 'exit') + let artifact + let params + let output = '' + let stderr = '' + child.stderr.on('data', (chunk) => { stderr += chunk }) + try { + const file = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('synthetic launcher startup timeout')), 20000) + child.once('error', (error) => { clearTimeout(timer); reject(error) }) + child.once('exit', (code) => { clearTimeout(timer); reject(new Error(`synthetic launcher exited ${code}: ${stderr}`)) }) + child.stdout.on('data', (chunk) => { + output += chunk + if (!output.includes('\n')) return + clearTimeout(timer) + resolve(output.trim()) + }) + }) + assert.equal(path.isAbsolute(file), true) + artifact = JSON.parse(fs.readFileSync(file, 'utf8')) + assert.equal(artifact.state, 'active') + params = JSON.parse(fs.readFileSync(artifact.parametersFile, 'utf8')) + assert.equal(fs.statSync(artifact.parametersFile).mode & 0o777, 0o600) + assert.equal(params.schema, db.schema) + assert.equal(params.syntheticProvider, true) + assert.equal(params.launchReceipt.fence.executor.kind, 'container') + assert.equal(JSON.parse(fs.readFileSync(params.trustFile, 'utf8')).nodes[params.nodeId].address, '127.0.0.1') + await fetch(params.controlUrl + '/pause', { method: 'POST' }) + const request = (id) => new Promise((resolve, reject) => { + const req = http.request({ socketPath: params.kernelSocket, method: 'POST', path: '/internal/v1/messages', headers: { 'content-type': 'application/json' } }, (res) => { + let text = '' + res.on('data', (chunk) => { text += chunk }) + res.on('end', () => resolve({ status: res.statusCode, text })) + res.on('error', reject) + }) + req.on('error', reject) + req.end(JSON.stringify({ request_id: id, body: {} })) + }) + const held = [request('synthetic-paused-a'), request('synthetic-paused-b')] + let observed + for (let i = 0; i < 80; i++) { + observed = await (await fetch(params.controlUrl + '/status')).json() + if (observed.active === 2) break + await new Promise((resolve) => setTimeout(resolve, 20)) + } + assert.equal(observed.active, 2) + assert.ok(observed.peak >= 2) + assert.equal(observed.responded, 0) + await fetch(params.controlUrl + '/resume', { method: 'POST' }) + for (const result of await Promise.all(held)) { + assert.equal(result.status, 200) + assert.match(result.text, /event: message_stop/) + } + await fetch(params.controlUrl + '/mode', { method: 'POST', body: JSON.stringify({ mode: 'json' }) }) + assert.equal(JSON.parse((await request('synthetic-json')).text).stop_reason, 'end_turn') + await fetch(params.controlUrl + '/mode', { method: 'POST', body: JSON.stringify({ mode: 'error' }) }) + assert.equal((await request('synthetic-upstream-error')).status, 502) + child.kill('SIGTERM') + assert.equal((await exited)[0], 0) + const cleaned = JSON.parse(fs.readFileSync(file, 'utf8')) + assert.equal(cleaned.state, 'cleaned') + assert.deepEqual(cleaned.remainingDirs, []) + assert.deepEqual(cleaned.remainingRoles, []) + assert.deepEqual(cleaned.remainingPrivateSchemas, []) + assert.equal(output.trim(), file, 'stdout contains only the resource manifest path') + } finally { + if (child.exitCode == null) { + child.kill('SIGTERM') + const timer = setTimeout(() => child.kill('SIGKILL'), 5000) + try { await exited } finally { clearTimeout(timer) } + } + fs.rmSync(root, { recursive: true, force: true }) + } +}) From 2ab0b944ff77ad239213d07d193fa42bd6686b9b Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:45:56 +0800 Subject: [PATCH 441/473] test: expose a controllable synthetic node fixture for isolated HA deployment --- test/fixtures/ha-node-launcher.md | 41 ++++++++ test/fixtures/ha-node-launcher.mjs | 87 +++++++++++++++ test/fixtures/node-management-fixture.mjs | 4 +- test/fixtures/real-allocation-gate.mjs | 8 +- test/fixtures/slotgate-allocation-node.go | 9 +- test/fixtures/synthetic-claude-node.mjs | 112 ++++++++++++++++++++ test/unit/ha-node-launcher.test.mjs | 123 +++++++++++++++++++--- 7 files changed, 361 insertions(+), 23 deletions(-) create mode 100644 test/fixtures/ha-node-launcher.md create mode 100644 test/fixtures/ha-node-launcher.mjs create mode 100644 test/fixtures/synthetic-claude-node.mjs diff --git a/test/fixtures/ha-node-launcher.md b/test/fixtures/ha-node-launcher.md new file mode 100644 index 000000000..9427cd6d2 --- /dev/null +++ b/test/fixtures/ha-node-launcher.md @@ -0,0 +1,41 @@ +# 独立合成节点 fixture + +此 fixture 仅验证 PostgreSQL 授权、真实 Go KSG2 pinned TLS、Unix 合成 kernel、正常控制面协议及持久完成记录的接线。Docker Engine 是本地 HTTP 假服务，不创建真实容器，不运行真实 provider 或闭源模型 payload。`/status` 的响应计数不是节点完成证明；完成证明仍须读取 Go Authority 的原 command journal。 + +用 Node 22.23.2 运行： + +```sh +node test/fixtures/ha-node-launcher.mjs /absolute/path/task-config.json +``` + +配置文件必须是当前用户所有的 0600 普通文件，禁止符号链接。固定字段为： + +```json +{ + "connectionString": "postgresql://TASK_TEST_USER:TASK_TEST_PASSWORD@TASK_TEST_PG/TASK_test", + "schema": "test_ha_648_task", + "listen": "10.249.248.1:0" +} +``` + +数据库名必须以 `_test` 结尾。schema 必须是普通 PostgreSQL 标识符，且以 `test_` 开头或包含 `648`。监听地址仅允许 `127.0.0.1` 或本任务明确指定的 k3d 网桥宿主地址 `10.249.248.1`；不接受通配地址或其他 LAN 地址。调用者先确认后者确为本任务网桥。端口 `0` 表示内核分配。Go fixture 保留给定地址和端口，不替换成 loopback。 + +launcher 生成合成凭据，经独立 PG grant 完成 allocate、初始文件发布和 fake Engine launch，并通过 `VmsRepo` 写入 metadata。所有出站节点管理调用都使用真实 Go PG 受限 authorizer。标准输出只有一行：资源 manifest 的绝对路径。manifest 的 `parametersFile` 指向 0600 的连接参数文件，其中包含 application schema、数据库连接、固定 trust 文件、VM metadata、原 launch receipt、Unix socket 和本地控制 URL。这些私有文件不能作为公开日志。节点 token 和 PG 密码不写入标准输出。 + +控制 URL 只监听宿主 `127.0.0.1`，提供： + +| 请求 | 作用 | +| --- | --- | +| `GET /status` | 返回 `active`、`peak`、`responded`、`healthCalls` 及仅含原 request ID/状态/模式的观测记录。 | +| `POST /pause` | 后续 messages 请求进入等待，保持 Unix 连接，不返回终态。 | +| `POST /resume` | 原等待请求继续返回响应，不生成新的执行 ID。 | +| `POST /mode`，body `{"mode":"sse"}` | 完整正常 Claude SSE，四个主 usage 计数明确；默认模式。 | +| `POST /mode`，body `{"mode":"json"}` | 完整正常 Claude JSON，四个主 usage 计数明确。 | +| `POST /mode`，body `{"mode":"error"}` | 返回 502 内部 provider 错误；不能作为正常终止证明。 | +| `POST /mode`，body `{"mode":"truncated"}` | 返回缺失字节的 SSE/HTTP body；不能作为正常终止证明。 | + +`/mode` 修改尚未返回响应的请求采用的模式。可先 pause，发起两个请求，从 status 确认 `active >= 2` 后选择响应模式。用于 controller 崩溃实验时，保持 fixture 存活，先终止本任务 controller，再 resume 原 kernel 响应。该计数用于合成测试时序，不能替代 Go 持久 command 状态。 + +停止顺序：先停止本任务 controller/Pod，再向 manifest 中的 launcher PID 发送 SIGTERM。launcher 关闭自己的控制和 Unix 监听，终止自己的 Go 子进程，删除生成的临时源码、副本二进制、配置和 trust 文件，并删除自身受限 PG role、node authority 和 maintenance 私有 schema。应用 schema/VM metadata 属于调用者，用于保留本任务结果，不由 launcher 删除。manifest 留作无凭据的清理证据；只有精确检查剩余目录、role 和私有 schema 均为空时才标记 `cleaned`。 + +本工具不创建 Docker/k3d/Kubernetes 资源，不执行 SSH/SFTP，不迁移真实节点权限，不能单独作为生产 HA 验收。 diff --git a/test/fixtures/ha-node-launcher.mjs b/test/fixtures/ha-node-launcher.mjs new file mode 100644 index 000000000..d3fd1e3dc --- /dev/null +++ b/test/fixtures/ha-node-launcher.mjs @@ -0,0 +1,87 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { openDatabase, closeDatabase } from '../../src/lib/db/database.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { loadSettingsState } from '../../src/lib/db/settings-state.mjs' +import { nodeManagementFixture } from './node-management-fixture.mjs' +import { syntheticClaudeNode } from './synthetic-claude-node.mjs' + +// Dedicated synthetic test resources only. Never accept a production database, +// arbitrary interface, command, credential, host mount or provider endpoint. +function readConfig(file) { + try { + const stat = fs.lstatSync(file) + if (!path.isAbsolute(file) || !stat.isFile() || stat.isSymbolicLink() || (stat.mode & 0o777) !== 0o600 || stat.uid !== process.getuid()) throw new Error() + const value = JSON.parse(fs.readFileSync(file, 'utf8')) + if (Object.keys(value).sort().join(',') !== 'connectionString,listen,schema' || !/_test$/.test(new URL(value.connectionString).pathname) || !/^[a-z_][a-z0-9_]{0,62}$/.test(value.schema) || (!value.schema.startsWith('test_') && !value.schema.includes('648'))) throw new Error() + const listen = /^(127\.0\.0\.1|10\.249\.248\.1):([0-9]{1,5})$/.exec(value.listen) + if (!listen || Number(listen[2]) > 65535) throw new Error() + return value + } catch { throw Object.assign(new Error('invalid synthetic node fixture config'), { code: 'fixture_config_invalid' }) } +} + +const hooks = [] +const config = readConfig(process.argv[2]) +const resourcesFile = path.join(os.tmpdir(), `vm2api-648-ha-node-${process.pid}-resources.json`) +fs.writeFileSync(resourcesFile, '', { mode: 0o600, flag: 'wx' }) +const resources = { state: 'starting', launcherPid: process.pid, applicationSchema: config.schema, applicationSchemaOwned: false } +const save = () => fs.writeFileSync(resourcesFile, JSON.stringify(resources, null, 2) + '\n', { mode: 0o600 }) +let fixture +let kernel +let stopped +const stop = new Promise((resolve) => { stopped = resolve }) +process.once('SIGTERM', stopped) +process.once('SIGINT', stopped) +try { + const db = await openDatabase({ connectionString: config.connectionString, schema: config.schema }) + await loadSettingsState(db) + fixture = await nodeManagementFixture({ after: (fn) => hooks.push(fn) }, { listen: config.listen }) + Object.assign(resources, { root: fixture.root, nodeRoot: fixture.node.root, nodePid: fixture.node.pid, ...fixture.resources }) + save() + const credential = { ...JSON.parse(fixture.payload).credential, expires_at: Date.now() + 8 * 3600000 } + const payload = Buffer.from(JSON.stringify({ credential })) + const published = await fixture.run('node_publish_initial', (operation) => fixture.service.publishInitial(fixture.input(operation, { payload }))) + const launched = await fixture.run('node_launch', (operation) => fixture.service.launch(fixture.input(operation, { payload, commandId: 'standalone-synthetic-launch', manifestDigest: published.manifest_digest }))) + const kernelSocket = path.join(fixture.node.root, 'runtime', 'vms', fixture.vmId, 'incarnations', launched.fence.incarnation, 'run', 'kernel.sock') + kernel = await syntheticClaudeNode(kernelSocket) + const metadata = { + id: fixture.vmId, name: 'synthetic-node-fixture', node_id: fixture.nodeId, status: 'running', schedulable: true, + timezone: 'UTC', locale: 'en_US.UTF-8', fingerprint: { device_id: 'ha-fixture-device', session_id: 'ha-fixture-session', machineID: 'ha-fixture-machine' }, + policy: { maxConcurrency: 2, weight: 1 }, + claude: { account_uuid: credential.account_uuid, org_uuid: credential.org_uuid, email: credential.email, has_access: true, has_refresh: true, expires_at: credential.expires_at, source: 'synthetic-node-fixture' }, + } + await new VmsRepo(db).upsertFromVmJson(metadata) + const trustFile = path.join(fixture.root, 'trusted-nodes.json') + const metadataFile = path.join(fixture.root, 'vm-metadata.json') + const parametersFile = path.join(fixture.root, 'deployment.json') + const privateFile = (file, data) => fs.writeFileSync(file, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 }) + privateFile(trustFile, { nodes: { [fixture.nodeId]: { ...fixture.node.link, policyDigest: fixture.policyDigest, presetId: 'claude-docker-v1' } } }) + privateFile(metadataFile, metadata) + privateFile(parametersFile, { syntheticProvider: true, connectionString: config.connectionString, schema: config.schema, nodeId: fixture.nodeId, vmId: fixture.vmId, accountId: credential.account_uuid, controlUrl: kernel.controlUrl, kernelSocket, trustFile, metadataFile, launchReceipt: launched }) + Object.assign(resources, { state: 'active', parametersFile, controlUrl: kernel.controlUrl }) + save() + process.stdout.write(resourcesFile + '\n') + await stop +} catch (error) { + resources.error = error.code || error.name + process.stderr.write(`synthetic node fixture failed: ${resources.error}\n`) + process.exitCode = 1 +} finally { + const failures = [] + const attempt = async (fn) => { try { await fn() } catch (error) { failures.push(error.code || error.name) } } + await attempt(() => kernel?.close()) + for (const cleanup of hooks.reverse()) await attempt(cleanup) + await attempt(async () => { + if (!fixture) return + resources.remainingDirs = [fixture.root, fixture.node.root].filter((file) => fs.existsSync(file)) + resources.remainingRoles = (await fixture.db.pool.query('SELECT rolname FROM pg_roles WHERE rolname=$1', [fixture.resources.readerRole])).rows + resources.remainingPrivateSchemas = (await fixture.db.pool.query('SELECT nspname FROM pg_namespace WHERE nspname=ANY($1::text[])', [[fixture.resources.authoritySchema, fixture.resources.maintenanceSchema]])).rows + }) + await attempt(closeDatabase) + const remaining = ['remainingDirs', 'remainingRoles', 'remainingPrivateSchemas'].some((key) => resources[key]?.length) + resources.state = !fixture ? 'startup_failed' : failures.length || remaining ? 'cleanup_failed' : 'cleaned' + resources.cleanupFailures = failures + if (resources.state !== 'cleaned') process.exitCode = 1 + save() +} diff --git a/test/fixtures/node-management-fixture.mjs b/test/fixtures/node-management-fixture.mjs index 4a363eb4c..de23c3ba4 100644 --- a/test/fixtures/node-management-fixture.mjs +++ b/test/fixtures/node-management-fixture.mjs @@ -26,7 +26,7 @@ export const payload = Buffer.from( }), ) -export async function nodeManagementFixture(t) { +export async function nodeManagementFixture(t, { listen } = {}) { const db = getDb() const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) const coordinator = new GlobalMaintenanceCoordinator(db) @@ -130,7 +130,7 @@ export async function nodeManagementFixture(t) { node = await realAllocationGate( t, { connection_string: url.toString(), authority_schema: service.schema, node_id: nodeId }, - { launch: policy, dockerSocket: socket }, + { launch: policy, dockerSocket: socket, ...(listen ? { listen } : {}) }, ) const run = (kind, callback) => new SharedVmOperations(db).run(vmId, callback, { kind }) const allocated = await run('node_allocate', (operation) => diff --git a/test/fixtures/real-allocation-gate.mjs b/test/fixtures/real-allocation-gate.mjs index ff08e48e2..74903be62 100644 --- a/test/fixtures/real-allocation-gate.mjs +++ b/test/fixtures/real-allocation-gate.mjs @@ -5,7 +5,9 @@ import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' -export async function realAllocationGate(t, management, { launch, dockerSocket } = {}) { +export async function realAllocationGate(t, management, { launch, dockerSocket, listen = '127.0.0.1:0' } = {}) { + const address = /^(127\.0\.0\.1|10\.249\.248\.1):([0-9]{1,5})$/.exec(listen) + if (!address || Number(address[2]) > 65535) throw Object.assign(new Error('invalid synthetic node listener'), {code:'fixture_listener_invalid'}) const root = fs.mkdtempSync(path.join(os.tmpdir(),'ag2-')) const worker = path.join(root,'worker') fs.cpSync(fileURLToPath(new URL('../../worker',import.meta.url)),worker,{recursive:true}) @@ -35,7 +37,7 @@ export async function realAllocationGate(t, management, { launch, dockerSocket } const runtime = path.join(root,'runtime') fs.mkdirSync(runtime,{mode:0o700}) const configFile = path.join(root,'config.json') - fs.writeFileSync(configFile,JSON.stringify({listen:'127.0.0.1:0',root:runtime,token:'synthetic-node-allocation-token-000000',tls_dir:path.join(root,'tls'), + fs.writeFileSync(configFile,JSON.stringify({listen,root:runtime,token:'synthetic-node-allocation-token-000000',tls_dir:path.join(root,'tls'), fenced:true,authority_dir:path.join(root,'authority'),docker_socket:dockerSocket || path.join(root,'never-opened.sock'),management, ...(launch ? {launch} : {})}),{mode:0o600}) child = spawn(binary,[configFile],{stdio:['ignore','pipe','pipe']}) @@ -54,6 +56,6 @@ export async function realAllocationGate(t, management, { launch, dockerSocket } try {resolve(JSON.parse(buffer.slice(0,newline)))} catch(error){reject(error)} }) }) - return {link,root,close} + return {link,root,close,pid:child.pid} } catch(error) {await close();throw error} } diff --git a/test/fixtures/slotgate-allocation-node.go b/test/fixtures/slotgate-allocation-node.go index 0f607d06e..37eb3b40a 100644 --- a/test/fixtures/slotgate-allocation-node.go +++ b/test/fixtures/slotgate-allocation-node.go @@ -26,16 +26,21 @@ func main() { var config slotgate.Config check(json.NewDecoder(file).Decode(&config)) check(file.Close()) + host, _, err := net.SplitHostPort(config.Listen) + check(err) + if host != "127.0.0.1" && host != "10.249.248.1" { + panic("invalid synthetic node listener") + } server, err := slotgate.New(config) check(err) - listener, err := net.Listen("tcp", "127.0.0.1:0") + listener, err := net.Listen("tcp", config.Listen) check(err) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() done := make(chan error, 1) go func() { done <- server.ServeListener(ctx, tls.NewListener(listener, server.TLSConfig())) }() cert := sha256.Sum256(server.TLSConfig().Certificates[0].Certificate[0]) - check(json.NewEncoder(os.Stdout).Encode(map[string]any{"address": "127.0.0.1", "port": listener.Addr().(*net.TCPAddr).Port, "token": config.Token, "certSha256": hex.EncodeToString(cert[:])})) + check(json.NewEncoder(os.Stdout).Encode(map[string]any{"address": listener.Addr().(*net.TCPAddr).IP.String(), "port": listener.Addr().(*net.TCPAddr).Port, "token": config.Token, "certSha256": hex.EncodeToString(cert[:])})) <-ctx.Done() check(<-done) } diff --git a/test/fixtures/synthetic-claude-node.mjs b/test/fixtures/synthetic-claude-node.mjs new file mode 100644 index 000000000..b3f04f4f0 --- /dev/null +++ b/test/fixtures/synthetic-claude-node.mjs @@ -0,0 +1,112 @@ +import http from 'node:http' + +// A controllable local provider substitute. Its observations are test evidence, +// never execution completion proofs; only the Go journal supplies those. +export async function syntheticClaudeNode(socketPath, { mode = 'sse' } = {}) { + const model = 'claude-haiku-4-5-20251001' + const modes = new Set(['sse', 'json', 'error', 'truncated']) + let paused = false + let closed = false + let active = 0 + let peak = 0 + let responded = 0 + let healthCalls = 0 + const requests = [] + const waiting = new Set() + const resume = () => { paused = false; for (const wake of waiting) wake(); waiting.clear() } + const frames = (id) => [ + { type: 'message_start', message: { id, type: 'message', role: 'assistant', model, content: [], stop_reason: null, usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } } }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'synthetic-node-response' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 7 } }, + { type: 'message_stop' }, + ].map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join('') + const kernel = http.createServer(async (req, res) => { + try { + if (req.method === 'GET' && req.url === '/internal/health') { + healthCalls++ + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ ok: true, healthy: true, ready_slots: 8, version: 'synthetic-kernel' })) + return + } + if (req.method !== 'POST' || req.url !== '/internal/v1/messages') return res.writeHead(404).end() + let body = '' + for await (const chunk of req) { + body += chunk + if (body.length > 16 * 1024 * 1024) return res.writeHead(413).end() + } + const envelope = JSON.parse(body) + if (typeof envelope.request_id !== 'string' || !/^[A-Za-z0-9_-]{1,256}$/.test(envelope.request_id)) return res.writeHead(400).end() + const record = { requestId: envelope.request_id, state: 'waiting', mode: null } + requests.push(record) + active++ + peak = Math.max(peak, active) + try { + if (paused) await new Promise((resolve) => waiting.add(resolve)) + if (closed) return res.destroy() + record.mode = mode + record.state = 'responded' + const id = 'msg_' + envelope.request_id + if (mode === 'error') { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: { type: 'worker_error', code: 'provider_error', message: 'synthetic provider error: client disconnected' } })) + } else if (mode === 'json') { + res.writeHead(200, { 'content-type': 'application/json', 'x-terminal-state': 'verified' }) + res.end(JSON.stringify({ id, type: 'message', role: 'assistant', model, content: [{ type: 'text', text: 'synthetic-node-response' }], stop_reason: 'end_turn', usage: { input_tokens: 5, output_tokens: 7, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } })) + } else { + const body = frames(id) + res.writeHead(200, { 'content-type': 'text/event-stream', 'x-terminal-state': 'verified', ...(mode === 'truncated' ? { 'content-length': String(Buffer.byteLength(body) + 32), connection: 'close' } : {}) }) + res.end(mode === 'truncated' ? body.slice(0, -4) : body) + } + responded++ + } finally { + active-- + } + } catch { + res.destroy() + } + }) + const status = () => ({ syntheticProvider: true, paused, mode, active, peak, responded, healthCalls, requests }) + const control = http.createServer(async (req, res) => { + const send = (value, code = 200) => { res.writeHead(code, { 'content-type': 'application/json' }); res.end(JSON.stringify(value)) } + if (req.method === 'GET' && req.url === '/status') return send(status()) + if (req.method !== 'POST') return send({ error: 'unsupported' }, 405) + if (req.url === '/pause') paused = true + else if (req.url === '/resume') resume() + else if (req.url === '/mode') { + try { + let data = '' + for await (const chunk of req) { + data += chunk + if (data.length > 2048) return send({ error: 'invalid_mode' }, 400) + } + const input = JSON.parse(data) + if (Object.keys(input).join(',') !== 'mode' || !modes.has(input.mode)) throw new Error() + mode = input.mode + } catch { return send({ error: 'invalid_mode' }, 400) } + } else return send({ error: 'unsupported' }, 404) + send(status()) + }) + const listen = (server, address) => new Promise((resolve, reject) => { server.once('error', reject); server.listen(address, resolve) }) + try { + await listen(kernel, socketPath) + await listen(control, { host: '127.0.0.1', port: 0 }) + } catch (error) { + kernel.close() + control.close() + throw error + } + return { + controlUrl: `http://127.0.0.1:${control.address().port}`, + async close() { + if (closed) return + closed = true + resume() + for (const server of [control, kernel]) { + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + } + }, + } +} diff --git a/test/unit/ha-node-launcher.test.mjs b/test/unit/ha-node-launcher.test.mjs index 2b98d8d38..33b6ddb56 100644 --- a/test/unit/ha-node-launcher.test.mjs +++ b/test/unit/ha-node-launcher.test.mjs @@ -6,9 +6,64 @@ import path from 'node:path' import http from 'node:http' import { spawn } from 'node:child_process' import { once } from 'node:events' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' import { getDb } from '../../src/lib/db/database.mjs' import { nodeManagementFixture } from '../fixtures/node-management-fixture.mjs' import { pickFreePort } from '../harness.mjs' +import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' + +test('synthetic node fixture refuses wildcard and unapproved LAN listeners before creating a child', async () => { + let hooks = 0 + for (const listen of ['0.0.0.0:0', '192.168.20.1:0', '10.249.248.2:0', '127.0.0.1:65536']) { + await assert.rejects( + realAllocationGate( + { + after() { + hooks++ + }, + }, + null, + { listen }, + ), + { code: 'fixture_listener_invalid' }, + ) + } + assert.equal(hooks, 0) +}) + +test('standalone node rejects nonprivate configs and unapproved addresses without exposing input credentials', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ha-launcher-invalid-')) + try { + const file = path.join(root, 'config.json') + for (const patch of [{ listen: '0.0.0.0:0' }, { schema: 'production' }, { mode: 0o644 }]) { + const config = { + connectionString: 'postgresql://synthetic:do-not-print@127.0.0.1:1/fixture_test', + schema: 'test_ha_648', + listen: '127.0.0.1:0', + ...patch, + } + delete config.mode + fs.writeFileSync(file, JSON.stringify(config), { mode: 0o600 }) + fs.chmodSync(file, patch.mode ?? 0o600) + await assert.rejects( + promisify(execFile)( + process.execPath, + [new URL('../fixtures/ha-node-launcher.mjs', import.meta.url).pathname, file], + { timeout: 10000 }, + ), + (error) => { + assert.equal(error.stdout, '') + assert.match(error.stderr, /invalid synthetic node fixture config/) + assert.equal(error.stderr.includes('do-not-print'), false) + return true + }, + ) + } + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) test('the real node fixture honors its exact configured listener instead of replacing it with loopback ephemeral', async (t) => { const port = await pickFreePort() @@ -21,23 +76,42 @@ test('the real node fixture honors its exact configured listener instead of repl } }) -test('standalone synthetic node exports private parameters, controls response completion, and cleans its exact resources', { timeout: 90000 }, async () => { +test('standalone synthetic node exports private parameters, controls response completion, and cleans its exact resources', { + timeout: 90000, +}, async () => { const db = getDb() const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ha-launcher-test-')) const config = path.join(root, 'config.json') - fs.writeFileSync(config, JSON.stringify({ connectionString: db.connectionString, schema: db.schema, listen: '127.0.0.1:0' }), { mode: 0o600 }) - const child = spawn(process.execPath, [new URL('../fixtures/ha-node-launcher.mjs', import.meta.url).pathname, config], { stdio: ['ignore', 'pipe', 'pipe'] }) + fs.writeFileSync( + config, + JSON.stringify({ connectionString: db.connectionString, schema: db.schema, listen: '127.0.0.1:0' }), + { mode: 0o600 }, + ) + const child = spawn( + process.execPath, + [new URL('../fixtures/ha-node-launcher.mjs', import.meta.url).pathname, config], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ) const exited = once(child, 'exit') let artifact let params let output = '' let stderr = '' - child.stderr.on('data', (chunk) => { stderr += chunk }) + let pending = Promise.resolve() + child.stderr.on('data', (chunk) => { + stderr += chunk + }) try { const file = await new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error('synthetic launcher startup timeout')), 20000) - child.once('error', (error) => { clearTimeout(timer); reject(error) }) - child.once('exit', (code) => { clearTimeout(timer); reject(new Error(`synthetic launcher exited ${code}: ${stderr}`)) }) + child.once('error', (error) => { + clearTimeout(timer) + reject(error) + }) + child.once('exit', (code) => { + clearTimeout(timer) + reject(new Error(`synthetic launcher exited ${code}: ${stderr}`)) + }) child.stdout.on('data', (chunk) => { output += chunk if (!output.includes('\n')) return @@ -55,17 +129,29 @@ test('standalone synthetic node exports private parameters, controls response co assert.equal(params.launchReceipt.fence.executor.kind, 'container') assert.equal(JSON.parse(fs.readFileSync(params.trustFile, 'utf8')).nodes[params.nodeId].address, '127.0.0.1') await fetch(params.controlUrl + '/pause', { method: 'POST' }) - const request = (id) => new Promise((resolve, reject) => { - const req = http.request({ socketPath: params.kernelSocket, method: 'POST', path: '/internal/v1/messages', headers: { 'content-type': 'application/json' } }, (res) => { - let text = '' - res.on('data', (chunk) => { text += chunk }) - res.on('end', () => resolve({ status: res.statusCode, text })) - res.on('error', reject) + const request = (id) => + new Promise((resolve, reject) => { + const req = http.request( + { + socketPath: params.kernelSocket, + method: 'POST', + path: '/internal/v1/messages', + headers: { 'content-type': 'application/json' }, + }, + (res) => { + let text = '' + res.on('data', (chunk) => { + text += chunk + }) + res.on('end', () => resolve({ status: res.statusCode, text })) + res.on('error', reject) + }, + ) + req.on('error', reject) + req.end(JSON.stringify({ request_id: id, body: {} })) }) - req.on('error', reject) - req.end(JSON.stringify({ request_id: id, body: {} })) - }) const held = [request('synthetic-paused-a'), request('synthetic-paused-b')] + pending = Promise.allSettled(held) let observed for (let i = 0; i < 80; i++) { observed = await (await fetch(params.controlUrl + '/status')).json() @@ -96,8 +182,13 @@ test('standalone synthetic node exports private parameters, controls response co if (child.exitCode == null) { child.kill('SIGTERM') const timer = setTimeout(() => child.kill('SIGKILL'), 5000) - try { await exited } finally { clearTimeout(timer) } + try { + await exited + } finally { + clearTimeout(timer) + } } + await pending fs.rmSync(root, { recursive: true, force: true }) } }) From ceb2775221fb73549e542855356073bacf8e54a8 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:47:25 +0800 Subject: [PATCH 442/473] fix: connect HA billing to original node receipts and preserve backup fixtures --- src/lib/cluster/node-request-recovery.mjs | 21 +++++++++++++++++++++ src/server.mjs | 3 +++ test/unit/backup-service.test.mjs | 7 ++++--- test/unit/ha-routing-monitors.test.mjs | 12 +++++++++--- 4 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 src/lib/cluster/node-request-recovery.mjs diff --git a/src/lib/cluster/node-request-recovery.mjs b/src/lib/cluster/node-request-recovery.mjs new file mode 100644 index 000000000..aa5432814 --- /dev/null +++ b/src/lib/cluster/node-request-recovery.mjs @@ -0,0 +1,21 @@ +import { normalizeNodeBinding } from '../core/node-execution-proof.mjs' +import { SlotgateClient } from '../transport/slotgate-client.mjs' + +/** Historical status uses the PG job's original fence and the bootstrap TLS trust. */ +export function createNodeRecoveryReader(trustedNodes) { + const nodes = structuredClone(trustedNodes) + return async ({ requestId, nodeBinding }) => { + const binding = normalizeNodeBinding(nodeBinding) + const link = Object.hasOwn(nodes, binding.nodeId) ? nodes[binding.nodeId] : null + if (!link) throw Object.assign(new Error('original node is not trusted'), { code: 'node_recovery_unavailable' }) + const client = new SlotgateClient({ + trusted: true, + pgBindingReady: true, + nodeId: binding.nodeId, + vmId: binding.vmId, + fence: binding.fence, + link, + }) + return client.recoveryStatus(requestId, { timeoutMs: 3000 }) + } +} diff --git a/src/server.mjs b/src/server.mjs index 065101808..414db7d56 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -109,6 +109,7 @@ import { isKubernetesMode, isKubernetesHaMode, shutdownTimeoutMs } from './lib/c import { loadHaNodeTrust, haRouteAllowed, haUnsupported } from './lib/core/ha-control-plane-policy.mjs' import { HaVmCatalog, bindHaVmCatalog } from './lib/vm/ha-vm-catalog.mjs' import { captureNodeExecution } from './lib/cluster/node-execution-capabilities.mjs' +import { createNodeRecoveryReader } from './lib/cluster/node-request-recovery.mjs' import { withHaRequestContext } from './lib/core/ha-request-context.mjs' import { ControlPlaneLifecycle, withTimeout } from './lib/core/control-plane-lifecycle.mjs' import { getSharedBackgroundTasks } from './lib/core/shared-background-tasks.mjs' @@ -147,6 +148,7 @@ if (!getPanelAdmin().password) { const kubernetesMode = isKubernetesMode() const haMode = isKubernetesHaMode() const trustedNodes = haMode ? loadHaNodeTrust(process.env.VM2API_NODE_CONFIG_FILE) : null +const readNodeRecoveryStatus = haMode ? createNodeRecoveryReader(trustedNodes) : null const shutdownGraceMs = shutdownTimeoutMs() const allowRate = createRateLimiter({ @@ -1053,6 +1055,7 @@ const importCommit = createImportCommit({ const { commitImportedOauth, requireSlotProxy, officialCcStatsHandler } = importCommit const protocolHandlers = createHandleProtocol({ + readNodeRecoveryStatus, onExecutionState: noteRequestExecution, json, writeSSEHeaders, diff --git a/test/unit/backup-service.test.mjs b/test/unit/backup-service.test.mjs index 497cdb112..531488f05 100644 --- a/test/unit/backup-service.test.mjs +++ b/test/unit/backup-service.test.mjs @@ -80,7 +80,7 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current const db = getDb() await db.exec( `ALTER TABLE request_attempts DROP COLUMN ha_request_id; - DROP TABLE ha_request_contexts, panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; + DROP TABLE ha_request_settlements, ha_request_contexts, panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; @@ -107,6 +107,7 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current 'execution_seats', 'execution_jobs', 'ha_request_contexts', + 'ha_request_settlements', 'session_turns', 'vm_operations', 'background_task_runs', @@ -115,14 +116,14 @@ test('restoring a PG 001 snapshot removes later tables before reapplying current ]) { assert.equal((await db.prepare(`SELECT count(*) AS n FROM ${name}`).get()).n, 0) } - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 20) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 21) }) test('an outside dependency refuses older-snapshot cleanup without changing either schema', async () => { const db = getDb() await db.exec( `ALTER TABLE request_attempts DROP COLUMN ha_request_id; - DROP TABLE ha_request_contexts, panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; + DROP TABLE ha_request_settlements, ha_request_contexts, panel_sessions, api_key_admissions, gateway_rate_buckets, execution_jobs, execution_admissions, execution_seats, session_turns, vm_operations, background_task_runs, oauth_pkce_sessions, shell_tickets, execution_circuits; ALTER TABLE accounts DROP COLUMN observation_versions; ALTER TABLE proxies DROP COLUMN config_revision; ALTER TABLE sticky_sessions DROP COLUMN deleted_at; diff --git a/test/unit/ha-routing-monitors.test.mjs b/test/unit/ha-routing-monitors.test.mjs index 11036c521..833f3229e 100644 --- a/test/unit/ha-routing-monitors.test.mjs +++ b/test/unit/ha-routing-monitors.test.mjs @@ -46,13 +46,19 @@ test('HA routing updates keep unsupported monitors stopped and reject on-demand cfg: { paths: { project: root } }, routingConfigPath: file, getRoutingConfig: () => config, - setRoutingConfig: (next) => { config = next }, + setRoutingConfig: (next) => { + config = next + }, accountQuota: { reloadConfig() {}, attachRuntimeRepo() {} }, getPoolScheduler: () => pool, - setPoolScheduler: (next) => { pool = next }, + setPoolScheduler: (next) => { + pool = next + }, healthMonitor: health, usageProbeMonitor: usage, - probeUsageOne: () => { probes++ }, + probeUsageOne: () => { + probes++ + }, }) const bootstrap = await coordinator.admitBootstrap() await bootstrap.run(async () => { From 54f203c0d5cf0c8fff374fbb71ed0c52b34f1576 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:49:24 +0800 Subject: [PATCH 443/473] test: specify HA recovery monitor lifecycle and exact PG settlement --- test/fixtures/ha-billing.mjs | 4 +- test/unit/ha-recovery-monitor.test.mjs | 257 +++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 test/unit/ha-recovery-monitor.test.mjs diff --git a/test/fixtures/ha-billing.mjs b/test/fixtures/ha-billing.mjs index 9e99dc5bc..535173df9 100644 --- a/test/fixtures/ha-billing.mjs +++ b/test/fixtures/ha-billing.mjs @@ -43,7 +43,7 @@ export async function haBillingFixture(t) { return { db, peer, original, recovery, originalGate, recoveryGate } } -export async function billingRequest(f, callback = async () => {}, { snapshot = syntheticBillingSnapshot(), jobs = 1 } = {}) { +export async function billingRequest(f, callback = async () => {}, { snapshot = syntheticBillingSnapshot(), jobs = 1, stopOwner = true } = {}) { const nodeBinding = { protocol: 'KSG2', nodeId: 'original-node', vmId: 'vm-fixture', kind: 'kernel', fence: syntheticFence() } const data = { nodeBinding, commandIds: [] } await f.originalGate.run(() => withHaRequestContext({ db: f.db, requestId: 'billing-parent', activity: f.originalGate, recoveryKind: 'claude_messages' }, async () => { @@ -64,7 +64,7 @@ export async function billingRequest(f, callback = async () => {}, { snapshot = await data.admission.markUncertain() })) await f.originalGate.complete({ ended: false, reason: 'synthetic_owner_lost' }) - await f.original.stop() + if (stopOwner) await f.original.stop() return data } diff --git a/test/unit/ha-recovery-monitor.test.mjs b/test/unit/ha-recovery-monitor.test.mjs new file mode 100644 index 000000000..14d033321 --- /dev/null +++ b/test/unit/ha-recovery-monitor.test.mjs @@ -0,0 +1,257 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { haBillingFixture, billingRequest, statusFor } from '../fixtures/ha-billing.mjs' +import { withHaRequestContext } from '../../src/lib/core/ha-request-context.mjs' + +function intervals(t) { + const timers = [] + t.mock.method(globalThis, 'setInterval', (callback, delay) => { + const timer = { + callback, + delay, + unrefCalled: false, + unref() { + this.unrefCalled = true + }, + } + timers.push(timer) + return timer + }) + t.mock.method(globalThis, 'clearInterval', (timer) => { + timer.cleared = true + }) + return timers +} + +async function monitorFor(t, f, readNodeRecoveryStatus) { + const { createHaRecoveryMonitor } = await import('../../src/lib/core/ha-recovery-monitor.mjs') + const monitor = createHaRecoveryMonitor({ db: f.peer, coordinator: f.recovery, readNodeRecoveryStatus }) + t.after(() => monitor.stop()) + return monitor +} + +async function scannerActivities(f) { + return ( + await f.peer.pool.query( + `SELECT * FROM ${f.recovery.qualified}.activities WHERE kind='ha_request_recovery' ORDER BY created_at,id`, + ) + ).rows +} + +function assertScannersEnded(rows, expectedCount) { + assert.equal(rows.length, expectedCount) + for (const row of rows) { + assert.ok(row.completed_at) + assert.equal(row.uncertain_at, null) + } +} + +test('HA monitor immediately scans 20 live-owner parents, advances its local cursor every five seconds and resumes after stop', async (t) => { + const f = await haBillingFixture(t) + const timers = intervals(t) + for (let index = 0; index < 21; index++) { + const requestId = `live-${String(index).padStart(2, '0')}` + const gate = await f.original.admit({ kind: 'request', requestId }) + await gate.run(() => + withHaRequestContext({ db: f.db, activity: gate, requestId, recoveryKind: 'claude_messages' }, async () => {}), + ) + await gate.complete({ ended: false, reason: 'synthetic_live_owner' }) + } + const monitor = await monitorFor(t, f, async () => assert.fail('a connected owner must not be queried')) + const first = await monitor.start() + assert.equal(first.results.length, 20) + assert.ok(first.results.every((result) => result.reason === 'owner_connected')) + assert.equal(first.nextCursor, 'live-19') + assert.equal(timers.length, 1) + assert.equal(timers[0].delay, 5000) + assert.equal(timers[0].unrefCalled, true) + await monitor.start() + assertScannersEnded(await scannerActivities(f), 1) + timers[0].callback() + const second = await monitor.flush() + assert.deepEqual( + second.results.map((result) => result.requestId), + ['live-20'], + ) + assert.equal(second.nextCursor, null) + await monitor.stop() + assert.equal(timers[0].cleared, true) + timers[0].callback() + await monitor.flush() + assertScannersEnded(await scannerActivities(f), 2) + const resumed = await monitor.start() + assert.equal(resumed.results.length, 20) + assert.equal(timers.length, 2) + await monitor.stop() + assert.equal(timers[1].cleared, true) + assertScannersEnded(await scannerActivities(f), 3) + assert.equal( + (await f.peer.pool.query('SELECT count(*) AS n FROM ha_request_contexts WHERE recovery_sealed_at IS NOT NULL')) + .rows[0].n, + 0, + ) +}) + +test('HA monitor leaves a connected owner alone then recovers its exact original proof and bills once after disconnection', async (t) => { + const f = await haBillingFixture(t) + const timers = intervals(t) + const data = await billingRequest(f, async () => {}, { stopOwner: false }) + let reads = 0 + const monitor = await monitorFor(t, f, async ({ requestId, nodeBinding }) => { + reads++ + assert.equal(requestId, data.commandIds[0]) + assert.deepEqual(nodeBinding, data.nodeBinding) + return statusFor(data, requestId) + }) + assert.equal((await monitor.start()).results[0].reason, 'owner_connected') + assert.equal(reads, 0) + assert.equal( + ( + await f.peer.pool.query('SELECT released_at FROM api_key_admissions WHERE id=$1', [ + data.reservation.reservationId, + ]) + ).rows[0].released_at, + null, + ) + await f.original._client.end() + timers[0].callback() + const recovered = await monitor.flush() + assert.equal(recovered.results[0].recovered, true) + assert.equal(recovered.results[0].billingHeld, false) + assert.equal(reads, 1) + timers[0].callback() + assert.deepEqual((await monitor.flush()).results, []) + await monitor.runOnce() + const key = (await f.peer.pool.query('SELECT requests,tokens_in,quota_used FROM api_keys WHERE id=$1', [data.key.id])) + .rows[0] + assert.equal(key.requests, 1) + assert.equal(key.tokens_in, 100) + assert.ok(Math.abs(Number(key.quota_used) - 0.001272) < 1e-12) + assert.equal((await f.peer.pool.query('SELECT count(*) AS n FROM ha_request_settlements')).rows[0].n, 1) + const reservation = ( + await f.peer.pool.query('SELECT settled_at,released_at FROM api_key_admissions WHERE id=$1', [ + data.reservation.reservationId, + ]) + ).rows[0] + assert.ok(reservation.settled_at) + assert.ok(reservation.released_at) + assert.ok( + (await f.peer.pool.query('SELECT released_at FROM execution_admissions WHERE id=$1', [data.admission.admissionId])) + .rows[0].released_at, + ) + assertScannersEnded(await scannerActivities(f), 4) +}) + +test('a real PG scan error completes only the scanner activity and logs its public code without retaining a scanner claim', async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f) + const messages = [] + t.mock.method(console, 'warn', (...args) => { + messages.push(args) + }) + const monitor = await monitorFor(t, f, async ({ requestId }) => statusFor(data, requestId)) + await f.recoveryGate.run(() => f.peer.query('ALTER TABLE ha_request_contexts RENAME TO synthetic_hidden_requests')) + const failed = await monitor.runOnce() + assert.equal(failed.errorCode, '42P01') + assert.deepEqual(messages, [['[ha-recovery] 42P01']]) + assertScannersEnded(await scannerActivities(f), 1) + assert.equal( + ( + await f.peer.pool.query('SELECT released_at FROM api_key_admissions WHERE id=$1', [ + data.reservation.reservationId, + ]) + ).rows[0].released_at, + null, + ) + await f.recoveryGate.run(() => f.peer.query('ALTER TABLE synthetic_hidden_requests RENAME TO ha_request_contexts')) + assert.equal((await monitor.runOnce()).results[0].billingHeld, false) + assertScannersEnded(await scannerActivities(f), 2) +}) + +for (const response of ['unavailable', 'not_found']) { + test(`HA monitor retains original unknown execution and quota when the node is ${response}`, async (t) => { + const f = await haBillingFixture(t) + const data = await billingRequest(f) + await f.recoveryGate.run(() => + f.peer + .prepare('UPDATE execution_jobs SET ended_at=NULL,node_completion=NULL WHERE request_id=$1') + .run(data.commandIds[0]), + ) + const monitor = await monitorFor(t, f, async () => { + if (response === 'unavailable') + throw Object.assign(new Error('synthetic private detail'), { code: 'synthetic_node_offline' }) + return { found: false, unknown: true } + }) + const result = await monitor.runOnce() + assert.equal(result.results[0].recovered, false) + assert.equal(result.results[0].reason, 'node_completion_unproven') + assert.equal( + ( + await f.peer.pool.query('SELECT released_at FROM execution_admissions WHERE id=$1', [ + data.admission.admissionId, + ]) + ).rows[0].released_at, + null, + ) + assert.equal( + ( + await f.peer.pool.query('SELECT released_at FROM api_key_admissions WHERE id=$1', [ + data.reservation.reservationId, + ]) + ).rows[0].released_at, + null, + ) + assert.equal( + ( + await f.peer.pool.query('SELECT completed_at FROM ' + f.recovery.qualified + '.activities WHERE id=$1', [ + f.originalGate.activityId, + ]) + ).rows[0].completed_at, + null, + ) + assert.equal((await f.peer.pool.query('SELECT count(*) AS n FROM ha_request_settlements')).rows[0].n, 0) + assertScannersEnded(await scannerActivities(f), 1) + }) +} + +test('HA monitor serializes runOnce and stop waits for its original in-flight read and durable settlement', async (t) => { + const f = await haBillingFixture(t) + const timers = intervals(t) + const data = await billingRequest(f) + let entered + const reading = new Promise((resolve) => { + entered = resolve + }) + let resolveStatus + const pendingStatus = new Promise((resolve) => { + resolveStatus = resolve + }) + let reads = 0 + const monitor = await monitorFor(t, f, async () => { + reads++ + entered() + return pendingStatus + }) + const first = monitor.start() + assert.equal(monitor.runOnce(), first) + assert.equal(monitor.flush(), first) + await reading + timers[0].callback() + assert.equal(reads, 1) + let stopped = false + const stop = monitor.stop().then(() => { + stopped = true + }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(timers[0].cleared, true) + assert.equal(stopped, false) + resolveStatus(statusFor(data, data.commandIds[0])) + await stop + assert.equal(stopped, true) + assert.equal((await first).results[0].billingHeld, false) + assertScannersEnded(await scannerActivities(f), 1) + assert.equal( + (await f.peer.pool.query('SELECT requests FROM api_keys WHERE id=$1', [data.key.id])).rows[0].requests, + 1, + ) +}) From 617131f5a7404492940f3d0ed7bad4d2c7df00a3 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:50:26 +0800 Subject: [PATCH 444/473] feat: poll exact HA orphan recovery under completed scan activities --- src/lib/core/ha-recovery-monitor.mjs | 64 ++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/lib/core/ha-recovery-monitor.mjs diff --git a/src/lib/core/ha-recovery-monitor.mjs b/src/lib/core/ha-recovery-monitor.mjs new file mode 100644 index 000000000..4baf7c311 --- /dev/null +++ b/src/lib/core/ha-recovery-monitor.mjs @@ -0,0 +1,64 @@ +import { HaRequestRecovery } from './ha-request-recovery.mjs' + +function reportFailure(error) { + const code = + typeof error?.code === 'string' && /^[A-Za-z0-9_]{1,80}$/.test(error.code) ? error.code : 'ha_recovery_failed' + console.warn(`[ha-recovery] ${code}`) + return code +} + +/** Each Pod scans independently; exact parent transactions and the settlement ledger provide idempotency. */ +export function createHaRecoveryMonitor({ db, coordinator, readNodeRecoveryStatus }) { + const recovery = new HaRequestRecovery(db, { coordinator, readNodeRecoveryStatus }) + let cursor = null + let inflight = null + let timer = null + let generation = 0 + + const failed = (error) => ({ results: [], nextCursor: cursor, errorCode: reportFailure(error) }) + + const runOnce = () => { + if (inflight) return inflight + inflight = coordinator + .runActivity({ kind: 'ha_request_recovery' }, async () => { + try { + const value = await recovery.scan({ limit: 20, afterRequestId: cursor }) + cursor = value.nextCursor + for (const result of value.results) { + if (result.errorCode) reportFailure({ code: result.errorCode }) + } + return { ended: true, value } + } catch (error) { + // Only historical node reads are sent. Their failure cannot leave work owned by this scan running. + const value = failed(error) + return { ended: true, reason: value.errorCode, value } + } + }) + .catch(failed) + .finally(() => { + inflight = null + }) + return inflight + } + + const flush = () => inflight || Promise.resolve() + + const stop = () => { + generation++ + if (timer) clearInterval(timer) + timer = null + return flush() + } + + const start = () => { + if (timer) return flush() + const current = ++generation + timer = setInterval(() => { + if (current === generation) void runOnce() + }, 5000) + timer.unref?.() + return runOnce() + } + + return { start, stop, flush, runOnce } +} From 564580f568107e19c737cc24e2195cd66cd4aace Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:51:55 +0800 Subject: [PATCH 445/473] test: verify AT-only projection with the prebuilt kernel and CLI --- ...rebuilt-credential-projection.e2e.test.mjs | 332 ++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 test/e2e/prebuilt-credential-projection.e2e.test.mjs diff --git a/test/e2e/prebuilt-credential-projection.e2e.test.mjs b/test/e2e/prebuilt-credential-projection.e2e.test.mjs new file mode 100644 index 000000000..c689078a4 --- /dev/null +++ b/test/e2e/prebuilt-credential-projection.e2e.test.mjs @@ -0,0 +1,332 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import { spawn } from 'node:child_process' +import { once } from 'node:events' + +const repository = path.resolve(import.meta.dirname, '../..') +// Use the same explicit payload overrides as auto-mode.e2e.test.mjs. No mock +// kernel or provider credentials are loaded from a developer's project. +const kernelBin = process.env.KIN_AUTOMODE_KERNEL || path.join(repository, 'bin/kin-kernel') +const cliBin = process.env.KIN_AUTOMODE_CLI || path.join(repository, 'share/wrap-cli/cli-node') + +test('the prebuilt kernel consumes an atomic AT-only projection without restarting its kernel or CLI', { + timeout: 45000, +}, async (t) => { + fs.accessSync(kernelBin, fs.constants.X_OK) + fs.accessSync(cliBin, fs.constants.X_OK) + const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-at-projection-')) + const home = path.join(temp, 'home') + const socketPath = path.join(temp, 'kernel.sock') + const credentials = path.join(home, '.claude', 'credentials.json') + const tokens = ['synthetic-at-before', 'synthetic-at-after'] + const internalToken = 'synthetic-local-internal-token' + const observations = [] + let child + let closed + let upstream + let stderrBytes = 0 + let spawnError = null + const summary = (result) => + JSON.stringify({ + status: result?.status, + type: result?.body?.type, + error_code: result?.body?.error?.code, + error_type: result?.body?.error?.type, + kernel_exit: child?.exitCode, + spawn_error: spawnError, + stderr_bytes: stderrBytes, + provider_calls: observations.length, + token_indices: observations.map((row) => row.tokenIndex), + }) + .replaceAll(tokens[0], '[redacted]') + .replaceAll(tokens[1], '[redacted]') + .replaceAll(internalToken, '[redacted]') + .slice(0, 800) + const publish = (token) => { + fs.writeFileSync( + credentials + '.next', + JSON.stringify({ + claudeAiOauth: { + accessToken: token, + expiresAt: Date.now() + 86400000, + accountUuid: 'synthetic-account', + scopes: ['user:inference', 'user:profile'], + subscriptionType: 'max', + }, + }), + { mode: 0o600, flag: 'wx' }, + ) + fs.renameSync(credentials + '.next', credentials) + } + const assertATOnly = () => { + const parsed = JSON.parse(fs.readFileSync(credentials, 'utf8')) + assert.deepEqual(Object.keys(parsed), ['claudeAiOauth']) + assert.deepEqual(Object.keys(parsed.claudeAiOauth).sort(), [ + 'accessToken', + 'accountUuid', + 'expiresAt', + 'scopes', + 'subscriptionType', + ]) + assert.equal(Object.hasOwn(parsed.claudeAiOauth, 'refreshToken'), false) + assert.equal(fs.statSync(credentials).mode & 0o777, 0o600) + } + const request = (endpoint, payload, timeout = 15000) => + new Promise((resolve, reject) => { + const req = http.request( + { + socketPath, + path: endpoint, + method: payload ? 'POST' : 'GET', + agent: false, + headers: { 'x-internal-token': internalToken, 'content-type': 'application/json' }, + timeout, + }, + (res) => { + let data = '' + res.on('data', (chunk) => { + data += chunk + if (data.length > 65536) res.destroy(new Error('synthetic response limit')) + }) + res.on('error', reject) + res.on('end', () => { + let body + try { + body = JSON.parse(data) + } catch {} + resolve({ status: res.statusCode, body, complete: res.complete }) + }) + }, + ) + req.on('error', reject) + req.on('timeout', () => req.destroy(new Error('synthetic local request timeout'))) + req.end(payload ? JSON.stringify(payload) : undefined) + }) + try { + fs.mkdirSync(path.dirname(credentials), { recursive: true, mode: 0o700 }) + fs.symlinkSync('credentials.json', path.join(path.dirname(credentials), '.credentials.json')) + publish(tokens[0]) + assertATOnly() + upstream = http.createServer(async (req, res) => { + try { + for await (const _ of req) { + /* Consume only this synthetic request. */ + } + } catch { + return res.destroy() + } + const tokenIndex = tokens.findIndex( + (token) => req.headers.authorization === `Bearer ${token}` || req.headers['x-api-key'] === token, + ) + observations.push({ method: req.method, path: new URL(req.url, 'http://local').pathname, tokenIndex }) + if (observations.at(-1).path !== '/v1/messages') return res.writeHead(404).end() + if (tokenIndex < 0) + return res.writeHead(401, { 'content-type': 'application/json' }).end( + JSON.stringify({ + type: 'error', + error: { type: 'authentication_error', message: 'synthetic credential mismatch' }, + }), + ) + res.writeHead(200, { 'content-type': 'text/event-stream' }) + const events = [ + { + type: 'message_start', + message: { + id: 'msg_synthetic', + type: 'message', + role: 'assistant', + model: 'claude-sonnet-4-6', + content: [], + usage: { input_tokens: 1, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'synthetic answer' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 2 } }, + { type: 'message_stop' }, + ] + for (const event of events) res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) + res.end() + }) + await new Promise((resolve, reject) => { + upstream.once('error', reject) + upstream.listen(0, '127.0.0.1', resolve) + }) + const url = `http://127.0.0.1:${upstream.address().port}` + const config = path.join(temp, 'kernel.json') + fs.writeFileSync( + config, + JSON.stringify({ + vm_id: 'synthetic-prebuilt', + socket_path: socketPath, + credential_path: credentials, + internal_token: internalToken, + provider: 'local_cli', + claude_bin: cliBin, + slots_per_worker: 1, + test_endpoints: true, + anthropic_base_url: url, + }), + { mode: 0o600 }, + ) + const env = { + ...process.env, + CLAUDE_CONFIG_DIR: home, + ANTHROPIC_BASE_URL: url, + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', + DISABLE_TELEMETRY: '1', + DISABLE_ERROR_REPORTING: '1', + NO_PROXY: '127.0.0.1,localhost', + KIN_ENVELOPE_PATH: path.join(temp, 'no-envelope.json'), + } + for (const key of [ + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'ALL_PROXY', + 'http_proxy', + 'https_proxy', + 'all_proxy', + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_AUTH_TOKEN', + 'CLAUDE_CODE_OAUTH_TOKEN', + 'KIN_SOCKS5', + 'KIN_HTTPS_PROXY', + 'KIN_CLAUDE_AI_OAUTH_JSON', + 'KIN_CLAUDE_CODE_OAUTH_TOKEN', + 'KIN_CRS_MOCK', + 'KIN_FAKE_SESSION_OAUTH', + ]) + delete env[key] + child = spawn(kernelBin, ['--gateway-worker', '--config', config], { + cwd: temp, + env, + stdio: ['ignore', 'ignore', 'pipe'], + detached: true, + }) + closed = once(child, 'close').then( + () => {}, + () => {}, + ) + child.once('error', (error) => { + spawnError = error.code || error.name + }) + child.stderr.on('data', (part) => { + stderrBytes += part.length + }) + const kernelPid = child.pid + let before + const deadline = Date.now() + 8000 + while (Date.now() < deadline && child.exitCode == null && !spawnError) { + try { + before = await request('/internal/health', null, 250) + if (before.status === 200 && before.body?.ready_slots === 1) break + } catch {} + await new Promise((resolve) => setTimeout(resolve, 50)) + } + assert.equal(before?.body?.ready_slots, 1, summary(before)) + assert.ok(Number.isInteger(before.body.cli_pid) && before.body.cli_pid > 0, summary(before)) + const socketInode = fs.statSync(socketPath).ino + const credentialInode = fs.statSync(credentials).ino + for (let index = 0; index < 2; index++) { + if (index === 1) publish(tokens[index]) + assertATOnly() + const result = await request('/internal/v1/messages', { + request_id: 'synthetic-rotation-' + index, + body: { + model: 'claude-sonnet-4-6', + max_tokens: 32, + messages: [{ role: 'user', content: 'synthetic local request' }], + }, + stream: false, + }) + assert.equal(result.status, 200, summary(result)) + assert.equal(result.complete, true, summary(result)) + assert.equal(result.body?.type, 'message', summary(result)) + assert.equal(result.body?.stop_reason, 'end_turn', summary(result)) + assert.equal( + result.body?.content?.find((block) => block.type === 'text')?.text, + 'synthetic answer', + summary(result), + ) + assert.equal(child.pid, kernelPid) + assert.equal(child.exitCode, null) + const health = await request('/internal/health') + assert.equal(health.body?.cli_pid, before.body.cli_pid, summary(health)) + assert.equal(health.body?.cli_restarts, before.body.cli_restarts, summary(health)) + assert.equal(fs.statSync(socketPath).ino, socketInode) + } + assert.notEqual(fs.statSync(credentials).ino, credentialInode, 'credential projection must use atomic replacement') + assertATOnly() + assert.deepEqual(observations, [ + { method: 'POST', path: '/v1/messages', tokenIndex: 0 }, + { method: 'POST', path: '/v1/messages', tokenIndex: 1 }, + ]) + t.diagnostic( + JSON.stringify({ + boundary: + 'prebuilt payload with AT-only synthetic credentials and localhost fake provider; no refresh/provider acceptance', + kernel_sha256: crypto.createHash('sha256').update(fs.readFileSync(kernelBin)).digest('hex'), + cli_sha256: crypto.createHash('sha256').update(fs.readFileSync(cliBin)).digest('hex'), + kernel_pid: kernelPid, + cli_pid: before.body.cli_pid, + cli_restarts: before.body.cli_restarts, + temp, + }), + ) + } finally { + let cleanupError + try { + if (child?.pid) { + const killGroup = (signal) => { + try { + process.kill(-child.pid, signal) + } catch (error) { + if (error.code !== 'ESRCH') throw error + } + } + killGroup('SIGTERM') + let timer + try { + await Promise.race([ + closed, + new Promise((resolve) => { + timer = setTimeout(resolve, 1000) + }), + ]) + } finally { + clearTimeout(timer) + } + killGroup('SIGKILL') + await closed + let groupPresent = true + for (let attempt = 0; attempt < 40; attempt++) { + try { + process.kill(-child.pid, 0) + } catch (error) { + if (error.code !== 'ESRCH') throw error + groupPresent = false + break + } + await new Promise((resolve) => setTimeout(resolve, 25)) + } + assert.equal(groupPresent, false, `owned process group still exists: ${child.pid}`) + } + } catch (error) { + cleanupError = error + } + if (upstream) { + upstream.closeAllConnections() + await new Promise((resolve) => upstream.close(resolve)) + } + fs.rmSync(temp, { recursive: true, force: true }) + assert.equal(fs.existsSync(temp), false) + if (cleanupError) throw cleanupError + t.diagnostic(`owned process group stopped, localhost provider closed, temporary directory removed: ${temp}`) + } +}) From 0fee98f1a18bba0f7f6eeae16a2d9705a959b417 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:45:43 +0800 Subject: [PATCH 446/473] test: require trusted offline HA node bootstrap commands --- test/unit/ha-node-bootstrap.test.mjs | 252 +++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 test/unit/ha-node-bootstrap.test.mjs diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs new file mode 100644 index 000000000..a9c870bbd --- /dev/null +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -0,0 +1,252 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import crypto from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import http from 'node:http' +import { fileURLToPath } from 'node:url' +import pg from 'pg' +import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' +import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' + +const nodeId = 'command-node' +const vmId = 'command-vm' +const allocationId = 'command-allocation' +const payload = Buffer.from( + JSON.stringify({ + credential: { + type: 'oauth', + access_token: 'synthetic-command-access', + refresh_token: 'synthetic-command-refresh', + account_uuid: 'synthetic-account', + org_uuid: 'synthetic-org', + email: 'synthetic@example.invalid', + }, + }), +) + +async function fixture(t) { + const db = getDb() + const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) + const coordinator = new GlobalMaintenanceCoordinator(db) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'nmc-')) + let node + let readerRole + let engine + let closed = false + const service = new NodeManagementGrants(db, { resolveNodeLink: async () => node.link }) + const close = async () => { + if (closed) return + closed = true + await node?.close() + if (engine) { + engine.closeAllConnections() + await new Promise((resolve) => engine.close(resolve)) + } + await coordinator.stop() + await peer.close() + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + if (readerRole) + await cleanup.query(`DROP OWNED BY ${quoteIdentifier(readerRole)}; DROP ROLE ${quoteIdentifier(readerRole)}`) + await cleanup.query( + `DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE; DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`, + ) + } finally { + await cleanup.end() + fs.rmSync(root, { recursive: true, force: true }) + } + } + t.after(close) + await coordinator.start() + await coordinator.acknowledgeEpoch(1) + readerRole = 'command_wire_' + crypto.randomBytes(8).toString('hex') + const password = crypto.randomBytes(24).toString('hex') + await db.pool.query( + `CREATE ROLE ${quoteIdentifier(readerRole)} LOGIN NOINHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD '${password}'`, + ) + const url = new URL(db.connectionString) + url.username = readerRole + url.password = password + const policy = { + image_id: 'sha256:' + 'a'.repeat(64), + network_id: 'none', + uid: process.getuid() || 1000, + gid: process.getgid() || 1000, + } + const policyDigest = crypto + .createHash('sha256') + .update( + [ + 'slotgate-policy-v1', + 'claude-docker-v1', + policy.image_id, + policy.network_id, + String(policy.uid), + String(policy.gid), + ].join('\0'), + ) + .digest('hex') + const calls = [] + let spec + const containerId = 'b'.repeat(64) + engine = http.createServer(async (req, res) => { + calls.push(`${req.method} ${req.url}`) + const parsed = new URL(req.url, 'http://synthetic-engine') + const send = (body, status = 200) => { + res.writeHead(status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(body)) + } + if (parsed.pathname === '/info') return send({ ID: 'synthetic-daemon' }) + if (parsed.pathname.startsWith('/images/')) + return send({ Id: policy.image_id, Config: { Labels: { 'kin.slot.full': '1' } } }) + if (parsed.pathname === '/containers/create') { + const chunks = [] + for await (const chunk of req) chunks.push(chunk) + spec = JSON.parse(Buffer.concat(chunks).toString()) + return send({ Id: containerId }, 201) + } + if (parsed.pathname === `/containers/${containerId}/start`) return res.writeHead(204).end() + if (parsed.pathname === `/containers/${containerId}/json`) + return send({ + Id: containerId, + Image: policy.image_id, + Config: { Labels: spec.Labels }, + State: { Running: true, Restarting: false, StartedAt: '2026-10-22T00:00:00.000000001Z' }, + }) + res.writeHead(404).end() + }) + const socket = path.join(root, 'engine.sock') + await new Promise((resolve, reject) => { + engine.once('error', reject) + engine.listen(socket, resolve) + }) + node = await realAllocationGate( + t, + { connection_string: url.toString(), authority_schema: service.schema, node_id: nodeId }, + { launch: policy, dockerSocket: socket }, + ) + const file = path.join(root, 'bootstrap.json'), + output = path.join(root, 'reader.sql'), + credentialFile = path.join(root, 'credential.json') + const config = { + database: { url: db.connectionString, schema: db.schema }, + node: { id: nodeId, ...node.link, presetId: 'claude-docker-v1', policyDigest }, + reader: { role: readerRole, sqlOutput: output }, + vm: { id: vmId, accountId: 'synthetic-account' }, + credentialFile, + } + fs.writeFileSync(file, JSON.stringify(config), { mode: 0o600 }) + fs.writeFileSync(credentialFile, payload, { mode: 0o600 }) + const cli = (command, extra = []) => + new Promise((resolve) => { + const env = { ...process.env } + delete env.NODE_TEST_CONTEXT + const child = import('node:child_process').then(({ spawn }) => { + const p = spawn( + process.execPath, + [ + fileURLToPath(new URL('../../scripts/ha-node-bootstrap.mjs', import.meta.url)), + command, + '--config', + file, + ...extra, + ], + { env, stdio: ['ignore', 'pipe', 'pipe'] }, + ) + let out = '', + err = '' + p.stdout.on('data', (c) => (out += c)) + p.stderr.on('data', (c) => (err += c)) + p.once('exit', (status) => resolve({ status, out, err })) + p.once('error', (e) => resolve({ status: -1, out, err: e.code })) + }) + }) + return { + db, + peer, + coordinator, + service, + cli, + file, + config, + output, + credentialFile, + node, + calls, + root, + close, + q: quoteIdentifier(service.schema), + } +} + +function nodeCase(name, callback) { + test(name, async (t) => { + const fx = await fixture(t) + try { + await callback(fx, t) + } finally { + await fx.close() + } + }) +} + +nodeCase( + 'trusted-init writes nonsecret reader SQL and slot-create persists only confirmed public metadata', + async (fx) => { + const initialized = await fx.cli('trusted-init') + assert.equal(initialized.status, 0, initialized.out + initialized.err) + const sql = fs.readFileSync(fx.output, 'utf8') + assert.equal(fs.statSync(fx.output).mode & 0o777, 0o600) + assert.equal(sql.includes('synthetic-command-refresh'), false) + assert.equal(sql.includes(fx.db.connectionString), false) + await fx.db.pool.query(sql) + const result = await fx.cli('slot-create') + assert.equal(result.status, 0, result.out + result.err) + assert.equal(result.out.includes('synthetic-command-refresh'), false) + assert.equal(result.err.includes('synthetic-command-access'), false) + const row = (await fx.peer.query('SELECT vm_json,access_token,refresh_token FROM vms WHERE id=$1', [vmId])).rows[0] + const vm = JSON.parse(row.vm_json) + assert.equal(vm.node_id, nodeId) + assert.equal(vm.claude.account_uuid, 'synthetic-account') + assert.ok(vm.fingerprint.device_id) + assert.ok(vm.fingerprint.session_id) + assert.equal(row.access_token, null) + assert.equal(row.refresh_token, null) + assert.equal(row.vm_json.includes('synthetic-command-'), false) + const before = fx.calls.length + const repeated = await fx.cli('slot-create') + assert.equal(repeated.status, 1) + assert.equal(fx.calls.length, before) + assert.equal((await fx.peer.query('SELECT count(*)::int AS n FROM vms')).rows[0].n, 1) + }, +) +nodeCase('bootstrap rejects public config permissions before any node request', async (fx) => { + fs.chmodSync(fx.file, 0o644) + const result = await fx.cli('trusted-init') + assert.equal(result.status, 1) + assert.match(result.err, /bootstrap_config_invalid/) + assert.equal(fx.calls.length, 0) +}) +nodeCase('bootstrap refuses an existing VM without overwriting its fingerprint or issuing grants', async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + await fx.db.query('INSERT INTO vms(id,vm_json) VALUES($1,$2)', [ + vmId, + JSON.stringify({ id: vmId, fingerprint: { device_id: 'original', session_id: 'original-session' } }), + ]) + const result = await fx.cli('slot-create') + assert.equal(result.status, 1) + assert.match(result.err, /bootstrap_vm_exists/) + assert.equal(fx.calls.length, 0) + assert.equal( + JSON.parse((await fx.peer.query('SELECT vm_json FROM vms WHERE id=$1', [vmId])).rows[0].vm_json).fingerprint + .device_id, + 'original', + ) +}) From 835d20d859444b17448c0f120af615c2af52d656 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:51:33 +0800 Subject: [PATCH 447/473] test: cover bootstrap refusal and unknown execution boundaries --- test/unit/ha-node-bootstrap.test.mjs | 141 ++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 2 deletions(-) diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs index a9c870bbd..8fcf6a9e8 100644 --- a/test/unit/ha-node-bootstrap.test.mjs +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -92,6 +92,8 @@ async function fixture(t) { ) .digest('hex') const calls = [] + let failStart = false, + beforeStart = null let spec const containerId = 'b'.repeat(64) engine = http.createServer(async (req, res) => { @@ -110,7 +112,10 @@ async function fixture(t) { spec = JSON.parse(Buffer.concat(chunks).toString()) return send({ Id: containerId }, 201) } - if (parsed.pathname === `/containers/${containerId}/start`) return res.writeHead(204).end() + if (parsed.pathname === `/containers/${containerId}/start`) { + if (beforeStart) await beforeStart() + return res.writeHead(failStart ? 500 : 204).end() + } if (parsed.pathname === `/containers/${containerId}/json`) return send({ Id: containerId, @@ -156,8 +161,9 @@ async function fixture(t) { file, ...extra, ], - { env, stdio: ['ignore', 'pipe', 'pipe'] }, + { env, stdio: ['pipe', 'pipe', 'pipe'] }, ) + p.stdin.end(extra.includes('--credential-fd') ? payload : undefined) let out = '', err = '' p.stdout.on('data', (c) => (out += c)) @@ -178,6 +184,12 @@ async function fixture(t) { credentialFile, node, calls, + setFailStart: () => { + failStart = true + }, + setBeforeStart: (fn) => { + beforeStart = fn + }, root, close, q: quoteIdentifier(service.schema), @@ -250,3 +262,128 @@ nodeCase('bootstrap refuses an existing VM without overwriting its fingerprint o 'original', ) }) + +nodeCase('slot-create accepts credential FD input without persisting it in config or public output', async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + fx.config.vm.fingerprint = { device_id: '9'.repeat(64), session_id: '11111111-2222-4333-8444-555555555555' } + delete fx.config.credentialFile + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const result = await fx.cli('slot-create', ['--credential-fd', '0']) + assert.equal(result.status, 0, result.out + result.err) + assert.equal(result.out.includes('synthetic-command-refresh'), false) + assert.deepEqual( + JSON.parse((await fx.peer.query('SELECT vm_json FROM vms WHERE id=$1', [vmId])).rows[0].vm_json).fingerprint, + fx.config.vm.fingerprint, + ) +}) +nodeCase('an unknown node launch retains bootstrap and VM claims and refuses replacement attempts', async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + fx.setFailStart() + const result = await fx.cli('slot-create') + assert.equal(result.status, 1) + assert.match(result.err, /node_launch_unconfirmed/) + assert.equal((await fx.peer.query('SELECT count(*)::int AS n FROM vms')).rows[0].n, 0) + const bg = ( + await fx.peer.query('SELECT unknown_at,finished_at FROM background_task_runs WHERE task_key=$1', [ + `ha-node-bootstrap:${vmId}`, + ]) + ).rows[0] + assert.ok(bg.unknown_at) + assert.equal(bg.finished_at, null) + assert.ok( + (await fx.peer.query("SELECT uncertain_at FROM vm_operations WHERE kind='node_launch'")).rows[0].uncertain_at, + ) + const before = fx.calls.length, + retry = await fx.cli('slot-create') + assert.equal(retry.status, 1) + assert.match(retry.err, /bootstrap_claim_unavailable/) + assert.equal(fx.calls.length, before) +}) +nodeCase('an existing VM inserted during node launch is not overwritten by metadata publication', async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + fx.setBeforeStart(async () => + fx.db.query('INSERT INTO vms(id,vm_json) VALUES($1,$2)', [ + vmId, + JSON.stringify({ id: vmId, fingerprint: { device_id: 'racing-owner', session_id: 'racing-session' } }), + ]), + ) + const result = await fx.cli('slot-create') + assert.equal(result.status, 1) + assert.match(result.err, /bootstrap_vm_exists/) + assert.equal( + JSON.parse((await fx.peer.query('SELECT vm_json FROM vms WHERE id=$1', [vmId])).rows[0].vm_json).fingerprint + .device_id, + 'racing-owner', + ) + const bg = ( + await fx.peer.query('SELECT unknown_at,finished_at FROM background_task_runs WHERE task_key=$1', [ + `ha-node-bootstrap:${vmId}`, + ]) + ).rows[0] + assert.ok(bg.unknown_at) + assert.equal(bg.finished_at, null) +}) + +nodeCase('private credential permissions and exact account identity are checked before node admission', async (fx) => { + fs.chmodSync(fx.credentialFile, 0o644) + let result = await fx.cli('slot-create') + assert.equal(result.status, 1) + assert.match(result.err, /bootstrap_credential_invalid/) + fs.chmodSync(fx.credentialFile, 0o600) + const payload = JSON.parse(fs.readFileSync(fx.credentialFile, 'utf8')) + payload.credential.account_uuid = 'different-account' + fs.writeFileSync(fx.credentialFile, JSON.stringify(payload)) + result = await fx.cli('slot-create') + assert.equal(result.status, 1) + assert.match(result.err, /bootstrap_credential_invalid/) + assert.equal(fx.calls.length, 0) +}) +nodeCase('trusted-init rejects a symlink configuration without creating authority tables', async (fx) => { + const target = path.join(fx.root, 'real-config.json') + fs.renameSync(fx.file, target) + fs.symlinkSync(target, fx.file) + const result = await fx.cli('trusted-init') + assert.equal(result.status, 1) + assert.match(result.err, /bootstrap_config_invalid/) + assert.equal( + (await fx.peer.query('SELECT to_regclass($1) AS present', [`${fx.service.schema}.allocations`])).rows[0].present, + null, + ) +}) + +nodeCase( + 'a busy VM before the first node call is known not started and permits a later explicit invocation', + async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + const { SharedVmOperations } = await import('../../src/lib/vm/shared-vm-operations.mjs') + const held = await new SharedVmOperations(fx.db).tryAcquire({ + vmId, + accountId: 'synthetic-account', + kind: 'synthetic_hold', + }) + try { + const first = await fx.cli('slot-create') + assert.equal(first.status, 1) + assert.equal(fx.calls.length, 0) + const bg = ( + await fx.peer.query('SELECT finished_at,unknown_at FROM background_task_runs WHERE task_key=$1', [ + `ha-node-bootstrap:${vmId}`, + ]) + ).rows[0] + assert.ok(bg.finished_at) + assert.equal(bg.unknown_at, null) + } finally { + await held.release() + } + const retry = await fx.cli('slot-create') + assert.equal(retry.status, 0, retry.out + retry.err) + }, +) From d3049840daa4b3c9e6a4c3643631bb3337696ca5 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:52:42 +0800 Subject: [PATCH 448/473] feat: add trusted offline HA node bootstrap CLI --- docs/deploy/ha-node-bootstrap.md | 51 +++++ scripts/ha-node-bootstrap.mjs | 373 +++++++++++++++++++++++++++++++ 2 files changed, 424 insertions(+) create mode 100644 docs/deploy/ha-node-bootstrap.md create mode 100644 scripts/ha-node-bootstrap.mjs diff --git a/docs/deploy/ha-node-bootstrap.md b/docs/deploy/ha-node-bootstrap.md new file mode 100644 index 000000000..d5d06ea78 --- /dev/null +++ b/docs/deploy/ha-node-bootstrap.md @@ -0,0 +1,51 @@ +# HA 节点的可信离线初始化 + +`scripts/ha-node-bootstrap.mjs` 提供 `trusted-init` 和 `slot-create` 两个操作者命令。它不会安装节点、修改 Docker daemon、上传镜像、创建数据库角色或增加 HTTP 管理接口。执行前须由可信部署流程安装固定版本节点 daemon、镜像/网络/UID/GID policy、节点证书与 PostgreSQL 连接配置，并通过独立可信渠道核对 TLS pin 与 policy digest。 + +运行环境需要 Node.js 22、PostgreSQL 17 应用 owner 的连接，以及到所选节点的固定 TLS 链路。配置必须是当前操作者所有、权限恰为 `0600` 的普通文件；拒绝符号链接和 hard link。业务 schema 必须显式指定且不能是 `public`、PostgreSQL 系统 schema 或 node/coord 派生 schema。DBA 预先创建无继承、无成员关系、无超级用户/建库/建角色/复制/BYPASSRLS 权限的独立 LOGIN reader 角色。密码由 DBA 单独配置，不能写入命令参数。 + +以下是配置结构。用安全编辑器写入私有文件，示例字符串必须替换；不要把实际连接串、节点 token 或凭据贴到终端历史。 + +```json +{ + "database": { "url": "postgresql://APP_OWNER@DB/APP", "schema": "vm2api" }, + "node": { + "id": "original-node-id", + "address": "192.0.2.10", + "port": 9443, + "token": "REPLACE_WITH_PRIVATE_NODE_TRANSPORT_TOKEN", + "certSha256": "REPLACE_WITH_64_LOWERCASE_HEX", + "presetId": "claude-docker-v1", + "policyDigest": "REPLACE_WITH_64_LOWERCASE_HEX" + }, + "reader": { "role": "node_reader", "sqlOutput": "/private/node-reader.sql" }, + "vm": { "id": "original-vm-id", "accountId": "original-account-uuid" }, + "credentialFile": "/private/initial-credential.json" +} +``` + +node ID、VM ID、account UUID 使用原始值。TLS/policy 字段必须与应用的固定 HA trust 文件以及节点配置一致。`policyDigest` 的生成规则见 [节点初次发布协议](../plans/2026-10-22-node-initial-publication.md)，不能从未授权的 runtime State 猜测。`vm.fingerprint` 可显式提供 `{device_id:64位小写hex,session_id:UUID}`；未提供时仅在首次新记录中生成并与该记录原子保存。 + +先运行： + +```sh +node scripts/ha-node-bootstrap.mjs trusted-init --config /private/bootstrap.json +``` + +该命令显式安装 allocation、command 和 refresh 三套 private authority/coordination marker。它不授予运行时自动升级权限，也不创建角色。stdout 仅输出节点 ID、私有 authority schema 和 SQL 文件位置。SQL 文件以 `0600`、禁止覆盖的方式创建，包含 reader 与节点的绑定以及最小 view/function GRANT，不含密码或 DSN。 + +DBA 检查并以应用 owner 身份执行该 SQL 文件。随后把返回的 authority schema、原 node ID 和该 reader 的连接串装入节点原有 `management` 配置；连接串只保存在受控节点文件。节点 reader 不能修改业务表或签发 grant。数据库连接角色拥有额外权限时，节点 authorizer 会拒绝。 + +初始凭据文件也是当前操作者所有的 `0600` 普通文件，最大 64 KiB，结构为 `{ "credential": { "type": "oauth", "account_uuid": "原始账号UUID", "access_token": "…", "refresh_token": "…", "expires_at": 0 } }`。`expires_at` 可省略；实际值须是非负整数。也支持既有 `setup-token`、`official-setup-token`、`apikey` 固定类型，遵循节点初始发布字段限制。account UUID 必须与配置一致。不接受 provider URL、path、argv、env 或任意配置内容。 + +```sh +node scripts/ha-node-bootstrap.mjs slot-create --config /private/bootstrap.json +``` + +也可从显式文件描述符读取凭据。此时从配置移除 `credentialFile`，调用 `slot-create --config /private/bootstrap.json --credential-fd 0` 并由操作者的受控 secret 管道提供 stdin。凭据内容不进入 argv、stdout 或 PostgreSQL VM 元数据。 + +`slot-create` 持有一个持久 bootstrap BG claim，顺序执行独立的 allocate、publish-initial、launch VM operation。每一步都等待严格 receipt 后再进入下一步。RT 仅保存在节点未挂载的 canonical 路径；guest 只得到 AT 投影。launch receipt 确认后才原子新建公开 VM 元数据和指纹。已有 VM、并发插入或未知旧任务均拒绝，不覆盖记录。stdout 只输出公开 IDs、manifest 摘要与原 captured Fence；它证明节点 launch/元数据步骤，不证明真实模型推理成功。 + +发生超时、断连、节点启动未知或元数据提交失败时，相关持久 claim 保留。再次执行不会换 command ID 重发。首个节点回调前明确 busy 的拒绝会结束本次未提交任务，操作者可稍后显式再运行。不能删 claim、改 VM ID 或重新发 allocation 来猜测恢复；须先按节点终止/恢复证据流程人工核实。CLI 不提供未知任务自动恢复，也不升级旧的 RT 已暴露 manifest。 + +测试只覆盖合成凭据、任务 PostgreSQL、TLS 和 fake Engine；没有真实 provider 或现有节点操作。闭源 kernel 在投影更新后是否重新加载 AT，仍需独立业务验证。旧 HA credential monitor 不能因此直接启用。 diff --git a/scripts/ha-node-bootstrap.mjs b/scripts/ha-node-bootstrap.mjs new file mode 100644 index 000000000..5c4464c40 --- /dev/null +++ b/scripts/ha-node-bootstrap.mjs @@ -0,0 +1,373 @@ +#!/usr/bin/env node +import fs from 'node:fs' +import path from 'node:path' +import net from 'node:net' +import crypto from 'node:crypto' +import { fileURLToPath } from 'node:url' +import { createDatabase, getActivityCoordinator, withTransaction } from '../src/lib/db/database.mjs' +import { NodeManagementGrants } from '../src/lib/cluster/node-management-grants.mjs' +import { NodeCredentialRefreshGrants } from '../src/lib/cluster/node-credential-refresh-grants.mjs' +import { SharedVmOperations } from '../src/lib/vm/shared-vm-operations.mjs' +import { SharedBackgroundTasks } from '../src/lib/core/shared-background-tasks.mjs' +import { VmsRepo } from '../src/lib/db/repos/vms-repo.mjs' + +function failure(code, ended = false) { + return Object.assign(new Error(code), { code, ...(ended ? { activityEnded: true } : {}) }) +} +function shape(value, required, optional = []) { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + required.some((k) => !Object.hasOwn(value, k)) || + Object.keys(value).some((k) => !required.includes(k) && !optional.includes(k)) + ) + throw failure('bootstrap_config_invalid', true) +} +function id(value) { + if (typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(value)) + throw failure('bootstrap_config_invalid', true) + return value +} +function boundedRead(fd, limit) { + const chunks = [] + let size = 0 + while (size <= limit) { + const part = Buffer.alloc(Math.min(8192, limit + 1 - size)) + const count = fs.readSync(fd, part, 0, part.length, null) + if (!count) break + chunks.push(part.subarray(0, count)) + size += count + } + if (!size || size > limit) throw failure('bootstrap_input_invalid', true) + return Buffer.concat(chunks) +} +function privateRead(file, limit, code) { + let fd + try { + if (typeof file !== 'string' || !path.isAbsolute(file)) throw new Error() + fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW) + const stat = fs.fstatSync(fd) + if ( + !stat.isFile() || + stat.nlink !== 1 || + (stat.mode & 0o777) !== 0o600 || + stat.uid !== process.geteuid() || + stat.size > limit + ) + throw new Error() + return boundedRead(fd, limit) + } catch { + throw failure(code, true) + } finally { + if (fd !== undefined) fs.closeSync(fd) + } +} +function configFrom(file) { + let c + try { + c = JSON.parse(privateRead(file, 65536, 'bootstrap_config_invalid')) + } catch { + throw failure('bootstrap_config_invalid', true) + } + shape(c, ['database', 'node'], ['reader', 'vm', 'credentialFile']) + shape(c.database, ['url', 'schema']) + if ( + typeof c.database.url !== 'string' || + !c.database.url || + !/^postgres(?:ql)?:\/\//.test(c.database.url) || + typeof c.database.schema !== 'string' || + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(c.database.schema) || + c.database.schema === 'public' || + /^(pg_|vm2api_node_|vm2api_coord_)/.test(c.database.schema) + ) + throw failure('bootstrap_config_invalid', true) + shape(c.node, ['id', 'address', 'port', 'token', 'certSha256', 'presetId', 'policyDigest']) + id(c.node.id) + if ( + !net.isIP(c.node.address) || + !Number.isInteger(c.node.port) || + c.node.port < 1 || + c.node.port > 65535 || + typeof c.node.token !== 'string' || + !/^[\x21-\x7e]{32,256}$/.test(c.node.token) || + !/^[a-f0-9]{64}$/.test(c.node.certSha256) || + !/^[a-f0-9]{64}$/.test(c.node.policyDigest) || + c.node.presetId !== 'claude-docker-v1' + ) + throw failure('bootstrap_config_invalid', true) + if (c.reader !== undefined) { + shape(c.reader, ['role', 'sqlOutput']) + if ( + typeof c.reader.role !== 'string' || + !/^[a-z_][a-z0-9_]{0,62}$/.test(c.reader.role) || + typeof c.reader.sqlOutput !== 'string' || + !path.isAbsolute(c.reader.sqlOutput) + ) + throw failure('bootstrap_config_invalid', true) + } + if (c.vm !== undefined) { + shape(c.vm, ['id', 'accountId'], ['fingerprint']) + id(c.vm.id) + id(c.vm.accountId) + if (c.vm.fingerprint !== undefined) { + shape(c.vm.fingerprint, ['device_id', 'session_id']) + if ( + typeof c.vm.fingerprint.device_id !== 'string' || + !/^[a-f0-9]{64}$/.test(c.vm.fingerprint.device_id) || + typeof c.vm.fingerprint.session_id !== 'string' || + !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(c.vm.fingerprint.session_id) + ) + throw failure('bootstrap_config_invalid', true) + } + } + return c +} +function credentialFrom(config, fd) { + let bytes, value + try { + if (fd !== undefined) { + if (config.credentialFile !== undefined || !Number.isSafeInteger(fd) || fd < 0) throw new Error() + bytes = boundedRead(fd, 65536) + } else bytes = privateRead(config.credentialFile, 65536, 'bootstrap_credential_invalid') + value = JSON.parse(bytes) + shape(value, ['credential']) + const c = value.credential + shape( + c, + ['type', 'account_uuid'], + ['access_token', 'refresh_token', 'api_key', 'expires_at', 'scopes', 'org_uuid', 'email'], + ) + id(c.account_uuid) + if (c.account_uuid !== config.vm?.accountId) throw new Error() + for (const name of ['access_token', 'refresh_token', 'api_key']) + if ( + c[name] !== undefined && + (typeof c[name] !== 'string' || Buffer.byteLength(c[name]) > 16384 || !c[name].trim()) + ) + throw new Error() + if (c.org_uuid !== undefined) id(c.org_uuid) + if ( + c.email !== undefined && + (typeof c.email !== 'string' || Buffer.byteLength(c.email) > 512 || /[\x00-\x1f\x7f]/.test(c.email)) + ) + throw new Error() + if (c.expires_at !== undefined && (!Number.isSafeInteger(c.expires_at) || c.expires_at < 0)) throw new Error() + if ( + c.scopes !== undefined && + (!Array.isArray(c.scopes) || + c.scopes.length > 32 || + c.scopes.some((s) => typeof s !== 'string' || !s || Buffer.byteLength(s) > 128 || /[\x00\r\n]/.test(s))) + ) + throw new Error() + if (c.type === 'apikey') { + if (!c.api_key || c.access_token || c.refresh_token || c.expires_at || c.scopes?.length) throw new Error() + } else if (['oauth', 'setup-token', 'official-setup-token'].includes(c.type)) { + if (!c.access_token || c.api_key || (c.type !== 'oauth' && c.refresh_token)) throw new Error() + } else throw new Error() + return { bytes, credential: c } + } catch { + throw failure('bootstrap_credential_invalid', true) + } +} +async function trustedInit(db, c, output) { + if (!c.reader) throw failure('bootstrap_config_invalid', true) + const role = ( + await db.query( + `SELECT oid,rolsuper,rolinherit,rolcreaterole,rolcreatedb,rolreplication,rolbypassrls,rolcanlogin,rolname=current_user AS owner FROM pg_roles WHERE rolname=$1`, + [c.reader.role], + ) + ).rows[0] + if ( + !role || + role.owner || + role.rolsuper || + role.rolinherit || + role.rolcreaterole || + role.rolcreatedb || + role.rolreplication || + role.rolbypassrls || + !role.rolcanlogin || + (await db.query('SELECT EXISTS(SELECT 1 FROM pg_auth_members WHERE member=$1) AS member', [role.oid])).rows[0] + .member + ) + throw failure('bootstrap_reader_invalid', true) + const grants = new NodeManagementGrants(db), + refresh = new NodeCredentialRefreshGrants(db) + await grants.initialize() + await grants.initializeCommands() + await refresh.initialize() + const sql = + grants.readerRoleSql({ role: c.reader.role, nodeId: c.node.id }) + + '\n' + + grants.commandReaderRoleSql({ role: c.reader.role }) + + '\n' + + refresh.readerRoleSql({ role: c.reader.role }) + + '\n' + try { + fs.writeFileSync(output, sql) + fs.fsyncSync(output) + } catch { + throw failure('bootstrap_output_unavailable', true) + } + return { ok: true, command: 'trusted-init', node_id: c.node.id, reader_sql_file: c.reader.sqlOutput } +} +async function slotCreate(db, c, input) { + if (!c.vm) throw failure('bootstrap_config_invalid', true) + const vms = new VmsRepo(db) + if (await vms.get(c.vm.id)) throw failure('bootstrap_vm_exists', true) + if (!(await new NodeCredentialRefreshGrants(db).state()).enabled) + throw failure('bootstrap_authority_unavailable', true) + const tasks = new SharedBackgroundTasks(db), + operations = new SharedVmOperations(db), + grants = new NodeManagementGrants(db, { resolveNodeLink: async () => c.node }) + let nodeCall = false + let result + try { + result = await tasks.run( + { taskKey: `ha-node-bootstrap:${c.vm.id}`, executionId: crypto.randomUUID() }, + async () => { + try { + const allocationId = crypto.randomUUID(), + publishId = crypto.randomUUID(), + launchId = crypto.randomUUID() + const run = (kind, callback) => operations.run(c.vm.id, callback, { kind, accountId: c.vm.accountId }) + const allocated = await run('node_allocate', (operation) => { + nodeCall = true + return grants.allocate({ operation, nodeId: c.node.id, vmId: c.vm.id, allocationId }) + }) + const common = { + nodeId: c.node.id, + vmId: c.vm.id, + allocationId, + capturedFence: allocated.fence, + presetId: c.node.presetId, + policyDigest: c.node.policyDigest, + } + const published = await run('node_publish_initial', (operation) => + grants.publishInitial({ ...common, operation, commandId: publishId, payload: input.bytes }), + ) + const launched = await run('node_launch', (operation) => + grants.launch({ ...common, operation, commandId: launchId, manifestDigest: published.manifest_digest }), + ) + const cred = input.credential, + expiry = cred.expires_at > 0 && cred.expires_at < 10000000000 ? cred.expires_at * 1000 : cred.expires_at + const vm = { + id: c.vm.id, + node_id: c.node.id, + runtime: 'docker', + status: 'running', + schedulable: true, + timezone: 'UTC', + locale: 'en-US', + fingerprint: c.vm.fingerprint || { + device_id: crypto.randomBytes(32).toString('hex'), + session_id: crypto.randomUUID(), + }, + claude: { + account_uuid: c.vm.accountId, + has_access: true, + has_refresh: !!cred.refresh_token, + ...(expiry ? { expires_at: expiry } : {}), + ...(cred.org_uuid ? { org_uuid: cred.org_uuid } : {}), + ...(cred.email ? { email: cred.email } : {}), + }, + } + await withTransaction(db, async () => { + const inserted = await db.query( + 'INSERT INTO vms(id,vm_json) VALUES($1,$2) ON CONFLICT(id) DO NOTHING RETURNING id', + [vm.id, JSON.stringify(vm)], + ) + if (inserted.rowCount !== 1) throw failure('bootstrap_vm_exists') + await vms.upsertFromVmJson(vm) + }) + return { + terminal: true, + value: { + ok: true, + command: 'slot-create', + node_id: c.node.id, + vm_id: c.vm.id, + allocation_id: allocationId, + publish_command_id: publishId, + launch_command_id: launchId, + manifest_revision: launched.manifest_revision, + manifest_digest: launched.manifest_digest, + fence: launched.fence, + }, + sharedResult: { ok: true, vmId: c.vm.id, launchCommandId: launchId }, + } + } catch (error) { + if (!nodeCall && ['vm_operation_active', 'vm_execution_active'].includes(error.code)) { + error.taskNotStarted = true + error.activityEnded = true + } + throw error + } + }, + ) + } catch (error) { + if (!nodeCall && error.code === 'database_maintenance') error.activityEnded = true + throw error + } + if (!result.ran) throw failure('bootstrap_claim_unavailable', true) + return result.value +} +export async function main(args = process.argv.slice(2)) { + let db, + output, + config, + outputWritten = false + try { + const command = args.shift() + if (!['trusted-init', 'slot-create'].includes(command)) throw failure('bootstrap_arguments_invalid', true) + let configFile, credentialFd + while (args.length) { + const key = args.shift(), + value = args.shift() + if (key === '--config' && configFile === undefined) configFile = value + else if (key === '--credential-fd' && credentialFd === undefined && /^(0|[1-9][0-9]*)$/.test(value)) + credentialFd = Number(value) + else throw failure('bootstrap_arguments_invalid', true) + } + config = configFrom(configFile) + if (command === 'trusted-init') { + if (!config.reader || credentialFd !== undefined) throw failure('bootstrap_arguments_invalid', true) + try { + output = fs.openSync( + config.reader.sqlOutput, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, + 0o600, + ) + } catch { + throw failure('bootstrap_output_unavailable', true) + } + } + const input = command === 'slot-create' ? credentialFrom(config, credentialFd) : null + process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' + db = await createDatabase({ connectionString: config.database.url, schema: config.database.schema }) + const coordinator = getActivityCoordinator(db), + state = await coordinator.getState() + await coordinator.acknowledgeEpoch(state.epoch) + await db.bootstrapActivity.complete({ ended: true, reason: 'node_bootstrap_initialized' }) + const result = await coordinator.runActivity({ kind: 'node_bootstrap_command', requestId: command }, async () => ({ + ended: true, + value: command === 'trusted-init' ? await trustedInit(db, config, output) : await slotCreate(db, config, input), + })) + outputWritten = command === 'trusted-init' + process.stdout.write(JSON.stringify(result) + '\n') + return 0 + } catch (error) { + const code = + typeof error?.code === 'string' && /^[a-z][a-z0-9_]{0,100}$/.test(error.code) ? error.code : 'bootstrap_failed' + process.stderr.write(JSON.stringify({ ok: false, code }) + '\n') + return 1 + } finally { + if (output !== undefined) { + fs.closeSync(output) + if (!outputWritten) fs.unlinkSync(config.reader.sqlOutput) + } + await db?.close().catch(() => {}) + } +} +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = await main() From e632748ae1662c52701220dddf794190de4a4ee8 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:53:43 +0800 Subject: [PATCH 449/473] test: require public authority schema in bootstrap output --- test/unit/ha-node-bootstrap.test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs index 8fcf6a9e8..099aea6fc 100644 --- a/test/unit/ha-node-bootstrap.test.mjs +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -212,6 +212,7 @@ nodeCase( async (fx) => { const initialized = await fx.cli('trusted-init') assert.equal(initialized.status, 0, initialized.out + initialized.err) + assert.equal(JSON.parse(initialized.out).authority_schema, fx.service.schema) const sql = fs.readFileSync(fx.output, 'utf8') assert.equal(fs.statSync(fx.output).mode & 0o777, 0o600) assert.equal(sql.includes('synthetic-command-refresh'), false) From 933b9fd7b15fcb1e019d364defdd3146bd98f105 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:54:37 +0800 Subject: [PATCH 450/473] fix: report the initialized node authority schema --- scripts/ha-node-bootstrap.mjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/ha-node-bootstrap.mjs b/scripts/ha-node-bootstrap.mjs index 5c4464c40..bba94213d 100644 --- a/scripts/ha-node-bootstrap.mjs +++ b/scripts/ha-node-bootstrap.mjs @@ -210,7 +210,13 @@ async function trustedInit(db, c, output) { } catch { throw failure('bootstrap_output_unavailable', true) } - return { ok: true, command: 'trusted-init', node_id: c.node.id, reader_sql_file: c.reader.sqlOutput } + return { + ok: true, + command: 'trusted-init', + node_id: c.node.id, + authority_schema: grants.schema, + reader_sql_file: c.reader.sqlOutput, + } } async function slotCreate(db, c, input) { if (!c.vm) throw failure('bootstrap_config_invalid', true) From 3e56ed5309eaf79b5739842d0d6e8186c94de840 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:57:45 +0800 Subject: [PATCH 451/473] feat: run HA orphan recovery within gateway lifecycle --- Dockerfile | 1 + docs/deploy/kubernetes-ha.md | 2 +- src/server.mjs | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 77d004651..01822f35e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # Control plane. Only source is in git: the console and Go helpers build here, and the # prebuilt slot payloads come from the release pinned in deploy/payload.json. +ARG BUILDPLATFORM=linux/amd64 FROM --platform=$BUILDPLATFORM node:22-bookworm-slim AS web RUN npm install -g pnpm@10 WORKDIR /src/web diff --git a/docs/deploy/kubernetes-ha.md b/docs/deploy/kubernetes-ha.md index b81704edf..d22cb72f0 100644 --- a/docs/deploy/kubernetes-ha.md +++ b/docs/deploy/kubernetes-ha.md @@ -24,7 +24,7 @@ 1. PostgreSQL 17、相同业务 schema、相同业务加密密钥。所有副本使用相同管理员配置和初始 API key。 2. 至少两个可调度的 Kubernetes 节点。必需的 Pod 反亲和保证副本分布在不同节点。默认滚动更新为 `maxUnavailable: 0, maxSurge: 1`，因此两个副本更新时需要第三个可调度节点。 3. 已安装并校验固定镜像 ID、网络 ID 和 UID/GID 的远端 Go 节点服务。节点的原始 Docker/SSH 权限不能交给控制面 Pod。 -4. 通过可信初始化安装独立 allocation、command 私有 authority、节点专用只读 PostgreSQL role，以及授权的 allocation → 首次凭据发布 → launch 回执链。普通应用启动不会自行安装这些授权。相关接口见 [节点授权](../plans/2026-10-22-node-command-authority.md) 和 [首次发布](../plans/2026-10-22-node-initial-publication.md)。目前尚无面向操作者的完整安装命令。 +4. 通过可信初始化安装独立 allocation、command 私有 authority、节点专用只读 PostgreSQL role，以及授权的 allocation → 首次凭据发布 → launch 回执链。普通应用启动不会自行安装这些授权。相关接口见 [节点授权](../plans/2026-10-22-node-command-authority.md) 和 [首次发布](../plans/2026-10-22-node-initial-publication.md)。可信离线初始化与槽位创建命令见 [节点初始化 CLI](ha-node-bootstrap.md)。该命令不安装节点服务或配置宿主 Docker。 从旧安装切换前停止旧控制面并核实在途任务。`kubernetes` 单实例模式与 `kubernetes-ha` 多副本模式使用互斥的 PostgreSQL 模式锁，不能并存。旧版独立进程不持有该锁，必须由操作者停止。 diff --git a/src/server.mjs b/src/server.mjs index 414db7d56..b66cd5dee 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -111,6 +111,7 @@ import { HaVmCatalog, bindHaVmCatalog } from './lib/vm/ha-vm-catalog.mjs' import { captureNodeExecution } from './lib/cluster/node-execution-capabilities.mjs' import { createNodeRecoveryReader } from './lib/cluster/node-request-recovery.mjs' import { withHaRequestContext } from './lib/core/ha-request-context.mjs' +import { createHaRecoveryMonitor } from './lib/core/ha-recovery-monitor.mjs' import { ControlPlaneLifecycle, withTimeout } from './lib/core/control-plane-lifecycle.mjs' import { getSharedBackgroundTasks } from './lib/core/shared-background-tasks.mjs' import { ControlPlaneLock } from './lib/db/control-plane-lock.mjs' @@ -162,6 +163,9 @@ let _shuttingDown = false const dataDir = cfg.paths.data || path.join(cfg.paths.root, 'data') const db = await openDatabase({ dataDir }) const activityCoordinator = getActivityCoordinator(db) +const haRecoveryMonitor = haMode + ? createHaRecoveryMonitor({ db, coordinator: activityCoordinator, readNodeRecoveryStatus }) + : null db.bootstrapActivity?.enter() const haCatalog = haMode ? new HaVmCatalog(db) : null if (haCatalog) { @@ -1709,6 +1713,7 @@ async function shutdown(signal, { failed = false } = {}) { clusterRoutes.beginDrain() slotShell.beginDrain() console.log(`[shutdown] ${signal} — closing`) + void haRecoveryMonitor?.stop() const deadline = Date.now() + shutdownGraceMs const forceClose = () => { lifecycle.forceClose() @@ -1821,6 +1826,7 @@ function maintenanceMonitors() { ? [] : [healthMonitor, credentialRefreshMonitor, kernelWatchdog, cliNodeGuard, usageProbeMonitor, catalogMonitor]), notifyMonitor, + haRecoveryMonitor, ] } @@ -1889,6 +1895,7 @@ server.listen(cfg.port, cfg.host, async () => { } } } + await haRecoveryMonitor?.start() try { notifyMonitor?.start?.({ immediate: true }) } catch (e) { From 90a64af442d8584c842a86d920af12d9268275cc Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 01:59:54 +0800 Subject: [PATCH 452/473] test: reproduce unbound pre-dispatch command IDs retaining HA quota --- test/fixtures/ha-protocol-billing.mjs | 27 +++++++-- test/unit/ha-protocol-billing.test.mjs | 9 ++- test/unit/ha-request-billing.test.mjs | 76 ++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 5 deletions(-) diff --git a/test/fixtures/ha-protocol-billing.mjs b/test/fixtures/ha-protocol-billing.mjs index b517dd8bd..16c0e1e9e 100644 --- a/test/fixtures/ha-protocol-billing.mjs +++ b/test/fixtures/ha-protocol-billing.mjs @@ -5,6 +5,7 @@ import os from 'node:os' import path from 'node:path' import { EventEmitter } from 'node:events' import { Readable } from 'node:stream' +import { randomUUID } from 'node:crypto' import { haBillingFixture, syntheticUsageReceipt } from './ha-billing.mjs' import { syntheticFence } from './fake-slotgate.mjs' import { withHaRequestContext, withHaRequestAttempt } from '../../src/lib/core/ha-request-context.mjs' @@ -12,8 +13,10 @@ import { ApiKeyStore } from '../../src/lib/admin/api-keys.mjs' import { ExecutionAdmissionsRepo } from '../../src/lib/db/repos/execution-admissions-repo.mjs' import { RequestAttemptsRepo } from '../../src/lib/db/repos/request-attempts-repo.mjs' import { readBody } from '../../src/lib/http/respond.mjs' +import { clientCancelledResult } from '../../src/lib/core/errors.mjs' const kind = process.env.VM2API_HA_BILLING_CASE +const undispatched = ['not-started', 'not-started-with-id', 'cancelled-before-dispatch'].includes(kind) const router = await import('../../src/lib/transport/kernel-router.mjs') const identity = await import('../../src/lib/identity/vm-identity.mjs') let dispatch @@ -36,7 +39,14 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn const assistant = { id: 'synthetic-message', type: 'message', role: 'assistant', model: 'claude-sonnet-4-5-20250929', content: [{ type: 'text', text: 'synthetic' }], stop_reason: 'end_turn', usage: { input_tokens: 100, output_tokens: 10, cache_read_input_tokens: 20, cache_creation_input_tokens: 30 } } let rateReads = 0, receiptReads = 0 dispatch = async (opts) => { - if (kind === 'not-started') return { ok: false, status: 400, body: { type: 'error', error: { type: 'invalid_request_error', code: 'invalid_cache_control', message: 'synthetic validation failure' } }, execution: { requestId: null, started: false, ended: true, reason: 'not_started' } } + if (undispatched) { + const execution = { requestId: kind === 'not-started' ? null : randomUUID(), started: false, ended: true, reason: 'not_started' } + if (kind === 'cancelled-before-dispatch') { + res.closed = true + return clientCancelledResult({ execution }) + } + return { ok: false, status: 400, body: { type: 'error', error: { type: 'invalid_request_error', code: 'invalid_cache_control', message: 'synthetic validation failure' } }, execution } + } await opts.onExecutionStart({ requestId: commandId, nodeBinding }) const job = (await f.peer.pool.query('SELECT * FROM execution_jobs WHERE request_id=$1', [commandId])).rows[0] assert.equal(job.billing_snapshot?.rate_multiplier, 2, 'dispatch permission requires the original PG-priced group snapshot') @@ -78,9 +88,11 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn req.headers = { 'user-agent': 'synthetic-ha-client' } const res = new EventEmitter(); res.setHeader = () => {} await f.originalGate.run(() => withHaRequestContext({ db: f.db, requestId: 'billing-parent', activity: f.originalGate, recoveryKind: 'claude_messages' }, () => handler.handleProtocol(req, res, 'anthropic.messages', '/v1/messages'))) - assert.equal(res.statusCode, kind === 'not-started' ? 400 : 200, JSON.stringify(res.body)) + assert.equal(res.statusCode, kind === 'cancelled-before-dispatch' ? undefined : undispatched ? 400 : 200, JSON.stringify(res.body)) + if (kind === 'cancelled-before-dispatch') assert.equal(res.body, undefined) + else if (undispatched) assert.equal(res.body.error.code, 'upstream_invalid_request') assert.equal(rateReads, 1, 'group rates must not be reloaded after dispatch for billing') - assert.equal(receiptReads, kind === 'not-started' ? 0 : 1) + assert.equal(receiptReads, undispatched ? 0 : 1) await f.originalGate.run(async () => { const rows = await f.db.prepare('SELECT * FROM ha_request_settlements').all() const admission = await f.db.prepare('SELECT * FROM api_key_admissions WHERE ha_request_id=$1').get('billing-parent') @@ -88,12 +100,19 @@ test(`HA protocol captures and settles original billable command: ${kind}`, asyn assert.equal(rows.length, 0) assert.equal(admission.released_at, null) assert.equal((await keys.getById(key.id)).requests, 0) - } else if (kind === 'not-started') { + } else if (undispatched) { assert.equal(rows.length, 1) assert.equal(rows[0].outcome, 'not_dispatched') assert.equal(rows[0].actual_cost, null) + for (const field of ['command_id', 'node_binding', 'node_receipt', 'official_cost', 'tokens_in', 'tokens_out', 'cache_read_tokens', 'cache_creation_tokens']) assert.equal(rows[0][field], null, field) assert.ok(admission.released_at) + assert.equal(admission.settled_at, null) assert.equal((await keys.getById(key.id)).requests, 0) + assert.equal((await keys.getById(key.id)).quota_used, 0) + assert.equal((await f.db.prepare('SELECT count(*) AS n FROM execution_jobs WHERE ha_request_id=$1').get('billing-parent')).n, 0) + const parent = await f.db.prepare('SELECT accepting_children,billable_command_id FROM ha_request_contexts WHERE id=$1').get('billing-parent') + assert.equal(parent.accepting_children, false) + assert.equal(parent.billable_command_id, null) } else { assert.equal(rows.length, 1) assert.equal(rows[0].outcome, kind === 'deleted-key' ? 'key_deleted' : 'charged') diff --git a/test/unit/ha-protocol-billing.test.mjs b/test/unit/ha-protocol-billing.test.mjs index 85c7ad976..d60593d95 100644 --- a/test/unit/ha-protocol-billing.test.mjs +++ b/test/unit/ha-protocol-billing.test.mjs @@ -4,7 +4,14 @@ import { spawnSync } from 'node:child_process' import { fileURLToPath } from 'node:url' const fixture = fileURLToPath(new URL('../fixtures/ha-protocol-billing.mjs', import.meta.url)) -for (const kind of ['complete', 'no-usage', 'deleted-key', 'not-started']) { +for (const kind of [ + 'complete', + 'no-usage', + 'deleted-key', + 'not-started', + 'not-started-with-id', + 'cancelled-before-dispatch', +]) { test(`HA protocol original command billing: ${kind}`, () => { const env = { ...process.env, VM2API_CONTROL_PLANE_MODE: 'standalone', VM2API_HA_BILLING_CASE: kind } delete env.NODE_TEST_CONTEXT diff --git a/test/unit/ha-request-billing.test.mjs b/test/unit/ha-request-billing.test.mjs index 355ee5ecf..b6cab559b 100644 --- a/test/unit/ha-request-billing.test.mjs +++ b/test/unit/ha-request-billing.test.mjs @@ -272,6 +272,82 @@ test('multiple commands without a durable billable choice stay held rather than }) }) +for (const commandId of [null, 'unbound-pre-dispatch-id']) { + test(`an existing job cannot become not_dispatched from an absent or unrelated final command: ${commandId}`, async (t) => { + const f = await haBillingFixture(t) + await billingRequest(f, async (current) => { + const settle = () => current.keys.recordHaUsage({ reservation: current.reservation, commandId }) + if (commandId) await assert.rejects(settle(), { code: 'ha_billable_command_mismatch' }) + else assert.equal((await settle()).settled, false) + assert.equal((await f.db.prepare('SELECT count(*) AS n FROM ha_request_settlements').get()).n, 0) + assert.equal( + ( + await f.db + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(current.reservation.reservationId) + ).released_at, + null, + ) + assert.equal((await current.keys.getById(current.key.id)).requests, 0) + assert.equal( + (await f.db.prepare('SELECT accepting_children FROM ha_request_contexts WHERE id=$1').get('billing-parent')) + .accepting_children, + true, + ) + }) + }) +} + +test('a no-job handler ignores an unbound command ID, audits no dispatch and closes children before returning', async (t) => { + const f = await haBillingFixture(t) + const { withHaRequestAttempt } = await import('../../src/lib/core/ha-request-context.mjs') + await billingRequest( + f, + async (current) => { + const result = await current.keys.recordHaUsage({ + reservation: current.reservation, + commandId: 'unbound-pre-dispatch-id', + }) + assert.equal(result.settled, true) + assert.equal(result.outcome, 'not_dispatched') + assert.equal(current.reservation.settled, true) + assert.equal(current.reservation.billingFailed, undefined) + assert.equal( + (await f.db.prepare('SELECT accepting_children FROM ha_request_contexts WHERE id=$1').get('billing-parent')) + .accepting_children, + false, + ) + await assert.rejects( + withHaRequestAttempt(1, () => + current.admission.bindExecution({ + requestId: 'late-command', + nodeBinding: current.nodeBinding, + billingSnapshot: syntheticBillingSnapshot(), + }), + ), + { code: 'ha_request_context_closed' }, + ) + await current.reservation.release() + const ledger = await f.db + .prepare('SELECT * FROM ha_request_settlements WHERE ha_request_id=$1') + .get('billing-parent') + assert.equal(ledger.command_id, null) + assert.equal(ledger.actual_cost, null) + assert.equal(ledger.tokens_in, null) + assert.equal((await f.db.prepare('SELECT count(*) AS n FROM execution_jobs').get()).n, 0) + assert.equal((await current.keys.getById(current.key.id)).requests, 0) + assert.ok( + ( + await f.db + .prepare('SELECT released_at FROM api_key_admissions WHERE id=$1') + .get(current.reservation.reservationId) + ).released_at, + ) + }, + { jobs: 0 }, + ) +}) + test('an explicit final command choice bills only that original command in a multiattempt request', async (t) => { const f = await haBillingFixture(t) const data = await billingRequest( From 7f3a4c7a020d91ae8567ca820d748acd13a5588e Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:00:46 +0800 Subject: [PATCH 453/473] fix: audit undispatched HA requests before selecting a billable command --- src/lib/core/ha-request-billing.mjs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/lib/core/ha-request-billing.mjs b/src/lib/core/ha-request-billing.mjs index 061f49da0..1c05554ab 100644 --- a/src/lib/core/ha-request-billing.mjs +++ b/src/lib/core/ha-request-billing.mjs @@ -312,12 +312,24 @@ export async function recordHaRequestUsage(db, { reservation, commandId, nodeSta if (reservation?.shared !== true) throw failure('invalid_api_key_reservation') const context = currentHaRequestContext(db) if (!context) throw failure('ha_request_context_required') - if (commandId != null) await selectHaBillableCommand(db, commandId) - const result = await new HaRequestBilling(db).settle(context.requestId, { - source: 'handler', - commandId, - nodeStatus, - reservation, + const result = await withTransaction(db, async () => { + await lockHaRequestContext(db) + // A client-side command ID can exist before dispatch permission. Only the persisted graph proves no dispatch. + const job = await db + .prepare(`SELECT j.request_id FROM execution_jobs j + LEFT JOIN execution_admissions a ON a.id=j.admission_id + WHERE j.ha_request_id=$1 OR a.ha_request_id=$1 LIMIT 1`) + .get(context.requestId) + if (job && commandId == null) return { settled: false, reason: 'ha_billable_command_unproven' } + const billableCommandId = job ? commandId : null + if (billableCommandId != null) await selectHaBillableCommand(db, billableCommandId) + // settle() closes children for no-job requests before this parent transaction commits. + return new HaRequestBilling(db).settle(context.requestId, { + source: 'handler', + commandId: billableCommandId, + nodeStatus, + reservation, + }) }) if (result.settled) reservation?._markSettled?.() else if (reservation) reservation.billingFailed = true From 50bc1f884a446b7173f14d1d1f8cfe9be306053a Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:00:50 +0800 Subject: [PATCH 454/473] test: reproduce unsent refresh grant generation conflict --- .../node-credential-refresh-wire.test.mjs | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/test/unit/node-credential-refresh-wire.test.mjs b/test/unit/node-credential-refresh-wire.test.mjs index 440aaa311..0dac51935 100644 --- a/test/unit/node-credential-refresh-wire.test.mjs +++ b/test/unit/node-credential-refresh-wire.test.mjs @@ -8,6 +8,7 @@ import os from 'node:os' import path from 'node:path' import http from 'node:http' import tls from 'node:tls' +import net from 'node:net' import pg from 'pg' import { NodeCredentialRefreshGrants } from '../../src/lib/cluster/node-credential-refresh-grants.mjs' import { SharedCredentialRefresh } from '../../src/lib/oauth/shared-credential-refresh.mjs' @@ -270,3 +271,151 @@ nodeCase('real PG reader and pinned TLS authorize node-owned refresh without Pod await new Promise((resolve) => kernel.close(resolve)) } }) + +async function launchedRefresh(fx) { + const published = await fx.run('node_publish_initial', (operation) => fx.service.publishInitial(fx.input(operation))) + return fx.run('node_launch', (operation) => + fx.service.launch(fx.input(operation, { commandId: 'refresh-launch', manifestDigest: published.manifest_digest })), + ) +} + +async function closedPortLink(link) { + const server = net.createServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const port = server.address().port + await new Promise((resolve) => server.close(resolve)) + return { ...link, address: '127.0.0.1', port } +} + +function attemptRefresh(fx, launched, link) { + const current = { generation: '1', credential: { account_uuid: 'synthetic-account' } } + return new SharedCredentialRefresh(fx.db).run( + { platform: 'claude', vmId, accountId: 'synthetic-account', expectedGeneration: '1' }, + { + readCurrent: async () => current, + refresh: async (_current, { operation }) => { + const receipt = await fx.refreshGrants.refresh( + { + operation, + nodeId, + vmId, + accountId: 'synthetic-account', + commandId: crypto.randomUUID(), + capturedFence: launched.fence, + expectedGeneration: '1', + force: false, + }, + link, + ) + return { finished: true, value: { ok: receipt.ok, generation: receipt.credential.generation } } + }, + publish: async () => current, + }, + ) +} + +nodeCase( + 'a TLS connection refusal retires only the unsent grant and permits a fresh command at the same generation', + async (fx) => { + const launched = await launchedRefresh(fx) + await assert.rejects(attemptRefresh(fx, launched, await closedPortLink(fx.node.link)), { + code: 'node_refresh_unconfirmed', + vmOperationNotStarted: true, + credentialRefreshNotStarted: true, + }) + const first = (await fx.peer.query(`SELECT * FROM ${fx.q}.refresh_grants`)).rows[0] + assert.ok(first.revoked_at) + const next = await attemptRefresh(fx, launched, fx.node.link) + assert.equal(next.finished, true) + assert.equal(next.unknown, false) + assert.equal(next.value.ok, true) + const rows = (await fx.peer.query(`SELECT receipt,revoked_at FROM ${fx.q}.refresh_grants ORDER BY created_at`)).rows + assert.equal(rows.length, 2) + assert.equal(rows[0].receipt.not_started, true) + assert.notEqual(rows[0].receipt.finished, true) + assert.equal(rows[1].receipt.credential.generation, '1') + assert.equal( + rows.every((row) => row.revoked_at), + true, + ) + }, +) + +nodeCase('failure to commit an unsent grant terminal state retains both exact claims', async (fx) => { + const launched = await launchedRefresh(fx) + await fx.db.query( + `CREATE FUNCTION reject_unsent_refresh() RETURNS trigger LANGUAGE plpgsql AS $body$ BEGIN RAISE EXCEPTION 'synthetic unsent commit failure' USING ERRCODE='P648N'; END $body$; CREATE TRIGGER reject_unsent_refresh BEFORE UPDATE OF receipt ON ${fx.q}.refresh_grants FOR EACH ROW EXECUTE FUNCTION reject_unsent_refresh()`, + ) + await assert.rejects(attemptRefresh(fx, launched, await closedPortLink(fx.node.link)), (error) => { + assert.equal(error.code, 'P648N') + assert.notEqual(error.vmOperationNotStarted, true) + assert.notEqual(error.credentialRefreshNotStarted, true) + assert.equal(error.vmOperationUncertain, true) + assert.equal(error.credentialRefreshUnknown, true) + return true + }) + const op = (await fx.peer.query("SELECT uncertain_at,released_at FROM vm_operations WHERE kind='credential_refresh'")) + .rows[0] + assert.ok(op.uncertain_at) + assert.equal(op.released_at, null) + const bg = ( + await fx.peer.query( + "SELECT unknown_at,finished_at FROM background_task_runs WHERE task_key='oauth-refresh:claude:synthetic-account'", + ) + ).rows[0] + assert.ok(bg.unknown_at) + assert.equal(bg.finished_at, null) + const grant = (await fx.peer.query(`SELECT receipt,revoked_at FROM ${fx.q}.refresh_grants`)).rows[0] + assert.equal(grant.receipt, null) + assert.ok(grant.revoked_at) +}) + +nodeCase( + 'TLS application data followed by a missing acknowledgement keeps generation occupied after revocation', + async (fx) => { + const launched = await launchedRefresh(fx) + let received = false + const sockets = new Set() + const server = tls.createServer( + { + cert: fs.readFileSync(path.join(fx.node.root, 'tls', 'cert.pem')), + key: fs.readFileSync(path.join(fx.node.root, 'tls', 'key.pem')), + minVersion: 'TLSv1.3', + }, + (socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + socket.once('data', () => { + received = true + socket.destroy() + }) + }, + ) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + try { + await assert.rejects(attemptRefresh(fx, launched, { ...fx.node.link, port: server.address().port }), (error) => { + assert.equal(error.code, 'node_refresh_unconfirmed') + assert.equal(error.vmOperationUncertain, true) + assert.notEqual(error.vmOperationNotStarted, true) + assert.notEqual(error.credentialRefreshNotStarted, true) + return true + }) + assert.equal(received, true) + const row = (await fx.peer.query(`SELECT receipt,revoked_at FROM ${fx.q}.refresh_grants`)).rows[0] + assert.equal(row.receipt, null) + assert.ok(row.revoked_at) + await assert.rejects( + fx.peer.query( + `INSERT INTO ${fx.q}.refresh_grants SELECT 'late-command',node_id,vm_id,allocation_id,operation_id,owner_id,account_id,bg_task_key,bg_execution_id,bg_owner_id,epoch,expected_generation,intent_digest,intent,receipt,created_at,revoked_at FROM ${fx.q}.refresh_grants`, + ), + { code: '23505' }, + ) + const next = await attemptRefresh(fx, launched, fx.node.link) + assert.equal(next.unknown, true) + assert.equal((await fx.peer.query(`SELECT count(*)::int AS n FROM ${fx.q}.refresh_grants`)).rows[0].n, 1) + } finally { + for (const socket of sockets) socket.destroy() + await new Promise((resolve) => server.close(resolve)) + } + }, +) From 5f549b9ba6951c1cb8be986a7e85c298933d66a2 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:02:20 +0800 Subject: [PATCH 455/473] fix: persist terminal state for unsent node refresh grants --- .../node-credential-refresh-grants.mjs | 58 ++++++++++++++++++- test/unit/node-credential-refresh.test.mjs | 51 ++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/src/lib/cluster/node-credential-refresh-grants.mjs b/src/lib/cluster/node-credential-refresh-grants.mjs index dfd0fc535..8c47f50ec 100644 --- a/src/lib/cluster/node-credential-refresh-grants.mjs +++ b/src/lib/cluster/node-credential-refresh-grants.mjs @@ -103,7 +103,7 @@ export class NodeCredentialRefreshGrants { }) const latest = ( await this.db.query( - `SELECT receipt FROM ${state.q}.refresh_grants WHERE node_id=$1 AND vm_id=$2 AND epoch=$3 AND receipt IS NOT NULL ORDER BY created_at DESC,command_id DESC LIMIT 1`, + `SELECT receipt FROM ${state.q}.refresh_grants WHERE node_id=$1 AND vm_id=$2 AND epoch=$3 AND receipt->'finished'='true'::jsonb ORDER BY created_at DESC,command_id DESC LIMIT 1`, [intent.node_id, intent.vm_id, intent.epoch], ) ).rows[0] @@ -156,7 +156,23 @@ export class NodeCredentialRefreshGrants { try { receipt = await refreshNodeSlotCredential({ link, grant }) } catch (error) { - if (error.vmOperationNotStarted === true) error.credentialRefreshNotStarted = true + if (error.vmOperationNotStarted === true && !error.vmOperationUncertain && !error.credentialRefreshUnknown) { + try { + // The transport proved no application bytes were sent. Retire this exact + // authorization before releasing its claims or allowing another command. + await this._recordNotStarted(grant) + } catch (commitError) { + delete commitError.vmOperationNotStarted + delete commitError.credentialRefreshNotStarted + commitError.vmOperationUncertain = true + commitError.credentialRefreshUnknown = true + throw commitError + } + error.credentialRefreshNotStarted = true + } else { + delete error.vmOperationNotStarted + delete error.credentialRefreshNotStarted + } throw error } try { @@ -167,6 +183,44 @@ export class NodeCredentialRefreshGrants { throw error } } + async _recordNotStarted(grantInput) { + const grant = normalizeRefreshGrant(grantInput), + i = grant.intent, + scope = currentCredentialRefreshAttempt(this.db) + return withTransaction(this.db, async () => { + const { state, epoch } = await this._state({ + operation: scope?.operation, + nodeId: i.node_id, + vmId: i.vm_id, + accountId: i.account_id, + capturedFence: i.fence, + }) + if (epoch !== i.epoch) throw refreshFailure('node_refresh_epoch_stale') + const row = ( + await this.db.query(`SELECT * FROM ${state.q}.refresh_grants WHERE command_id=$1 FOR UPDATE`, [i.command_id]) + ).rows[0] + if ( + !row || + row.revoked_at || + row.receipt || + row.intent_digest !== grant.intentDigest || + !isDeepStrictEqual(normalizeRefreshIntent(row.intent), i) + ) + throw refreshFailure('node_refresh_conflict') + // This is an owner-side transport terminal state, not a node refresh + // receipt. It supplies no credential generation or execution completion. + const terminal = { + not_started: true, + reason: 'application_data_not_sent', + command_id: i.command_id, + intent_digest: grant.intentDigest, + } + await this.db.query( + `UPDATE ${state.q}.refresh_grants SET receipt=$2::jsonb,revoked_at=clock_timestamp() WHERE command_id=$1`, + [i.command_id, JSON.stringify(terminal)], + ) + }) + } async recordReceipt(grantInput, value) { const grant = normalizeRefreshGrant(grantInput), receipt = normalizeRefreshReceipt(grant, value), diff --git a/test/unit/node-credential-refresh.test.mjs b/test/unit/node-credential-refresh.test.mjs index cbea21319..790f89724 100644 --- a/test/unit/node-credential-refresh.test.mjs +++ b/test/unit/node-credential-refresh.test.mjs @@ -2,6 +2,7 @@ import test from 'node:test' import assert from 'node:assert/strict' import crypto from 'node:crypto' import pg from 'pg' +import net from 'node:net' import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' @@ -293,3 +294,53 @@ refreshCase('a failed PG receipt commit retains the exact background and VM clai assert.equal(g.receipt, null) assert.ok(g.revoked_at) }) + +refreshCase('an unsent terminal grant cannot authorize or replace the last confirmed generation', async (f) => { + await f.grants.initialize() + await f.attempt(async (_current, { operation }) => { + const grant = await f.grants.issue(f.input(operation)) + await f.grants.recordReceipt(grant, f.receipt(grant, '2')) + return { finished: true, value: { ok: true, generation: '2' } } + }) + const reader = await f.reader() + const server = net.createServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const port = server.address().port + await new Promise((resolve) => server.close(resolve)) + let unsentGrant + await assert.rejects( + f.attempt(async (_current, { operation }) => { + const input = f.input(operation) + try { + await f.grants.refresh(input, { address: '127.0.0.1', port, token: 's'.repeat(32), certSha256: 'a'.repeat(64) }) + } catch (error) { + assert.equal(error.vmOperationNotStarted, true) + const row = ( + await f.peer.query( + `SELECT intent,intent_digest,receipt,revoked_at FROM ${f.q}.refresh_grants WHERE command_id=$1`, + [input.commandId], + ) + ).rows[0] + assert.equal(row.receipt.not_started, true) + assert.ok(row.revoked_at) + unsentGrant = { intent: row.intent, intentDigest: row.intent_digest } + assert.equal( + (await reader.query(`SELECT ${f.q}.refresh_current($1) AS active`, [input.commandId])).rows[0].active, + false, + ) + throw error + } + }), + { code: 'node_refresh_unconfirmed', credentialRefreshNotStarted: true }, + ) + const next = await f.attempt(async (_current, { operation }) => { + await assert.rejects(f.grants.recordReceipt(unsentGrant, f.receipt(unsentGrant, '3')), { + code: 'node_refresh_conflict', + }) + const grant = await f.grants.issue(f.input(operation)) + assert.equal(grant.intent.expected_credential_generation, '2') + await f.grants.recordReceipt(grant, f.receipt(grant, '3')) + return { finished: true, value: { ok: true, generation: '3' } } + }) + assert.equal(next.current.generation, '3') +}) From 4de403f9ee5411fee1c2266ef80cfd4315267a4b Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:03:53 +0800 Subject: [PATCH 456/473] fix: identify HA volume mount deletions by their merge keys --- deploy/k8s-ha/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/k8s-ha/deployment.yaml b/deploy/k8s-ha/deployment.yaml index 0376cfcc6..f0bca1833 100644 --- a/deploy/k8s-ha/deployment.yaml +++ b/deploy/k8s-ha/deployment.yaml @@ -35,8 +35,10 @@ spec: - name: run mountPath: /opt/vm2api/data/run - name: bin + mountPath: /opt/vm2api/bin $patch: delete - name: share + mountPath: /opt/vm2api/share $patch: delete - name: node-trust mountPath: /etc/vm2api From c6b8b1e839f5ecffa16c08a88048b9a2c0b5298f Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:06:46 +0800 Subject: [PATCH 457/473] test: reproduce killed HA scan activities blocking maintenance --- test/fixtures/orphan-recovery-scan.mjs | 14 ++ test/unit/ha-scan-activity-recovery.test.mjs | 216 +++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 test/fixtures/orphan-recovery-scan.mjs create mode 100644 test/unit/ha-scan-activity-recovery.test.mjs diff --git a/test/fixtures/orphan-recovery-scan.mjs b/test/fixtures/orphan-recovery-scan.mjs new file mode 100644 index 000000000..4542881c8 --- /dev/null +++ b/test/fixtures/orphan-recovery-scan.mjs @@ -0,0 +1,14 @@ +import { PostgresDatabase, registerActivityCoordinator } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' + +const url = process.env.VM2API_TEST_DATABASE_URL +if (!url || !/_test$/.test(new URL(url).pathname)) throw new Error('dedicated test database required') +const db = new PostgresDatabase({ connectionString: url, schema: process.env.KIN_DATABASE_SCHEMA }) +const coordinator = new GlobalMaintenanceCoordinator(db, { ownerId: 'synthetic-killed-scanner' }) +await coordinator.start() +registerActivityCoordinator(db, coordinator) +await coordinator.acknowledgeEpoch(1) +await coordinator.runActivity({ kind: 'ha_request_recovery' }, async (gate) => { + process.send({ ready: true, activityId: gate.activityId, ownerId: gate.ownerId }) + await new Promise(() => {}) +}) diff --git a/test/unit/ha-scan-activity-recovery.test.mjs b/test/unit/ha-scan-activity-recovery.test.mjs new file mode 100644 index 000000000..4483a15bb --- /dev/null +++ b/test/unit/ha-scan-activity-recovery.test.mjs @@ -0,0 +1,216 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import pg from 'pg' +import { getDb, quoteIdentifier, registerActivityCoordinator } from '../../src/lib/db/database.mjs' +import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { createRepositoryTestDatabase } from '../repository-database.mjs' +import { createHaRecoveryMonitor } from '../../src/lib/core/ha-recovery-monitor.mjs' + +async function fixture(t) { + const db = getDb() + const peer = await createRepositoryTestDatabase({ connectionString: db.connectionString, schema: db.schema }) + const coordinator = new GlobalMaintenanceCoordinator(db, { ownerId: 'scan-recovering-owner' }) + const original = new GlobalMaintenanceCoordinator(peer, { ownerId: 'original-scanner' }) + await coordinator.start() + await original.start() + registerActivityCoordinator(db, coordinator) + registerActivityCoordinator(peer, original) + await coordinator.acknowledgeEpoch(1) + await original.acknowledgeEpoch(1) + t.after(async () => { + await Promise.allSettled([coordinator.stop(), original.stop()]) + const cleanup = new pg.Client({ connectionString: db.connectionString }) + await cleanup.connect() + try { + await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`) + } finally { + await cleanup.end() + } + await peer.close() + }) + const monitor = createHaRecoveryMonitor({ + db, + coordinator, + readNodeRecoveryStatus: async () => assert.fail('no node command exists'), + }) + t.after(() => monitor.stop()) + return { db, peer, coordinator, original, monitor } +} + +async function killedScanner(t) { + const child = spawn( + process.execPath, + [fileURLToPath(new URL('../fixtures/orphan-recovery-scan.mjs', import.meta.url))], + { stdio: ['ignore', 'ignore', 'ignore', 'ipc'] }, + ) + t.after(() => { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + }) + const identity = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('synthetic scanner readiness timeout')), 5000) + child.once('error', (error) => { + clearTimeout(timer) + reject(error) + }) + child.once('exit', () => { + clearTimeout(timer) + reject(new Error('synthetic scanner exited before readiness')) + }) + child.once('message', (message) => { + clearTimeout(timer) + message.ready ? resolve(message) : reject(new Error('invalid readiness')) + }) + }) + return { + ...identity, + kill: async () => { + const exited = new Promise((resolve) => child.once('exit', resolve)) + child.kill('SIGKILL') + await exited + }, + } +} + +async function row(f, activityId) { + return (await f.db.pool.query(`SELECT * FROM ${f.coordinator.qualified}.activities WHERE id=$1`, [activityId])) + .rows[0] +} + +async function failedMaintenance(f) { + await assert.rejects(f.coordinator.beginMaintenance({ kind: 'backup', timeoutMs: 2000 }), { + code: 'maintenance_claims_active', + }) + await f.coordinator.cancelDrain((await f.coordinator.getState()).operationId) +} + +test('SIGKILL leaves an unrecoverable scanner claim until another real monitor completes only that read-only activity', async (t) => { + const f = await fixture(t) + const child = await killedScanner(t) + await f.monitor.runOnce() + assert.equal((await row(f, child.activityId)).completed_at, null) + await child.kill() + await failedMaintenance(f) + await f.monitor.runOnce() + const ended = await row(f, child.activityId) + assert.ok(ended.completed_at, 'the killed scanner must not permanently block maintenance') + assert.equal(ended.reason, 'ha_recovery_scan_owner_lost') + assert.equal(ended.uncertain_at, null) + await f.monitor.runOnce() + assert.deepEqual((await row(f, child.activityId)).completed_at, ended.completed_at) + const own = ( + await f.db.pool.query(`SELECT * FROM ${f.coordinator.qualified}.activities WHERE owner_id=$1`, [ + f.coordinator.ownerId, + ]) + ).rows + assert.equal(own.length, 3) + assert.ok(own.every((activity) => activity.completed_at && !activity.uncertain_at)) + const maintenance = await f.coordinator.beginMaintenance({ kind: 'backup', timeoutMs: 2000 }) + try { + await maintenance.resume() + } finally { + await maintenance.close() + } +}) + +test('scanner recovery never completes another unknown activity or releases a business claim of the same killed owner', async (t) => { + const f = await fixture(t) + const child = await killedScanner(t) + await f.db.pool.query( + `INSERT INTO ${f.coordinator.qualified}.activities(id,owner_id,epoch,kind,uncertain_at) VALUES('synthetic-business',$1,1,'api_endpoint',clock_timestamp())`, + [child.ownerId], + ) + await f.db.pool.query( + "INSERT INTO api_key_admissions(id,key_id,owner_id) VALUES('synthetic-held-key','synthetic-key',$1)", + [child.ownerId], + ) + await child.kill() + await f.monitor.runOnce() + assert.ok((await row(f, child.activityId)).completed_at) + const business = await row(f, 'synthetic-business') + assert.equal(business.completed_at, null) + assert.ok(business.uncertain_at) + assert.equal( + (await f.db.pool.query("SELECT released_at FROM api_key_admissions WHERE id='synthetic-held-key'")).rows[0] + .released_at, + null, + ) + await failedMaintenance(f) +}) + +for (const condition of ['legacy_owner', 'other_epoch', 'parent', 'handoff', 'business_kind']) { + test(`scanner recovery retains the exact activity when ${condition} prevents its proof`, async (t) => { + const f = await fixture(t) + const gate = await f.original.admit({ + kind: condition === 'business_kind' ? 'orphan_recovery' : 'ha_request_recovery', + }) + await gate.complete({ ended: false, reason: 'synthetic_owner_loss' }) + if (condition === 'legacy_owner') + await f.db.pool.query(`UPDATE ${f.coordinator.qualified}.members SET owner_lock_version=NULL WHERE owner_id=$1`, [ + gate.ownerId, + ]) + if (condition === 'other_epoch') + await f.db.pool.query(`UPDATE ${f.coordinator.qualified}.activities SET epoch=2 WHERE id=$1`, [gate.activityId]) + if (condition === 'parent') + await f.db.pool.query( + "INSERT INTO ha_request_contexts(id,activity_id,activity_owner_id,activity_epoch) VALUES('synthetic-parent',$1,$2,1)", + [gate.activityId, gate.ownerId], + ) + if (condition === 'handoff') + await f.db.pool.query( + `INSERT INTO ${f.coordinator.qualified}.operations(id,owner_id,kind,source_activity_id) VALUES('synthetic-handoff',$1,'backup',$2)`, + [gate.ownerId, gate.activityId], + ) + await f.original.stop() + await f.monitor.runOnce() + assert.equal((await row(f, gate.activityId)).completed_at, null) + assert.equal((await row(f, gate.activityId)).reason, 'synthetic_owner_loss') + }) +} + +test('one scanner page does not let twenty connected owners hide the next orphan page', async (t) => { + const f = await fixture(t) + for (let index = 0; index < 20; index++) { + const gate = await f.original.admit({ kind: 'ha_request_recovery' }) + await f.db.pool.query(`UPDATE ${f.coordinator.qualified}.activities SET id=$2 WHERE id=$1`, [ + gate.activityId, + `a-live-${String(index).padStart(2, '0')}`, + ]) + } + await f.db.pool.query( + `INSERT INTO ${f.coordinator.qualified}.members(owner_id,owner_lock_version,bound_epoch) VALUES('synthetic-dead-scanner',1,1)`, + ) + await f.db.pool.query( + `INSERT INTO ${f.coordinator.qualified}.activities(id,owner_id,epoch,kind) VALUES('z-orphan','synthetic-dead-scanner',1,'ha_request_recovery')`, + ) + await f.monitor.runOnce() + assert.equal((await row(f, 'z-orphan')).completed_at, null) + await f.monitor.runOnce() + assert.ok((await row(f, 'z-orphan')).completed_at) + const live = ( + await f.db.pool.query(`SELECT completed_at FROM ${f.coordinator.qualified}.activities WHERE id LIKE 'a-live-%'`) + ).rows + assert.equal(live.length, 20) + assert.ok(live.every((activity) => !activity.completed_at)) +}) + +test('concurrent scanner sweep transactions complete one exact row once and reject calls without an activity', async (t) => { + const f = await fixture(t) + const original = await f.original.admit({ kind: 'ha_request_recovery' }) + await original.complete({ ended: false, reason: 'synthetic_owner_loss' }) + await f.original.stop() + await assert.rejects(f.coordinator.recoverHaRecoveryScans(), { code: 'maintenance_activity_closed' }) + const recover = () => + f.coordinator.runActivity({ kind: 'ha_request_recovery' }, async () => ({ + ended: true, + value: await f.coordinator.recoverHaRecoveryScans(), + })) + const results = await Promise.all([recover(), recover()]) + assert.equal(results.flatMap((page) => page.results).filter((result) => result.recovered).length, 1) + const ended = await row(f, original.activityId) + assert.ok(ended.completed_at) + assert.ok(ended.uncertain_at) + assert.deepEqual((await recover()).results, []) + assert.deepEqual((await row(f, original.activityId)).completed_at, ended.completed_at) +}) From 9050cd6fc1edadc622fa43df9c13dc347919bf38 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:08:16 +0800 Subject: [PATCH 458/473] fix: recover exact read-only HA scans after proven owner loss --- src/lib/core/ha-recovery-monitor.mjs | 3 ++ src/lib/db/global-maintenance.mjs | 64 +++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/lib/core/ha-recovery-monitor.mjs b/src/lib/core/ha-recovery-monitor.mjs index 4baf7c311..b842a575d 100644 --- a/src/lib/core/ha-recovery-monitor.mjs +++ b/src/lib/core/ha-recovery-monitor.mjs @@ -11,6 +11,7 @@ function reportFailure(error) { export function createHaRecoveryMonitor({ db, coordinator, readNodeRecoveryStatus }) { const recovery = new HaRequestRecovery(db, { coordinator, readNodeRecoveryStatus }) let cursor = null + let scanCursor = null let inflight = null let timer = null let generation = 0 @@ -22,6 +23,8 @@ export function createHaRecoveryMonitor({ db, coordinator, readNodeRecoveryStatu inflight = coordinator .runActivity({ kind: 'ha_request_recovery' }, async () => { try { + const scans = await coordinator.recoverHaRecoveryScans({ limit: 20, afterActivityId: scanCursor }) + scanCursor = scans.nextCursor const value = await recovery.scan({ limit: 20, afterRequestId: cursor }) cursor = value.nextCursor for (const result of value.results) { diff --git a/src/lib/db/global-maintenance.mjs b/src/lib/db/global-maintenance.mjs index bc3006fba..eabffa230 100644 --- a/src/lib/db/global-maintenance.mjs +++ b/src/lib/db/global-maintenance.mjs @@ -2,7 +2,7 @@ import crypto from 'node:crypto' import { AsyncLocalStorage } from 'node:async_hooks' import pg from 'pg' -import { quoteIdentifier } from './database.mjs' +import { quoteIdentifier, withTransaction } from './database.mjs' const activities = new AsyncLocalStorage() const TRY_SHARED = 'SELECT pg_try_advisory_lock_shared(hashtext(current_database()), hashtext($1)) AS acquired' @@ -271,6 +271,68 @@ export class GlobalMaintenanceCoordinator { return lock.rows[0].acquired ? { absent: true } : { absent: false, reason: 'owner_connected' } } + /** + * This internal kind only reads historical node status. Its DB settlements are atomic transactions; + * it never dispatches node work or creates its own HA parent/handoff. After proven owner loss, + * an interrupted scan has no external mutation to await. Other activity kinds keep their claims. + */ + async recoverHaRecoveryScans({ limit = 20, afterActivityId = null } = {}) { + const scope = this.assertActivity() + if (scope.kind !== 'ha_request_recovery') + throw failure('maintenance_scan_scope_required', 'Scanner recovery requires its own scan activity') + if (this.db.isTransaction) + throw failure('maintenance_transaction_active', 'Scanner recovery requires separate short transactions') + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 100 || + (afterActivityId !== null && + (typeof afterActivityId !== 'string' || !afterActivityId || afterActivityId.length > 256)) + ) + throw failure('maintenance_scan_invalid', 'Invalid scanner recovery page') + const rows = await this.db + .prepare(`SELECT id,owner_id FROM ${this.qualified}.activities + WHERE kind='ha_request_recovery' AND epoch=$1 AND completed_at IS NULL AND id<>$2 + AND ($3::text IS NULL OR id>$3) ORDER BY id LIMIT $4`) + .all(scope.epoch, scope.id, afterActivityId, limit + 1) + const page = rows.slice(0, limit) + const results = [] + for (const candidate of page) { + const result = await withTransaction(this.db, async () => { + // Control/owner before activity matches the handoff and exact request-recovery lock order. + const absent = await this.lockAbsentOwner(candidate.owner_id) + if (!absent.absent) return { recovered: false, reason: absent.reason } + const row = await this.db + .prepare(`SELECT * FROM ${this.qualified}.activities WHERE id=$1 FOR UPDATE SKIP LOCKED`) + .get(candidate.id) + if (!row || row.completed_at) return { recovered: false, reason: 'scan_inactive' } + if ( + row.kind !== 'ha_request_recovery' || + row.owner_id !== candidate.owner_id || + String(row.epoch) !== String(scope.epoch) + ) + return { recovered: false, reason: 'scan_changed' } + if (await this.db.prepare('SELECT id FROM ha_request_contexts WHERE activity_id=$1 LIMIT 1').get(row.id)) + return { recovered: false, reason: 'scan_parent_present' } + if ( + await this.db + .prepare( + `SELECT id FROM ${this.qualified}.operations WHERE source_activity_id=$1 AND ended_at IS NULL LIMIT 1`, + ) + .get(row.id) + ) + return { recovered: false, reason: 'scan_handoff_pending' } + await this.db + .prepare(`UPDATE ${this.qualified}.activities SET completed_at=clock_timestamp(),reason='ha_recovery_scan_owner_lost' + WHERE id=$1 AND owner_id=$2 AND epoch=$3 AND kind='ha_request_recovery' AND completed_at IS NULL`) + .run(row.id, row.owner_id, scope.epoch) + return { recovered: true } + }) + results.push({ activityId: candidate.id, ...result }) + } + return { results, nextCursor: rows.length > limit ? page.at(-1).id : null } + } + async acknowledgeEpoch(epoch) { this._assertHealthy() return this._transaction(async (client) => { From b9f86dfa9c047c4f9553c70513c669dda527db8a Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:06:00 +0800 Subject: [PATCH 459/473] test: reproduce allocation pin rejection retaining bootstrap claims --- test/unit/ha-node-bootstrap.test.mjs | 95 ++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs index 099aea6fc..01603b488 100644 --- a/test/unit/ha-node-bootstrap.test.mjs +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -5,6 +5,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import http from 'node:http' +import tls from 'node:tls' import { fileURLToPath } from 'node:url' import pg from 'pg' import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' @@ -388,3 +389,97 @@ nodeCase( assert.equal(retry.status, 0, retry.out + retry.err) }, ) + +nodeCase( + 'an allocate pin mismatch records an unsent terminal grant and permits a later corrected invocation', + async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + const originalPin = fx.config.node.certSha256 + fx.config.node.certSha256 = originalPin === '0'.repeat(64) ? '1'.repeat(64) : '0'.repeat(64) + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const first = await fx.cli('slot-create') + assert.equal(first.status, 1) + assert.match(first.err, /node_allocation_unconfirmed/) + assert.equal(fx.calls.length, 0) + const original = (await fx.peer.query(`SELECT * FROM ${fx.q}.allocations`)).rows[0] + fx.config.node.certSha256 = originalPin + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const retry = await fx.cli('slot-create') + assert.equal(retry.status, 0, retry.out + retry.err) + const rows = (await fx.peer.query(`SELECT * FROM ${fx.q}.allocations ORDER BY created_at`)).rows + assert.equal(rows.length, 2) + assert.equal(rows[0].allocation_id, original.allocation_id) + assert.ok(rows[0].not_started_at) + assert.ok(rows[0].revoked_at) + assert.equal(rows[0].receipt.not_started, true) + assert.notEqual(rows[0].allocation_id, rows[1].allocation_id) + assert.equal(rows[1].not_started_at, null) + const firstOp = ( + await fx.peer.query('SELECT released_at,uncertain_at FROM vm_operations WHERE id=$1', [original.operation_id]) + ).rows[0] + assert.ok(firstOp.released_at) + assert.equal(firstOp.uncertain_at, null) + const activity = ( + await fx.peer.query( + `SELECT uncertain_at,completed_at FROM ${quoteIdentifier(fx.coordinator.schema)}.activities WHERE kind='node_bootstrap_command'`, + ) + ).rows + assert.equal( + activity.some((row) => row.uncertain_at !== null || row.completed_at === null), + false, + ) + }, +) + +nodeCase( + 'an allocate acknowledgement lost after application data keeps both target uniqueness and claims', + async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + let received = false + const sockets = new Set() + const server = tls.createServer( + { + cert: fs.readFileSync(path.join(fx.node.root, 'tls', 'cert.pem')), + key: fs.readFileSync(path.join(fx.node.root, 'tls', 'key.pem')), + minVersion: 'TLSv1.3', + }, + (socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + socket.once('data', () => { + received = true + socket.destroy() + }) + }, + ) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + try { + fx.config.node.port = server.address().port + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const first = await fx.cli('slot-create') + assert.equal(first.status, 1) + assert.equal(received, true) + const row = (await fx.peer.query(`SELECT * FROM ${fx.q}.allocations`)).rows[0] + assert.equal(row.receipt, null) + assert.ok(row.revoked_at) + fx.config.node.port = fx.node.link.port + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const retry = await fx.cli('slot-create') + assert.equal(retry.status, 1) + assert.match(retry.err, /bootstrap_claim_unavailable/) + assert.equal((await fx.peer.query(`SELECT count(*)::int AS n FROM ${fx.q}.allocations`)).rows[0].n, 1) + const op = ( + await fx.peer.query('SELECT released_at,uncertain_at FROM vm_operations WHERE id=$1', [row.operation_id]) + ).rows[0] + assert.ok(op.uncertain_at) + assert.equal(op.released_at, null) + } finally { + for (const socket of sockets) socket.destroy() + await new Promise((resolve) => server.close(resolve)) + } + }, +) From b1a6c9512607b762a0d8eb7290fa744a0a690a51 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:10:06 +0800 Subject: [PATCH 460/473] fix: retire proven unsent node allocation grants explicitly --- docs/deploy/ha-node-bootstrap.md | 6 +- scripts/ha-node-bootstrap.mjs | 10 ++- src/lib/cluster/node-management-grants.mjs | 76 +++++++++++++++- src/lib/db/node-management-authority.mjs | 87 +++++++++++++++++++ .../transport/slotgate-management-client.mjs | 20 +++-- test/unit/ha-node-bootstrap.test.mjs | 53 +++++++++++ test/unit/node-management-grants.test.mjs | 83 +++++++++++++++++- 7 files changed, 319 insertions(+), 16 deletions(-) diff --git a/docs/deploy/ha-node-bootstrap.md b/docs/deploy/ha-node-bootstrap.md index d5d06ea78..ec08328e9 100644 --- a/docs/deploy/ha-node-bootstrap.md +++ b/docs/deploy/ha-node-bootstrap.md @@ -32,7 +32,9 @@ node ID、VM ID、account UUID 使用原始值。TLS/policy 字段必须与应 node scripts/ha-node-bootstrap.mjs trusted-init --config /private/bootstrap.json ``` -该命令显式安装 allocation、command 和 refresh 三套 private authority/coordination marker。它不授予运行时自动升级权限，也不创建角色。stdout 仅输出节点 ID、私有 authority schema 和 SQL 文件位置。SQL 文件以 `0600`、禁止覆盖的方式创建，包含 reader 与节点的绑定以及最小 view/function GRANT，不含密码或 DSN。 +该命令显式安装 allocation、command 和 refresh 三套 private authority/coordination marker，并安装 `allocation_terminal_protocol_version=1`。后者保留未发送 allocation 的原 ID/终态历史，仅在明确未发送时释放 VM/physical target 的唯一占位。各项升级在可信初始化事务锁内执行；运行时不执行 DDL，也不创建角色。stdout 仅输出节点 ID、私有 authority schema 和 SQL 文件位置。SQL 文件以 `0600`、禁止覆盖的方式创建，包含 reader 与节点的绑定以及最小 view/function GRANT，不含密码或 DSN。 + +已有 authority 升级时，仍由操作者显式执行 `trusted-init`，并为 `reader.sqlOutput` 选用新的私有输出路径。已有 reader 绑定和函数权限保持；无需重复执行包含同一 reader 绑定的 SQL。升级不自动清除旧未知任务。尚未升级的 authority 在 allocate 明确未发送时返回 `node_allocation_terminal_unavailable` 并保留占位，须按恢复证据流程处理原任务。 DBA 检查并以应用 owner 身份执行该 SQL 文件。随后把返回的 authority schema、原 node ID 和该 reader 的连接串装入节点原有 `management` 配置；连接串只保存在受控节点文件。节点 reader 不能修改业务表或签发 grant。数据库连接角色拥有额外权限时，节点 authorizer 会拒绝。 @@ -46,6 +48,6 @@ node scripts/ha-node-bootstrap.mjs slot-create --config /private/bootstrap.json `slot-create` 持有一个持久 bootstrap BG claim，顺序执行独立的 allocate、publish-initial、launch VM operation。每一步都等待严格 receipt 后再进入下一步。RT 仅保存在节点未挂载的 canonical 路径；guest 只得到 AT 投影。launch receipt 确认后才原子新建公开 VM 元数据和指纹。已有 VM、并发插入或未知旧任务均拒绝，不覆盖记录。stdout 只输出公开 IDs、manifest 摘要与原 captured Fence；它证明节点 launch/元数据步骤，不证明真实模型推理成功。 -发生超时、断连、节点启动未知或元数据提交失败时，相关持久 claim 保留。再次执行不会换 command ID 重发。首个节点回调前明确 busy 的拒绝会结束本次未提交任务，操作者可稍后显式再运行。不能删 claim、改 VM ID 或重新发 allocation 来猜测恢复；须先按节点终止/恢复证据流程人工核实。CLI 不提供未知任务自动恢复，也不升级旧的 RT 已暴露 manifest。 +发生发送后的超时、断连、节点启动未知或元数据提交失败时，相关持久 claim 和目标占位保留。再次执行不会换 command ID 重发。首个节点回调前明确 busy 的拒绝会结束本次未提交任务。首个 allocate 在 TLS 应用数据发送前失败时，只有原 grant 的未开始终态及撤销已持久化后才结束本次任务；操作者修正 pin/连接配置后可显式再运行。终态写入失败仍保留未知占位。不能删 claim、改 VM ID 或重新发 allocation 来猜测恢复；须先按节点终止/恢复证据流程人工核实。CLI 不提供未知任务自动恢复，也不升级旧的 RT 已暴露 manifest。 测试只覆盖合成凭据、任务 PostgreSQL、TLS 和 fake Engine；没有真实 provider 或现有节点操作。闭源 kernel 在投影更新后是否重新加载 AT，仍需独立业务验证。旧 HA credential monitor 不能因此直接启用。 diff --git a/scripts/ha-node-bootstrap.mjs b/scripts/ha-node-bootstrap.mjs index bba94213d..fa9ba84b0 100644 --- a/scripts/ha-node-bootstrap.mjs +++ b/scripts/ha-node-bootstrap.mjs @@ -195,6 +195,7 @@ async function trustedInit(db, c, output) { const grants = new NodeManagementGrants(db), refresh = new NodeCredentialRefreshGrants(db) await grants.initialize() + await grants.initializeAllocationTerminals() await grants.initializeCommands() await refresh.initialize() const sql = @@ -227,7 +228,8 @@ async function slotCreate(db, c, input) { const tasks = new SharedBackgroundTasks(db), operations = new SharedVmOperations(db), grants = new NodeManagementGrants(db, { resolveNodeLink: async () => c.node }) - let nodeCall = false + let nodeCall = false, + allocationConfirmed = false let result try { result = await tasks.run( @@ -242,6 +244,7 @@ async function slotCreate(db, c, input) { nodeCall = true return grants.allocate({ operation, nodeId: c.node.id, vmId: c.vm.id, allocationId }) }) + allocationConfirmed = true const common = { nodeId: c.node.id, vmId: c.vm.id, @@ -304,7 +307,10 @@ async function slotCreate(db, c, input) { sharedResult: { ok: true, vmId: c.vm.id, launchCommandId: launchId }, } } catch (error) { - if (!nodeCall && ['vm_operation_active', 'vm_execution_active'].includes(error.code)) { + if ( + (!nodeCall && ['vm_operation_active', 'vm_execution_active'].includes(error.code)) || + (!allocationConfirmed && error.vmOperationNotStarted === true && !error.vmOperationUncertain) + ) { error.taskNotStarted = true error.activityEnded = true } diff --git a/src/lib/cluster/node-management-grants.mjs b/src/lib/cluster/node-management-grants.mjs index 9a2223997..cb72b734a 100644 --- a/src/lib/cluster/node-management-grants.mjs +++ b/src/lib/cluster/node-management-grants.mjs @@ -28,6 +28,8 @@ import { nodeGrantResource, nodeCommandAuthorityState, initializeNodeCommandAuthority, + nodeAllocationTerminalState, + initializeNodeAllocationTerminals, } from '../db/node-management-authority.mjs' export { nodeAuthoritySchemaFor } @@ -75,6 +77,10 @@ export class NodeManagementGrants { return initializeNodeCommandAuthority(this.db) } + initializeAllocationTerminals() { + return initializeNodeAllocationTerminals(this.db) + } + async _commandEpoch() { const row = ( await this.db.query( @@ -361,6 +367,7 @@ export class NodeManagementGrants { identifier(row.owner_id) const state = await nodeAuthorityState(this.db) if (!state.enabled) throw nodeAuthorityError() + const terminals = await nodeAllocationTerminalState(this.db) await lockNodeGrant(this.db, this.schema, nodeId, vmId) const epoch = ( await this.db.query( @@ -374,7 +381,7 @@ export class NodeManagementGrants { .digest('hex') const existing = ( await this.db.query( - `SELECT * FROM ${state.q}.allocations WHERE allocation_id=$1 OR vm_id=$2 OR physical_target=$3`, + `SELECT * FROM ${state.q}.allocations WHERE allocation_id=$1 OR ((vm_id=$2 OR physical_target=$3)${terminals.enabled ? ' AND not_started_at IS NULL' : ''})`, [allocationId, vmId, physicalContainerName(vmId)], ) ).rows[0] @@ -409,9 +416,23 @@ export class NodeManagementGrants { async allocate(input) { const link = await this.resolveNodeLink(input.nodeId) - if (!link) throw nodeAuthorityError('node_link_unavailable') + if (!link) throw Object.assign(nodeAuthorityError('node_link_unavailable'), { vmOperationNotStarted: true }) const grant = await this.issueAllocation(input) - const receipt = await allocateNodeSlot({ link, grant }) + let receipt + try { + receipt = await allocateNodeSlot({ link, grant }) + } catch (error) { + if (error.vmOperationNotStarted === true && !error.vmOperationUncertain) { + try { + await this._recordAllocationNotStarted(grant) + } catch (commitError) { + delete commitError.vmOperationNotStarted + commitError.vmOperationUncertain = true + throw commitError + } + } else delete error.vmOperationNotStarted + throw error + } await withTransaction(this.db, async () => { if (!currentVmOperationIds(this.db).has(grant.operationId)) throw nodeAuthorityError('node_operation_scope_required') @@ -449,6 +470,55 @@ export class NodeManagementGrants { return receipt } + async _recordAllocationNotStarted(grant) { + return withTransaction(this.db, async () => { + if (!currentVmOperationIds(this.db).has(grant.operationId)) + throw nodeAuthorityError('node_operation_scope_required') + const operation = await this.db + .prepare('SELECT * FROM vm_operations WHERE id=$1 AND owner_id=$2 FOR UPDATE') + .get(grant.operationId, grant.ownerId) + if ( + !operation || + operation.vm_id !== grant.vmId || + operation.kind !== 'node_allocate' || + operation.released_at || + operation.uncertain_at + ) + throw nodeAuthorityError('node_operation_invalid') + const state = await nodeAllocationTerminalState(this.db) + if (!state.enabled) throw nodeAuthorityError('node_allocation_terminal_unavailable') + await lockNodeGrant(this.db, this.schema, grant.nodeId, grant.vmId) + const row = ( + await this.db.query(`SELECT * FROM ${state.q}.allocations WHERE allocation_id=$1 FOR UPDATE`, [ + grant.allocationId, + ]) + ).rows[0] + if ( + !row || + row.revoked_at || + row.not_started_at || + row.receipt || + row.node_id !== grant.nodeId || + row.vm_id !== grant.vmId || + row.operation_id !== grant.operationId || + row.owner_id !== grant.ownerId || + row.intent_digest !== grant.intentDigest || + String(row.epoch) !== grant.epoch + ) + throw nodeAuthorityError('node_allocation_conflict') + const terminal = { + not_started: true, + reason: 'application_data_not_sent', + allocation_id: grant.allocationId, + intent_digest: grant.intentDigest, + } + await this.db.query( + `UPDATE ${state.q}.allocations SET receipt=$2::jsonb,not_started_at=clock_timestamp(),revoked_at=clock_timestamp() WHERE allocation_id=$1`, + [grant.allocationId, JSON.stringify(terminal)], + ) + }) + } + // Explicit node-role installation instructions only. Runtime initialization // never creates/alters roles and this SQL contains no credentials. readerRoleSql({ role, nodeId }) { diff --git a/src/lib/db/node-management-authority.mjs b/src/lib/db/node-management-authority.mjs index 5a85d0dea..74ad326a8 100644 --- a/src/lib/db/node-management-authority.mjs +++ b/src/lib/db/node-management-authority.mjs @@ -146,6 +146,93 @@ export async function initializeNodeAuthority(db) { }) } +/** Allocation retirement is an explicitly installed feature; legacy grants stay held. */ +export async function nodeAllocationTerminalState(db) { + const base = await nodeAuthorityState(db) + if (!base.enabled) return base + const coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + const unavailable = () => nodeAuthorityError('node_allocation_terminal_unavailable') + try { + const columns = ( + await db.query( + `SELECT + EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass($1) AND attname='allocation_terminal_protocol_version' AND NOT attisdropped) AS local_version, + EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass($2) AND attname='allocation_terminal_protocol_version' AND NOT attisdropped) AS marker_version, + EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid=to_regclass($3) AND attname='not_started_at' AND NOT attisdropped) AS terminal_column`, + [`${base.q}.meta`, `${coord}.node_authority_features`, `${base.q}.allocations`], + ) + ).rows[0] + if (Object.values(columns).every((value) => value === false)) return { enabled: false, schema: base.schema } + if (!Object.values(columns).every((value) => value === true)) throw unavailable() + const version = ( + await db.query( + `SELECT m.allocation_terminal_protocol_version AS local_version,f.allocation_terminal_protocol_version AS marker_version + FROM ${base.q}.meta m JOIN ${coord}.node_authority_features f ON f.application_schema=m.application_schema + WHERE m.id=1 AND f.authority_schema=$1`, + [base.schema], + ) + ).rows[0] + const indexes = ( + await db.query( + `SELECT count(*)::int AS n FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid + WHERE i.indrelid=to_regclass($1) AND i.indisunique AND i.indisvalid AND i.indnatts=1 + AND pg_get_expr(i.indpred,i.indrelid)='(not_started_at IS NULL)' + AND ((c.relname='allocations_vm_id_key' AND pg_get_indexdef(i.indexrelid,1,true)='vm_id') + OR (c.relname='allocations_physical_target_key' AND pg_get_indexdef(i.indexrelid,1,true)='physical_target'))`, + [`${base.q}.allocations`], + ) + ).rows[0] + if (version?.local_version !== 1 || version.marker_version !== 1 || indexes.n !== 2) throw unavailable() + return base + } catch { + throw unavailable() + } +} + +/** Trusted initialization only. No runtime read/issuance path upgrades this schema. */ +export async function initializeNodeAllocationTerminals(db) { + return withTransaction(db, async () => { + await db.query('SELECT pg_advisory_xact_lock(hashtext(current_database()),hashtext($1))', [ + `vm2api:node-authority-bootstrap:${db.schema}`, + ]) + const base = await nodeAuthorityState(db) + if (!base.enabled) throw nodeAuthorityError('node_allocation_terminal_unavailable') + const existing = await nodeAllocationTerminalState(db) + if (existing.enabled) return existing + const { q, schema } = base, + coord = quoteIdentifier(coordinationSchemaFor(db.schema)) + const phase = (await db.query(`SELECT phase FROM ${coord}.control WHERE id=1 FOR SHARE`)).rows[0] + if (phase?.phase !== 'open') throw nodeAuthorityError('node_allocation_terminal_unavailable') + await db.query(`ALTER TABLE ${q}.meta ADD COLUMN allocation_terminal_protocol_version integer NOT NULL DEFAULT 1 CHECK(allocation_terminal_protocol_version=1); + ALTER TABLE ${coord}.node_authority_features ADD COLUMN allocation_terminal_protocol_version integer NOT NULL DEFAULT 1 CHECK(allocation_terminal_protocol_version=1); + ALTER TABLE ${q}.allocations ADD COLUMN not_started_at timestamptz; + ALTER TABLE ${q}.allocations DROP CONSTRAINT allocations_vm_id_key, DROP CONSTRAINT allocations_physical_target_key; + CREATE UNIQUE INDEX allocations_vm_id_key ON ${q}.allocations(vm_id) WHERE not_started_at IS NULL; + CREATE UNIQUE INDEX allocations_physical_target_key ON ${q}.allocations(physical_target) WHERE not_started_at IS NULL; + CREATE OR REPLACE FUNCTION ${q}.allocation_current(p_allocation text) RETURNS boolean + LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $body$ + DECLARE g record; app text; allowed boolean; + BEGIN + SELECT a.*,m.application_schema INTO g FROM ${q}.allocations a CROSS JOIN ${q}.meta m + JOIN ${q}.reader_roles r ON r.role_name=session_user + JOIN ${coord}.control c ON c.id=1 + JOIN ${coord}.node_authority_features f ON f.application_schema=m.application_schema + WHERE a.allocation_id=p_allocation AND a.node_id=r.node_id AND a.revoked_at IS NULL AND a.not_started_at IS NULL AND a.action='allocate' + AND m.id=1 AND m.protocol_version=1 AND f.protocol_version=1 AND f.authority_schema='${schema}' + AND m.allocation_terminal_protocol_version=1 AND f.allocation_terminal_protocol_version=1 + AND c.application_schema=m.application_schema AND c.phase='open' AND c.epoch=a.epoch; + IF NOT FOUND OR NOT EXISTS(SELECT 1 FROM ${q}.reader_identity()) THEN RETURN false; END IF; + app:=g.application_schema; + EXECUTE format('SELECT EXISTS(SELECT 1 FROM %I.vm_operations WHERE id=$1 AND owner_id=$2 AND vm_id=$3 + AND kind=''node_allocate'' AND released_at IS NULL AND uncertain_at IS NULL)',app) + INTO allowed USING g.operation_id,g.owner_id,g.vm_id; + RETURN allowed; + EXCEPTION WHEN undefined_table OR invalid_schema_name THEN RETURN false; + END $body$`) + return nodeAllocationTerminalState(db) + }) +} + /** New command authority is a separately and explicitly installed feature. */ export async function nodeCommandAuthorityState(db) { const base = await nodeAuthorityState(db) diff --git a/src/lib/transport/slotgate-management-client.mjs b/src/lib/transport/slotgate-management-client.mjs index f090f3a75..f1256e10f 100644 --- a/src/lib/transport/slotgate-management-client.mjs +++ b/src/lib/transport/slotgate-management-client.mjs @@ -5,9 +5,10 @@ import { isDeepStrictEqual } from 'node:util' import crypto from 'node:crypto' import { normalizeNodeManagementGrant, normalizeNodeManagementReceipt } from '../core/node-management-command.mjs' -function denied() { +function denied(sent) { return Object.assign(new Error('node allocation acknowledgement unavailable'), { code: 'node_allocation_unconfirmed', + ...(sent === false ? { vmOperationNotStarted: true } : sent === true ? { vmOperationUncertain: true } : {}), }) } function exact(value, keys) { @@ -61,7 +62,7 @@ export function allocateNodeSlot({ link, grant }) { !/^[\x21-\x7e]{32,256}$/.test(link.token) || !/^[a-f0-9]{64}$/.test(link.certSha256) ) - return Promise.reject(denied()) + return Promise.reject(denied(false)) const request = { op: 'allocate', vm_id: grant.vmId, @@ -71,6 +72,7 @@ export function allocateNodeSlot({ link, grant }) { } return new Promise((resolve, reject) => { let settled = false + let sent = false let buffer = Buffer.alloc(0) const socket = tls.connect({ host: link.address, @@ -86,19 +88,21 @@ export function allocateNodeSlot({ link, grant }) { if (error) reject(error) else resolve(value) } - const timer = setTimeout(() => finish(denied()), 8000) - socket.once('error', () => finish(denied())) - socket.once('close', () => finish(denied())) + const failed = () => finish(denied(sent)) + const timer = setTimeout(failed, 8000) + socket.once('error', failed) + socket.once('close', failed) socket.once('secureConnect', () => { const pin = socket.getPeerCertificate()?.fingerprint256?.replaceAll(':', '').toLowerCase() - if (pin !== link.certSha256) return finish(denied()) + if (pin !== link.certSha256) return failed() + sent = true socket.write( `KSG2 ${link.token}\n${JSON.stringify(request, (key, value) => (key === 'generation' ? JSON.rawJSON(String(value)) : value))}\n`, ) }) socket.on('data', (chunk) => { buffer = Buffer.concat([buffer, chunk]) - if (buffer.length > 8192) return finish(denied()) + if (buffer.length > 8192) return failed() const newline = buffer.indexOf(10) if (newline < 0) return try { @@ -108,7 +112,7 @@ export function allocateNodeSlot({ link, grant }) { if (reply.ok !== true) throw denied() finish(null, normalizeAllocationReceipt(reply.allocation, grant)) } catch { - finish(denied()) + failed() } }) }) diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs index 01603b488..a7b64abcc 100644 --- a/test/unit/ha-node-bootstrap.test.mjs +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -12,6 +12,7 @@ import { NodeManagementGrants } from '../../src/lib/cluster/node-management-gran import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' +import { allocateNodeSlot } from '../../src/lib/transport/slotgate-management-client.mjs' const nodeId = 'command-node' const vmId = 'command-vm' @@ -404,6 +405,22 @@ nodeCase( assert.match(first.err, /node_allocation_unconfirmed/) assert.equal(fx.calls.length, 0) const original = (await fx.peer.query(`SELECT * FROM ${fx.q}.allocations`)).rows[0] + await assert.rejects( + allocateNodeSlot({ + link: fx.node.link, + grant: { + action: 'allocate', + nodeId: original.node_id, + vmId: original.vm_id, + allocationId: original.allocation_id, + operationId: original.operation_id, + ownerId: original.owner_id, + epoch: String(original.epoch), + intentDigest: original.intent_digest, + }, + }), + { code: 'node_allocation_unconfirmed' }, + ) fx.config.node.certSha256 = originalPin fs.writeFileSync(fx.file, JSON.stringify(fx.config)) const retry = await fx.cli('slot-create') @@ -466,6 +483,13 @@ nodeCase( const row = (await fx.peer.query(`SELECT * FROM ${fx.q}.allocations`)).rows[0] assert.equal(row.receipt, null) assert.ok(row.revoked_at) + assert.equal(row.not_started_at, null) + await assert.rejects( + fx.peer.query( + `INSERT INTO ${fx.q}.allocations(allocation_id,node_id,vm_id,physical_target,operation_id,owner_id,action,epoch,intent_digest) SELECT 'different-allocation',node_id,vm_id,physical_target,operation_id,owner_id,action,epoch,intent_digest FROM ${fx.q}.allocations`, + ), + { code: '23505' }, + ) fx.config.node.port = fx.node.link.port fs.writeFileSync(fx.file, JSON.stringify(fx.config)) const retry = await fx.cli('slot-create') @@ -483,3 +507,32 @@ nodeCase( } }, ) + +nodeCase('an unsent allocation terminal commit failure retains its grant and both claims', async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + await fx.db.query( + `CREATE FUNCTION reject_allocation_terminal() RETURNS trigger LANGUAGE plpgsql AS $body$ BEGIN RAISE EXCEPTION 'synthetic allocation terminal failure' USING ERRCODE='P648A'; END $body$; CREATE TRIGGER reject_allocation_terminal BEFORE UPDATE OF receipt ON ${fx.q}.allocations FOR EACH ROW EXECUTE FUNCTION reject_allocation_terminal()`, + ) + fx.config.node.certSha256 = fx.config.node.certSha256 === '0'.repeat(64) ? '1'.repeat(64) : '0'.repeat(64) + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const first = await fx.cli('slot-create') + assert.equal(first.status, 1) + assert.match(first.err, /bootstrap_failed/) + const row = (await fx.peer.query(`SELECT * FROM ${fx.q}.allocations`)).rows[0] + assert.equal(row.receipt, null) + assert.equal(row.not_started_at, null) + assert.ok(row.revoked_at) + const op = (await fx.peer.query('SELECT released_at,uncertain_at FROM vm_operations WHERE id=$1', [row.operation_id])) + .rows[0] + assert.equal(op.released_at, null) + assert.ok(op.uncertain_at) + const bg = ( + await fx.peer.query('SELECT unknown_at,finished_at FROM background_task_runs WHERE task_key=$1', [ + `ha-node-bootstrap:${vmId}`, + ]) + ).rows[0] + assert.ok(bg.unknown_at) + assert.equal(bg.finished_at, null) +}) diff --git a/test/unit/node-management-grants.test.mjs b/test/unit/node-management-grants.test.mjs index ee46be1ed..d695cba5b 100644 --- a/test/unit/node-management-grants.test.mjs +++ b/test/unit/node-management-grants.test.mjs @@ -1,11 +1,16 @@ import test from 'node:test' import assert from 'node:assert/strict' import pg from 'pg' +import net from 'node:net' import crypto from 'node:crypto' import fs from 'node:fs' import path from 'node:path' import { setTimeout as delay } from 'node:timers/promises' -import { nodeGrantNamespace, nodeGrantResource } from '../../src/lib/db/node-management-authority.mjs' +import { + nodeAllocationTerminalState, + nodeGrantNamespace, + nodeGrantResource, +} from '../../src/lib/db/node-management-authority.mjs' import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { SharedVmOperations } from '../../src/lib/vm/shared-vm-operations.mjs' @@ -361,3 +366,79 @@ test('a PG receipt write failure retains the unknown operation after a local all } }) }) + +test('allocation terminal upgrade is explicit, atomic across connections, and preserves unknown target reservations', async (t) => { + await usingFixture(t, async (fx) => { + const q = quoteIdentifier(fx.service.schema) + assert.equal((await nodeAllocationTerminalState(fx.db)).enabled, false) + const held = await fx.issue() + const peerService = new NodeManagementGrants(fx.peer) + const upgraded = await Promise.all([ + fx.service.initializeAllocationTerminals(), + peerService.initializeAllocationTerminals(), + ]) + assert.equal( + upgraded.every((value) => value.enabled), + true, + ) + assert.equal( + (await fx.peer.query(`SELECT allocation_terminal_protocol_version FROM ${q}.meta`)).rows[0] + .allocation_terminal_protocol_version, + 1, + ) + const row = (await fx.peer.query(`SELECT * FROM ${q}.allocations`)).rows[0] + assert.equal(row.allocation_id, held.grant.allocationId) + assert.equal(row.not_started_at, null) + await assert.rejects( + fx.peer.query( + `INSERT INTO ${q}.allocations(allocation_id,node_id,vm_id,physical_target,operation_id,owner_id,action,epoch,intent_digest) SELECT 'conflicting',node_id,vm_id,physical_target,operation_id,owner_id,action,epoch,intent_digest FROM ${q}.allocations`, + ), + { code: '23505' }, + ) + await fx.db.query( + `ALTER TABLE ${quoteIdentifier(fx.maintenance.schema)}.node_authority_features DROP COLUMN allocation_terminal_protocol_version`, + ) + await assert.rejects(nodeAllocationTerminalState(fx.db), { code: 'node_allocation_terminal_unavailable' }) + await assert.rejects(fx.service.initializeAllocationTerminals(), { code: 'node_allocation_terminal_unavailable' }) + assert.equal((await fx.peer.query(`SELECT count(*)::int AS n FROM ${q}.allocations`)).rows[0].n, 1) + }) +}) + +test('an unupgraded authority keeps a proven unsent allocation held and reports the required trusted upgrade', async (t) => { + await usingFixture(t, async (fx) => { + const server = net.createServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const port = server.address().port + await new Promise((resolve) => server.close(resolve)) + fx.service.resolveNodeLink = async () => ({ + address: '127.0.0.1', + port, + token: 's'.repeat(32), + certSha256: 'a'.repeat(64), + }) + await assert.rejects( + new SharedVmOperations(fx.db).run( + 'synthetic-vm', + (operation) => + fx.service.allocate({ operation, nodeId: 'synthetic-node', vmId: 'synthetic-vm', allocationId: 'not-sent' }), + { kind: 'node_allocate' }, + ), + (error) => { + assert.equal(error.code, 'node_allocation_terminal_unavailable') + assert.notEqual(error.vmOperationNotStarted, true) + assert.equal(error.vmOperationUncertain, true) + return true + }, + ) + const row = ( + await fx.peer.query(`SELECT receipt,revoked_at FROM ${quoteIdentifier(fx.service.schema)}.allocations`) + ).rows[0] + assert.equal(row.receipt, null) + assert.ok(row.revoked_at) + const op = (await fx.peer.query("SELECT released_at,uncertain_at FROM vm_operations WHERE kind='node_allocate'")) + .rows[0] + assert.equal(op.released_at, null) + assert.ok(op.uncertain_at) + assert.equal((await nodeAllocationTerminalState(fx.db)).enabled, false) + }) +}) From 0d9c434866532cfccd952ac990881400ed364dd1 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:18:28 +0800 Subject: [PATCH 461/473] test: initialize PostgreSQL for shared HA database fixtures --- scripts/run-isolated-tests.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-isolated-tests.mjs b/scripts/run-isolated-tests.mjs index 8ac6c6427..fe800b9ab 100644 --- a/scripts/run-isolated-tests.mjs +++ b/scripts/run-isolated-tests.mjs @@ -29,7 +29,7 @@ for (const file of files) { const offline = /(?:sqlite-legacy|db-migration-sub2api)\.test\.mjs$/.test(file) const database = !offline && - /db\/|database\.mjs|api-keys\.mjs|panel-users\.mjs|account-quota\.mjs|sticky-router\.mjs|proxy-pool\.mjs|request-log\.mjs|api-endpoints\.mjs|thinking-owner\.mjs|harness\.mjs/.test( + /db\/|database\.mjs|api-keys\.mjs|panel-users\.mjs|account-quota\.mjs|sticky-router\.mjs|proxy-pool\.mjs|request-log\.mjs|api-endpoints\.mjs|thinking-owner\.mjs|harness\.mjs|ha-billing\.mjs|node-management-fixture\.mjs/.test( source, ) const args = database ? ['--import', './test/pg-setup.mjs', '--test', file] : ['--test', file] From 73f2026b5a31bab8425c6cb3e8648166ac53346d Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:15:04 +0800 Subject: [PATCH 462/473] test: require zero-exit HA SIGTERM under continued SSE keepalive traffic --- test/e2e/ha-shutdown.e2e.test.mjs | 222 ++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 test/e2e/ha-shutdown.e2e.test.mjs diff --git a/test/e2e/ha-shutdown.e2e.test.mjs b/test/e2e/ha-shutdown.e2e.test.mjs new file mode 100644 index 000000000..bb25650fc --- /dev/null +++ b/test/e2e/ha-shutdown.e2e.test.mjs @@ -0,0 +1,222 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import http from 'node:http' +import fs from 'node:fs' +import path from 'node:path' +import { once } from 'node:events' +import { startGateway, api } from '../harness.mjs' +import { nodeManagementFixture } from '../fixtures/node-management-fixture.mjs' +import { syntheticClaudeNode } from '../fixtures/synthetic-claude-node.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +const model = 'claude-haiku-4-5-20251001' + +async function fixture(t) { + const fx = await nodeManagementFixture(t) + let kernel + let gateway + const close = async () => { + if (gateway) { + if (gateway.child.exitCode == null && gateway.child.signalCode == null) { + const exited = once(gateway.child, 'exit') + gateway.child.kill('SIGKILL') + await exited + } + await gateway.db.close() + } + await kernel?.close() + await fx.close() + } + try { + const credential = { ...JSON.parse(fx.payload).credential, expires_at: Date.now() + 3600000 } + const payload = Buffer.from(JSON.stringify({ credential })) + const published = await fx.run('node_publish_initial', (operation) => + fx.service.publishInitial(fx.input(operation, { payload })), + ) + const launched = await fx.run('node_launch', (operation) => + fx.service.launch( + fx.input(operation, { payload, commandId: 'shutdown-launch', manifestDigest: published.manifest_digest }), + ), + ) + const socket = path.join( + fx.node.root, + 'runtime', + 'vms', + fx.vmId, + 'incarnations', + launched.fence.incarnation, + 'run', + 'kernel.sock', + ) + kernel = await syntheticClaudeNode(socket) + await new VmsRepo(fx.db).upsertFromVmJson({ + id: fx.vmId, + name: 'synthetic-shutdown', + node_id: fx.nodeId, + status: 'running', + schedulable: true, + timezone: 'UTC', + locale: 'en_US.UTF-8', + fingerprint: { device_id: 'shutdown-device', session_id: 'shutdown-session' }, + policy: { maxConcurrency: 2, weight: 1 }, + claude: { + account_uuid: credential.account_uuid, + has_access: true, + has_refresh: true, + expires_at: credential.expires_at, + }, + }) + const trust = path.join(fx.root, 'nodes.json') + fs.writeFileSync( + trust, + JSON.stringify({ + nodes: { [fx.nodeId]: { ...fx.node.link, policyDigest: fx.policyDigest, presetId: 'claude-docker-v1' } }, + }), + { mode: 0o600 }, + ) + const project = path.join(fx.root, 'controller') + fs.mkdirSync(path.join(project, 'vms'), { recursive: true }) + fs.writeFileSync(path.join(project, 'vms', 'active.json'), '{}') + gateway = await startGateway({ + project, + env: { + VM2API_CONTROL_PLANE_MODE: 'kubernetes-ha', + KIN_DATABASE_SCHEMA: fx.db.schema, + VM2API_NODE_CONFIG_FILE: trust, + VM2API_SHUTDOWN_TIMEOUT_MS: '3000', + KIN_BACKUP_DISABLED: '1', + VM2API_TEST_HOST_BARRIER_FILE: path.join(fx.root, 'legacy-effects.log'), + NODE_OPTIONS: `--import=${new URL('../fixtures/ha-flow-no-host-loader.mjs', import.meta.url).href}`, + }, + }) + return { ...fx, gateway, kernel, close } + } catch (error) { + await close() + throw error + } +} + +function request(gateway, agent, endpoint, body) { + return new Promise((resolve, reject) => { + const req = http.request( + gateway.baseUrl + endpoint, + { + agent, + method: body ? 'POST' : 'GET', + headers: { authorization: `Bearer ${gateway.apiKey}`, ...(body ? { 'content-type': 'application/json' } : {}) }, + }, + (res) => { + let text = '' + res.on('data', (chunk) => { + text += chunk + }) + res.on('error', reject) + res.on('end', () => resolve({ status: res.statusCode, text, connection: res.headers.connection })) + }, + ) + req.on('error', reject) + req.setTimeout(7000, () => req.destroy(new Error('synthetic shutdown HTTP timeout'))) + req.end(body ? JSON.stringify(body) : undefined) + }) +} + +async function terminated(gateway, exit, diagnostic) { + let timer + try { + const result = await Promise.race([ + exit, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('normal SIGTERM did not exit within 5s: ' + diagnostic())), 5000) + }), + ]) + assert.deepEqual(result, [0, null], diagnostic()) + } finally { + clearTimeout(timer) + } +} + +test('an idle real HA process exits zero on SIGTERM without the harness SIGKILL fallback', { + timeout: 20000, +}, async (t) => { + const fx = await fixture(t) + try { + await delay(1200) // Include the real shared routing poll. + const exit = once(fx.gateway.child, 'exit') + fx.gateway.child.kill('SIGTERM') + await terminated(fx.gateway, exit, () => fx.gateway.stdout().slice(-1500) + fx.gateway.stderr().slice(-1500)) + } finally { + await fx.close() + } +}) + +test('an HA process finishes an accepted SSE then exits zero under continued keepalive requests', { + timeout: 25000, +}, async (t) => { + const fx = await fixture(t) + const gw = fx.gateway + const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }) + let traffic + let held + let pumping = true + const responses = [] + const diagnostic = () => + JSON.stringify({ + keepalive: responses.slice(-8), + stdout: gw.stdout().slice(-2000), + stderr: gw.stderr().slice(-1500), + }) + try { + const first = await api(gw, 'POST', '/v1/messages', { + body: { model, max_tokens: 16, messages: [{ role: 'user', content: 'completed before SIGTERM' }] }, + }) + assert.equal(first.status, 200, first.text) + await delay(1200) + await fetch(fx.kernel.controlUrl + '/pause', { method: 'POST' }) + held = request(gw, agent, '/v1/messages', { + model, + max_tokens: 16, + stream: true, + messages: [{ role: 'user', content: 'accepted before SIGTERM' }], + }) + held.catch(() => {}) + let active = 0 + for (let i = 0; i < 80; i++) { + active = (await (await fetch(fx.kernel.controlUrl + '/status')).json()).active + if (active === 1) break + await delay(20) + } + assert.equal(active, 1, 'the real node must own an active SSE request before SIGTERM') + // The first queued request uses exactly the active SSE connection once it + // finishes. Continuing its keepalive must not postpone process shutdown. + traffic = (async () => { + while (pumping && gw.child.exitCode == null && gw.child.signalCode == null) { + try { + const response = await request(gw, agent, '/v1/meta') + responses.push({ status: response.status, connection: response.connection }) + } catch {} + await delay(20) + } + })() + const exit = once(gw.child, 'exit') + gw.child.kill('SIGTERM') + for (let i = 0; i < 100 && !gw.stdout().includes('[shutdown] SIGTERM'); i++) await delay(10) + assert.match(gw.stdout(), /\[shutdown\] SIGTERM/) + await delay(150) + await fetch(fx.kernel.controlUrl + '/resume', { method: 'POST' }) + const streamed = await held + assert.equal(streamed.status, 200, streamed.text) + assert.match(streamed.text, /message_stop/) + await terminated(gw, exit, diagnostic) + assert.equal( + (await fx.peer.query('SELECT count(*)::int AS n FROM execution_jobs WHERE ended_at IS NULL')).rows[0].n, + 0, + ) + t.diagnostic(`normal SIGTERM exit=0; post-drain keepalive responses=${responses.length}`) + } finally { + pumping = false + agent.destroy() + await Promise.allSettled([traffic, held]) + await fx.close() + } +}) From f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:19:22 +0800 Subject: [PATCH 463/473] fix: close drained HTTP keepalive connections after accepted responses finish --- src/server.mjs | 16 ++ test/e2e/ha-shutdown.e2e.test.mjs | 250 +++++++++++++++++++++--------- 2 files changed, 194 insertions(+), 72 deletions(-) diff --git a/src/server.mjs b/src/server.mjs index b66cd5dee..4fec91333 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1311,8 +1311,14 @@ const handlePanel = createPanelHandler({ }) const sockets = new Set() +const httpResponses = new Set() const requestActivities = new WeakMap() +function closeHttpKeepAlive(res) { + res.shouldKeepAlive = false + if (!res.headersSent) res.setHeader('Connection', 'close') +} + function noteRequestExecution(req, execution) { const state = requestActivities.get(req) if (!state) return @@ -1588,6 +1594,15 @@ async function handleHttp(req, res) { } const server = http.createServer((req, res) => { + httpResponses.add(res) + res.once('close', () => httpResponses.delete(res)) + res.once('finish', () => { + httpResponses.delete(res) + // An accepted stream must finish before its connection becomes idle. + // A continued keepalive client cannot keep that idle connection draining. + if (lifecycle.draining) server.closeIdleConnections?.() + }) + if (lifecycle.draining) closeHttpKeepAlive(res) const url = parseRequestUrl(req) const probe = (req.method === 'GET' || req.method === 'HEAD') && ['/healthz', '/readyz'].includes(url?.pathname) const pending = probe ? handleHttp(req, res) : withRequestActivity(req, res, () => handleHttp(req, res)) @@ -1709,6 +1724,7 @@ async function shutdown(signal, { failed = false } = {}) { } catch {} } lifecycle.beginDrain() + for (const res of httpResponses) closeHttpKeepAlive(res) responsesWs.beginDrain() clusterRoutes.beginDrain() slotShell.beginDrain() diff --git a/test/e2e/ha-shutdown.e2e.test.mjs b/test/e2e/ha-shutdown.e2e.test.mjs index bb25650fc..446d8dce6 100644 --- a/test/e2e/ha-shutdown.e2e.test.mjs +++ b/test/e2e/ha-shutdown.e2e.test.mjs @@ -6,13 +6,89 @@ import path from 'node:path' import { once } from 'node:events' import { startGateway, api } from '../harness.mjs' import { nodeManagementFixture } from '../fixtures/node-management-fixture.mjs' -import { syntheticClaudeNode } from '../fixtures/synthetic-claude-node.mjs' import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) const model = 'claude-haiku-4-5-20251001' -async function fixture(t) { +async function holdableKernel(socket, afterHeaders) { + let paused = false + let active = 0 + let closing = false + const waits = new Set() + const resume = () => { + paused = false + for (const resolve of waits) resolve() + waits.clear() + } + const server = http.createServer(async (req, res) => { + if (req.url === '/internal/health') { + res.writeHead(200, { 'content-type': 'application/json' }) + return res.end(JSON.stringify({ ok: true, healthy: true, ready_slots: 8 })) + } + try { + for await (const _ of req) { + } + } catch { + return res.destroy() + } + const frames = [ + { + type: 'message_start', + message: { + id: 'msg_shutdown', + type: 'message', + role: 'assistant', + model, + content: [], + usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'synthetic accepted stream' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 7 } }, + { type: 'message_stop' }, + ].map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) + active++ + let emittedPrefix = false + try { + if (paused && afterHeaders) { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.write(frames.slice(0, 3).join('')) + res.flushHeaders() + emittedPrefix = true + } + if (paused) await new Promise((resolve) => waits.add(resolve)) + if (closing) return res.destroy() + if (!res.headersSent) res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.end(frames.slice(emittedPrefix ? 3 : 0).join('')) + } finally { + active-- + } + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(socket, resolve) + }) + return { + pause() { + paused = true + }, + resume, + status() { + return { active } + }, + async close() { + closing = true + resume() + server.closeAllConnections() + await new Promise((resolve) => server.close(resolve)) + }, + } +} + +async function fixture(t, afterHeaders = false) { const fx = await nodeManagementFixture(t) let kernel let gateway @@ -49,7 +125,7 @@ async function fixture(t) { 'run', 'kernel.sock', ) - kernel = await syntheticClaudeNode(socket) + kernel = await holdableKernel(socket, afterHeaders) await new VmsRepo(fx.db).upsertFromVmJson({ id: fx.vmId, name: 'synthetic-shutdown', @@ -97,7 +173,7 @@ async function fixture(t) { } } -function request(gateway, agent, endpoint, body) { +function request(gateway, agent, endpoint, body, onHeaders = () => {}) { return new Promise((resolve, reject) => { const req = http.request( gateway.baseUrl + endpoint, @@ -107,6 +183,7 @@ function request(gateway, agent, endpoint, body) { headers: { authorization: `Bearer ${gateway.apiKey}`, ...(body ? { 'content-type': 'application/json' } : {}) }, }, (res) => { + onHeaders() let text = '' res.on('data', (chunk) => { text += chunk @@ -150,73 +227,102 @@ test('an idle real HA process exits zero on SIGTERM without the harness SIGKILL } }) -test('an HA process finishes an accepted SSE then exits zero under continued keepalive requests', { - timeout: 25000, -}, async (t) => { - const fx = await fixture(t) - const gw = fx.gateway - const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }) - let traffic - let held - let pumping = true - const responses = [] - const diagnostic = () => - JSON.stringify({ - keepalive: responses.slice(-8), - stdout: gw.stdout().slice(-2000), - stderr: gw.stderr().slice(-1500), - }) - try { - const first = await api(gw, 'POST', '/v1/messages', { - body: { model, max_tokens: 16, messages: [{ role: 'user', content: 'completed before SIGTERM' }] }, - }) - assert.equal(first.status, 200, first.text) - await delay(1200) - await fetch(fx.kernel.controlUrl + '/pause', { method: 'POST' }) - held = request(gw, agent, '/v1/messages', { - model, - max_tokens: 16, - stream: true, - messages: [{ role: 'user', content: 'accepted before SIGTERM' }], - }) - held.catch(() => {}) - let active = 0 - for (let i = 0; i < 80; i++) { - active = (await (await fetch(fx.kernel.controlUrl + '/status')).json()).active - if (active === 1) break - await delay(20) - } - assert.equal(active, 1, 'the real node must own an active SSE request before SIGTERM') - // The first queued request uses exactly the active SSE connection once it - // finishes. Continuing its keepalive must not postpone process shutdown. - traffic = (async () => { - while (pumping && gw.child.exitCode == null && gw.child.signalCode == null) { - try { - const response = await request(gw, agent, '/v1/meta') - responses.push({ status: response.status, connection: response.connection }) - } catch {} +for (const afterHeaders of [false, true]) + test(`HA SIGTERM ${afterHeaders ? 'after' : 'before'} SSE headers preserves the stream and exits zero under continued keepalive`, { + timeout: 25000, + }, async (t) => { + const fx = await fixture(t, afterHeaders) + const gw = fx.gateway + const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }) + let traffic + let held + let pumping = true + const responses = [] + const diagnostic = () => + JSON.stringify({ + keepalive: responses.slice(-8), + stdout: gw.stdout().slice(-2000), + stderr: gw.stderr().slice(-1500), + }) + try { + const first = await api(gw, 'POST', '/v1/messages', { + body: { model, max_tokens: 16, messages: [{ role: 'user', content: 'completed before SIGTERM' }] }, + }) + assert.equal(first.status, 200, first.text) + assert.equal(first.json.usage.input_tokens, 5) + assert.equal(first.json.usage.output_tokens, 7) + await delay(1200) + fx.kernel.pause() + let headersSeen = false + held = request( + gw, + agent, + '/v1/messages', + { + model, + max_tokens: 16, + stream: true, + messages: [{ role: 'user', content: 'accepted before SIGTERM' }], + }, + () => { + headersSeen = true + }, + ) + held.catch(() => {}) + let active = 0 + for (let i = 0; i < 80; i++) { + active = fx.kernel.status().active + if (active === 1 && (!afterHeaders || headersSeen)) break await delay(20) } - })() - const exit = once(gw.child, 'exit') - gw.child.kill('SIGTERM') - for (let i = 0; i < 100 && !gw.stdout().includes('[shutdown] SIGTERM'); i++) await delay(10) - assert.match(gw.stdout(), /\[shutdown\] SIGTERM/) - await delay(150) - await fetch(fx.kernel.controlUrl + '/resume', { method: 'POST' }) - const streamed = await held - assert.equal(streamed.status, 200, streamed.text) - assert.match(streamed.text, /message_stop/) - await terminated(gw, exit, diagnostic) - assert.equal( - (await fx.peer.query('SELECT count(*)::int AS n FROM execution_jobs WHERE ended_at IS NULL')).rows[0].n, - 0, - ) - t.diagnostic(`normal SIGTERM exit=0; post-drain keepalive responses=${responses.length}`) - } finally { - pumping = false - agent.destroy() - await Promise.allSettled([traffic, held]) - await fx.close() - } -}) + assert.equal(active, 1, 'the real node must own an active SSE request before SIGTERM') + assert.equal(headersSeen, afterHeaders) + // The first queued request uses exactly the active SSE connection once it + // finishes. Continuing its keepalive must not postpone process shutdown. + traffic = (async () => { + while (pumping && gw.child.exitCode == null && gw.child.signalCode == null) { + try { + const response = await request(gw, agent, '/v1/meta') + responses.push({ status: response.status, connection: response.connection }) + } catch {} + await delay(20) + } + })() + const exit = once(gw.child, 'exit') + gw.child.kill('SIGTERM') + for (let i = 0; i < 100 && !gw.stdout().includes('[shutdown] SIGTERM'); i++) await delay(10) + assert.match(gw.stdout(), /\[shutdown\] SIGTERM/) + await delay(150) + fx.kernel.resume() + const streamed = await held + assert.equal(streamed.status, 200, streamed.text) + assert.match(streamed.text, /message_stop/) + const frames = streamed.text + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => JSON.parse(line.slice(5))) + assert.equal(frames.find((event) => event.type === 'message_delta').usage.output_tokens, 7) + await terminated(gw, exit, diagnostic) + assert.equal( + (await fx.peer.query('SELECT count(*)::int AS n FROM execution_jobs WHERE ended_at IS NULL')).rows[0].n, + 0, + ) + const jobs = (await fx.peer.query('SELECT request_id,node_binding,node_completion,ended_at FROM execution_jobs')) + .rows + assert.equal(jobs.length, 2) + for (const job of jobs) { + assert.equal(job.node_binding.protocol, 'KSG2') + assert.equal(job.node_completion.id, job.request_id) + assert.equal(job.node_completion.status, 'completed') + assert.equal(job.node_completion.dispatched, true) + assert.deepEqual(job.node_completion.fence, job.node_binding.fence) + assert.ok(job.ended_at) + } + t.diagnostic(`normal SIGTERM exit=0; post-drain keepalive responses=${responses.length}`) + } finally { + pumping = false + agent.destroy() + await Promise.allSettled([traffic, held]) + await fx.close() + } + }) From 8071a2cb383fc205e512e76cdf022c1bba5778a3 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:33:51 +0800 Subject: [PATCH 464/473] test: reproduce unsafe bootstrap IDs and implicit business migration --- test/unit/ha-node-bootstrap.test.mjs | 107 ++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs index a7b64abcc..89fb713c4 100644 --- a/test/unit/ha-node-bootstrap.test.mjs +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -9,8 +9,14 @@ import tls from 'node:tls' import { fileURLToPath } from 'node:url' import pg from 'pg' import { NodeManagementGrants } from '../../src/lib/cluster/node-management-grants.mjs' -import { getDb, createDatabase, quoteIdentifier } from '../../src/lib/db/database.mjs' -import { GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' +import { + getDb, + createDatabase, + quoteIdentifier, + PostgresDatabase, + applyMigrations, +} from '../../src/lib/db/database.mjs' +import { coordinationSchemaFor, GlobalMaintenanceCoordinator } from '../../src/lib/db/global-maintenance.mjs' import { realAllocationGate } from '../fixtures/real-allocation-gate.mjs' import { allocateNodeSlot } from '../../src/lib/transport/slotgate-management-client.mjs' @@ -536,3 +542,100 @@ nodeCase('an unsent allocation terminal commit failure retains its grant and bot assert.ok(bg.unknown_at) assert.equal(bg.finished_at, null) }) + +nodeCase('reserved VM ids are rejected before grants or node actions', async (fx) => { + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + for (const reserved of ['active', 'create', 'create-options', 'import', 'package', 'account-chat', 'ACTIVE']) { + fx.config.vm.id = reserved + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const result = await fx.cli('slot-create') + assert.equal(fx.calls.length, 0, `reserved id ${reserved} must not cause node actions`) + assert.equal(result.status, 1) + assert.match(result.err, /bootstrap_config_invalid/) + assert.equal((await fx.peer.query(`SELECT count(*)::int AS n FROM ${fx.q}.allocations`)).rows[0].n, 0) + } +}) + +async function businessSnapshot(db) { + const migrations = (await db.query('SELECT * FROM schema_migrations ORDER BY version')).rows + const columns = ( + await db.query( + `SELECT table_name,column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema=$1 ORDER BY table_name,ordinal_position`, + [db.schema], + ) + ).rows + const relations = ( + await db.query( + `SELECT c.relname,c.relkind FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 ORDER BY c.relname`, + [db.schema], + ) + ).rows + return crypto.createHash('sha256').update(JSON.stringify({ migrations, columns, relations })).digest('hex') +} +async function targetSchema(fx, suffix, callback) { + const schema = fx.db.schema + suffix + const db = new PostgresDatabase({ connectionString: fx.db.connectionString, schema }) + try { + await callback(db) + } finally { + await db.close() + const { nodeAuthoritySchemaFor } = await import('../../src/lib/db/node-management-authority.mjs') + for (const name of [nodeAuthoritySchemaFor(schema), coordinationSchemaFor(schema), schema]) + await fx.peer.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(name)} CASCADE`) + } +} + +nodeCase('an outdated business schema remains at 020 when trusted-init has an invalid reader', async (fx) => { + await targetSchema(fx, '_old', async (db) => { + const dir = path.join(fx.root, '020-migrations') + fs.mkdirSync(dir) + const source = fileURLToPath(new URL('../../src/lib/db/postgres-migrations/', import.meta.url)) + for (const file of fs.readdirSync(source)) + if (/^\d+.*\.sql$/.test(file) && Number(file.split('_')[0]) <= 20) + fs.copyFileSync(path.join(source, file), path.join(dir, file)) + await applyMigrations(db, { migrationsDir: dir }) + const before = await businessSnapshot(db) + fx.config.database.schema = db.schema + fx.config.reader.role = 'missing_reader_' + crypto.randomBytes(8).toString('hex') + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const result = await fx.cli('trusted-init') + assert.equal(result.status, 1) + assert.equal( + (await db.query("SELECT count(*)::int AS n FROM schema_migrations WHERE version='021'")).rows[0].n, + 0, + 'failed CLI must not apply migration 021', + ) + assert.equal(await businessSnapshot(db), before) + assert.match(result.err, /database_migrations_not_current/) + assert.equal( + (await fx.peer.query('SELECT to_regnamespace($1) AS present', [coordinationSchemaFor(db.schema)])).rows[0] + .present, + null, + ) + }) +}) + +nodeCase('neither CLI command creates a missing business schema', async (fx) => { + for (const command of ['trusted-init', 'slot-create']) { + await targetSchema(fx, '_absent_' + command.replace('-', '_'), async (db) => { + fx.config.database.schema = db.schema + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + const result = await fx.cli(command) + assert.equal( + (await fx.peer.query('SELECT to_regnamespace($1) AS present', [db.schema])).rows[0].present, + null, + `${command} must not create business schema`, + ) + assert.equal(result.status, 1) + assert.match(result.err, /database_migrations_not_current/) + assert.equal( + (await fx.peer.query('SELECT to_regnamespace($1) AS present', [coordinationSchemaFor(db.schema)])).rows[0] + .present, + null, + ) + assert.equal(fx.calls.length, 0) + }) + } +}) From 48aa2a284ab057c2b525d5426c6e7dd1c74a744d Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:36:05 +0800 Subject: [PATCH 465/473] fix: verify bootstrap prerequisites before node or business mutations --- docs/deploy/ha-node-bootstrap.md | 4 ++- scripts/ha-node-bootstrap.mjs | 8 +++++- src/lib/db/database.mjs | 43 +++++++++++++++++++++++++++- test/unit/ha-node-bootstrap.test.mjs | 31 ++++++++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/docs/deploy/ha-node-bootstrap.md b/docs/deploy/ha-node-bootstrap.md index ec08328e9..d54d9a198 100644 --- a/docs/deploy/ha-node-bootstrap.md +++ b/docs/deploy/ha-node-bootstrap.md @@ -4,6 +4,8 @@ 运行环境需要 Node.js 22、PostgreSQL 17 应用 owner 的连接，以及到所选节点的固定 TLS 链路。配置必须是当前操作者所有、权限恰为 `0600` 的普通文件；拒绝符号链接和 hard link。业务 schema 必须显式指定且不能是 `public`、PostgreSQL 系统 schema 或 node/coord 派生 schema。DBA 预先创建无继承、无成员关系、无超级用户/建库/建角色/复制/BYPASSRLS 权限的独立 LOGIN reader 角色。密码由 DBA 单独配置，不能写入命令参数。 +业务 schema 必须已由应用的授权迁移流程创建并升级到当前制品版本。两个命令都先只读核验现有 migration 集合、名称和 checksum，再进入私有 authority 操作；不会创建业务 schema 或应用业务迁移。缺失、落后、未知版本或 checksum 不符时返回 `database_migrations_not_current`。须先通过应用的授权迁移流程处理，不能修改迁移记录来绕过检查。 + 以下是配置结构。用安全编辑器写入私有文件，示例字符串必须替换；不要把实际连接串、节点 token 或凭据贴到终端历史。 ```json @@ -24,7 +26,7 @@ } ``` -node ID、VM ID、account UUID 使用原始值。TLS/policy 字段必须与应用的固定 HA trust 文件以及节点配置一致。`policyDigest` 的生成规则见 [节点初次发布协议](../plans/2026-10-22-node-initial-publication.md)，不能从未授权的 runtime State 猜测。`vm.fingerprint` 可显式提供 `{device_id:64位小写hex,session_id:UUID}`；未提供时仅在首次新记录中生成并与该记录原子保存。 +node ID、VM ID、account UUID 使用原始值。VM ID 遵循现有槽位 ID 校验，拒绝 `active`、`create`、`create-options`、`import`、`package` 和以 `-chat` 结尾的名称（不区分大小写）；该检查发生在任何节点动作前。TLS/policy 字段必须与应用的固定 HA trust 文件以及节点配置一致。`policyDigest` 的生成规则见 [节点初次发布协议](../plans/2026-10-22-node-initial-publication.md)，不能从未授权的 runtime State 猜测。`vm.fingerprint` 可显式提供 `{device_id:64位小写hex,session_id:UUID}`；未提供时仅在首次新记录中生成并与该记录原子保存。 先运行： diff --git a/scripts/ha-node-bootstrap.mjs b/scripts/ha-node-bootstrap.mjs index fa9ba84b0..c9c083d8e 100644 --- a/scripts/ha-node-bootstrap.mjs +++ b/scripts/ha-node-bootstrap.mjs @@ -10,6 +10,7 @@ import { NodeCredentialRefreshGrants } from '../src/lib/cluster/node-credential- import { SharedVmOperations } from '../src/lib/vm/shared-vm-operations.mjs' import { SharedBackgroundTasks } from '../src/lib/core/shared-background-tasks.mjs' import { VmsRepo } from '../src/lib/db/repos/vms-repo.mjs' +import { isValidVmId } from '../src/lib/vm/vm-file.mjs' function failure(code, ended = false) { return Object.assign(new Error(code), { code, ...(ended ? { activityEnded: true } : {}) }) @@ -109,6 +110,7 @@ function configFrom(file) { if (c.vm !== undefined) { shape(c.vm, ['id', 'accountId'], ['fingerprint']) id(c.vm.id) + if (!isValidVmId(c.vm.id)) throw failure('bootstrap_config_invalid', true) id(c.vm.accountId) if (c.vm.fingerprint !== undefined) { shape(c.vm.fingerprint, ['device_id', 'session_id']) @@ -357,7 +359,11 @@ export async function main(args = process.argv.slice(2)) { } const input = command === 'slot-create' ? credentialFrom(config, credentialFd) : null process.env.VM2API_CONTROL_PLANE_MODE = 'kubernetes-ha' - db = await createDatabase({ connectionString: config.database.url, schema: config.database.schema }) + db = await createDatabase({ + connectionString: config.database.url, + schema: config.database.schema, + verifyMigrations: true, + }) const coordinator = getActivityCoordinator(db), state = await coordinator.getState() await coordinator.acknowledgeEpoch(state.epoch) diff --git a/src/lib/db/database.mjs b/src/lib/db/database.mjs index a4b9793fd..edb2220c8 100644 --- a/src/lib/db/database.mjs +++ b/src/lib/db/database.mjs @@ -257,12 +257,52 @@ export async function applyMigrations(db, { migrationsDir = MIGRATIONS_DIR } = { }) } +/** Verify an operator-provisioned business schema without creating or migrating it. */ +async function verifyCurrentMigrations(db, { migrationsDir = MIGRATIONS_DIR } = {}) { + const unavailable = () => + Object.assign(new Error('database migrations are not current'), { + code: 'database_migrations_not_current', + }) + return withTransaction(db, async () => { + await db.exec('SET TRANSACTION READ ONLY') + await db.query('SELECT pg_advisory_xact_lock_shared(hashtext(current_database()),hashtext($1))', [ + `vm2api:${db.schema}`, + ]) + const present = ( + await db.query('SELECT to_regnamespace($1) IS NOT NULL AND to_regclass($2) IS NOT NULL AS ready', [ + db.schema, + `${quoteIdentifier(db.schema)}.schema_migrations`, + ]) + ).rows[0] + if (!present.ready) throw unavailable() + const applied = new Map( + (await db.query(`SELECT version,name,checksum FROM ${quoteIdentifier(db.schema)}.schema_migrations`)).rows.map( + (row) => [row.version, row], + ), + ) + const files = fs + .readdirSync(migrationsDir) + .filter((file) => /^\d+.*\.sql$/.test(file)) + .sort() + if (applied.size !== files.length) throw unavailable() + for (const file of files) { + const row = applied.get(file.split('_')[0]) + const sql = fs.readFileSync(path.join(migrationsDir, file), 'utf8').replaceAll('\r\n', '\n') + const checksum = crypto.createHash('sha256').update(sql).digest('hex') + if (row?.name !== file || row.checksum !== checksum) throw unavailable() + } + }) +} + export async function createDatabase(options = {}) { if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'restore-maintenance.json'))) { throw new Error('restore maintenance marker exists; complete offline recovery before startup') } const db = new PostgresDatabase({ ...options, schema: options.schema || process.env.KIN_DATABASE_SCHEMA || 'public' }) try { + // Offline administration must verify before even private coordination DDL. + // The live bootstrap scope repeats this check before using the schema. + if (options.verifyMigrations === true) await verifyCurrentMigrations(db, options) if (isKubernetesMode()) { const { GlobalMaintenanceCoordinator } = await import('./global-maintenance.mjs') const coordinator = new GlobalMaintenanceCoordinator(db) @@ -274,7 +314,8 @@ export async function createDatabase(options = {}) { const version = Number((await db.prepare('SHOW server_version_num').get()).server_version_num) if (version < 170000 || version >= 180000) throw new Error('vm2api requires PostgreSQL 17 and PostgreSQL 17 backup clients') - await applyMigrations(db, options) + if (options.verifyMigrations === true) await verifyCurrentMigrations(db, options) + else await applyMigrations(db, options) if (options.dataDir && fs.existsSync(path.join(options.dataDir, 'kin.db')) && !options.allowLegacySource) { const imported = await db.prepare("SELECT value FROM settings WHERE key = 'sqlite_import_done'").get() if (!imported) throw new Error('existing SQLite data requires db:import:sqlite before PostgreSQL startup') diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs index 89fb713c4..3e15d3c5d 100644 --- a/test/unit/ha-node-bootstrap.test.mjs +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -609,6 +609,10 @@ nodeCase('an outdated business schema remains at 020 when trusted-init has an in ) assert.equal(await businessSnapshot(db), before) assert.match(result.err, /database_migrations_not_current/) + const slot = await fx.cli('slot-create') + assert.equal(slot.status, 1) + assert.match(slot.err, /database_migrations_not_current/) + assert.equal(await businessSnapshot(db), before) assert.equal( (await fx.peer.query('SELECT to_regnamespace($1) AS present', [coordinationSchemaFor(db.schema)])).rows[0] .present, @@ -617,6 +621,33 @@ nodeCase('an outdated business schema remains at 020 when trusted-init has an in }) }) +nodeCase('both CLI commands reject mismatched or unknown migrations without changing business state', async (fx) => { + for (const kind of ['checksum', 'future']) { + await targetSchema(fx, '_' + kind, async (db) => { + await applyMigrations(db) + if (kind === 'checksum') await db.query("UPDATE schema_migrations SET checksum='invalid' WHERE version='001'") + else + await db.query( + "INSERT INTO schema_migrations(version,name,checksum,applied_at) VALUES('999','999_future.sql','synthetic','synthetic')", + ) + const before = await businessSnapshot(db) + fx.config.database.schema = db.schema + fs.writeFileSync(fx.file, JSON.stringify(fx.config)) + for (const command of ['trusted-init', 'slot-create']) { + const result = await fx.cli(command) + assert.equal(result.status, 1) + assert.match(result.err, /database_migrations_not_current/) + assert.equal(await businessSnapshot(db), before) + } + assert.equal( + (await fx.peer.query('SELECT to_regnamespace($1) AS present', [coordinationSchemaFor(db.schema)])).rows[0] + .present, + null, + ) + }) + } +}) + nodeCase('neither CLI command creates a missing business schema', async (fx) => { for (const command of ['trusted-init', 'slot-create']) { await targetSchema(fx, '_absent_' + command.replace('-', '_'), async (db) => { From 9ae50abd12f69546362fb854681d07ff25d318b0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:45:01 +0800 Subject: [PATCH 466/473] test: reproduce mixed-case bootstrap schema rejection --- test/unit/ha-node-bootstrap.test.mjs | 41 +++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/test/unit/ha-node-bootstrap.test.mjs b/test/unit/ha-node-bootstrap.test.mjs index 3e15d3c5d..99d650801 100644 --- a/test/unit/ha-node-bootstrap.test.mjs +++ b/test/unit/ha-node-bootstrap.test.mjs @@ -36,8 +36,10 @@ const payload = Buffer.from( }), ) -async function fixture(t) { - const db = getDb() +async function fixture(t, { mixedSchema = false } = {}) { + const db = mixedSchema + ? await createDatabase({ connectionString: getDb().connectionString, schema: getDb().schema + '_Mixed' }) + : getDb() const peer = await createDatabase({ connectionString: db.connectionString, schema: db.schema }) const coordinator = new GlobalMaintenanceCoordinator(db) const root = fs.mkdtempSync(path.join(os.tmpdir(), 'nmc-')) @@ -56,6 +58,7 @@ async function fixture(t) { } await coordinator.stop() await peer.close() + if (mixedSchema) await db.close() const cleanup = new pg.Client({ connectionString: db.connectionString }) await cleanup.connect() try { @@ -64,6 +67,7 @@ async function fixture(t) { await cleanup.query( `DROP SCHEMA IF EXISTS ${quoteIdentifier(service.schema)} CASCADE; DROP SCHEMA IF EXISTS ${quoteIdentifier(coordinator.schema)} CASCADE`, ) + if (mixedSchema) await cleanup.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(db.schema)} CASCADE`) } finally { await cleanup.end() fs.rmSync(root, { recursive: true, force: true }) @@ -204,9 +208,9 @@ async function fixture(t) { } } -function nodeCase(name, callback) { +function nodeCase(name, callback, options) { test(name, async (t) => { - const fx = await fixture(t) + const fx = await fixture(t, options) try { await callback(fx, t) } finally { @@ -215,6 +219,35 @@ function nodeCase(name, callback) { }) } +nodeCase( + 'both CLI commands accept an already migrated mixed-case schema without a lowercase alias', + async (fx) => { + assert.equal( + ( + await fx.peer.query('SELECT count(*)::int AS n FROM pg_namespace WHERE nspname=$1', [ + fx.db.schema.toLowerCase(), + ]) + ).rows[0].n, + 0, + ) + const init = await fx.cli('trusted-init') + assert.equal(init.status, 0, init.out + init.err) + await fx.db.pool.query(fs.readFileSync(fx.output, 'utf8')) + const created = await fx.cli('slot-create') + assert.equal(created.status, 0, created.out + created.err) + assert.equal((await fx.peer.query('SELECT count(*)::int AS n FROM vms WHERE id=$1', [vmId])).rows[0].n, 1) + assert.equal( + ( + await fx.peer.query('SELECT count(*)::int AS n FROM pg_namespace WHERE nspname=$1', [ + fx.db.schema.toLowerCase(), + ]) + ).rows[0].n, + 0, + ) + }, + { mixedSchema: true }, +) + nodeCase( 'trusted-init writes nonsecret reader SQL and slot-create persists only confirmed public metadata', async (fx) => { From ff4e43486fa98af21aee366f8c3779202e81e1d0 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:45:36 +0800 Subject: [PATCH 467/473] fix: quote mixed-case schema during bootstrap verification --- src/lib/db/database.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/db/database.mjs b/src/lib/db/database.mjs index edb2220c8..668a84a71 100644 --- a/src/lib/db/database.mjs +++ b/src/lib/db/database.mjs @@ -270,7 +270,7 @@ async function verifyCurrentMigrations(db, { migrationsDir = MIGRATIONS_DIR } = ]) const present = ( await db.query('SELECT to_regnamespace($1) IS NOT NULL AND to_regclass($2) IS NOT NULL AS ready', [ - db.schema, + quoteIdentifier(db.schema), `${quoteIdentifier(db.schema)}.schema_migrations`, ]) ).rows[0] From 37a4cac17e9006989aa2365d865f636b00634413 Mon Sep 17 00:00:00 2001 From: liubing Date: Thu, 22 Oct 2026 02:53:49 +0800 Subject: [PATCH 468/473] docs: record scoped multi-node HA verification and independent reviews --- deploy/k8s-ha/hpa.example.yaml | 2 +- docs/deploy/ha-node-bootstrap.md | 6 +- docs/deploy/kubernetes-ha.md | 23 +- .../2026-10-21-kubernetes-review-progress.md | 18 +- .../2026-10-22-kubernetes-ha-verification.md | 64 ++ .../ha-verification/48aa2a28-ha-smoke.json | 89 +++ .../48aa2a28-runtime-identity.json | 19 + .../kubernetes/ha-verification/checks.json | 345 ++++++++++ .../ha-verification/f53e005f-ha-crash.json | 71 ++ .../ha-verification/f53e005f-ha-hpa.json | 612 ++++++++++++++++++ .../f53e005f-ha-lifecycle.json | 348 ++++++++++ .../ha-verification/f53e005f-ha-smoke.json | 89 +++ .../ha-verification/f53e005f-ha-ws-drain.json | 45 ++ .../f53e005f-runtime-identity.json | 18 + .../independent-review-48aa2a28.md | 20 + .../independent-review-a6b67f3b.md | 56 ++ .../independent-review-ceb27752.md | 31 + .../independent-review-f53e005f.md | 17 + .../independent-review-ff4e4348.md | 13 + 19 files changed, 1880 insertions(+), 6 deletions(-) create mode 100644 docs/plans/2026-10-22-kubernetes-ha-verification.md create mode 100644 docs/plans/evidence/kubernetes/ha-verification/48aa2a28-ha-smoke.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/48aa2a28-runtime-identity.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/checks.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-crash.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-hpa.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-lifecycle.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-smoke.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-ws-drain.json create mode 100644 docs/plans/evidence/kubernetes/ha-verification/f53e005f-runtime-identity.json create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-48aa2a28.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-a6b67f3b.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-ceb27752.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-f53e005f.md create mode 100644 docs/plans/evidence/kubernetes/review-progress/independent-review-ff4e4348.md diff --git a/deploy/k8s-ha/hpa.example.yaml b/deploy/k8s-ha/hpa.example.yaml index 558ecefa0..4fafd240c 100644 --- a/deploy/k8s-ha/hpa.example.yaml +++ b/deploy/k8s-ha/hpa.example.yaml @@ -1,4 +1,4 @@ -# Optional. Install metrics-server and choose limits for your available nodes. +# Optional. Install metrics-server and reserve one extra node above maxReplicas for rolling updates. # Do not manage spec.replicas with kubectl apply after enabling this HPA. apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler diff --git a/docs/deploy/ha-node-bootstrap.md b/docs/deploy/ha-node-bootstrap.md index d54d9a198..36b67aea7 100644 --- a/docs/deploy/ha-node-bootstrap.md +++ b/docs/deploy/ha-node-bootstrap.md @@ -48,8 +48,10 @@ node scripts/ha-node-bootstrap.mjs slot-create --config /private/bootstrap.json 也可从显式文件描述符读取凭据。此时从配置移除 `credentialFile`，调用 `slot-create --config /private/bootstrap.json --credential-fd 0` 并由操作者的受控 secret 管道提供 stdin。凭据内容不进入 argv、stdout 或 PostgreSQL VM 元数据。 -`slot-create` 持有一个持久 bootstrap BG claim，顺序执行独立的 allocate、publish-initial、launch VM operation。每一步都等待严格 receipt 后再进入下一步。RT 仅保存在节点未挂载的 canonical 路径；guest 只得到 AT 投影。launch receipt 确认后才原子新建公开 VM 元数据和指纹。已有 VM、并发插入或未知旧任务均拒绝，不覆盖记录。stdout 只输出公开 IDs、manifest 摘要与原 captured Fence；它证明节点 launch/元数据步骤，不证明真实模型推理成功。 +`slot-create` 持有一个持久 bootstrap BG claim，顺序执行独立的 allocate、publish-initial、launch VM operation。每一步都等待严格 receipt 后再进入下一步。refresh token（RT）仅保存在节点未挂载的 canonical 路径；guest 只得到 access token（AT）投影。launch receipt 确认后才原子新建公开 VM 元数据和指纹。已有 VM、并发插入或未知旧任务均拒绝，不覆盖记录。stdout 只输出公开 IDs、manifest 摘要与原 captured Fence；它证明节点 launch/元数据步骤，不证明真实模型推理成功。 发生发送后的超时、断连、节点启动未知或元数据提交失败时，相关持久 claim 和目标占位保留。再次执行不会换 command ID 重发。首个节点回调前明确 busy 的拒绝会结束本次未提交任务。首个 allocate 在 TLS 应用数据发送前失败时，只有原 grant 的未开始终态及撤销已持久化后才结束本次任务；操作者修正 pin/连接配置后可显式再运行。终态写入失败仍保留未知占位。不能删 claim、改 VM ID 或重新发 allocation 来猜测恢复；须先按节点终止/恢复证据流程人工核实。CLI 不提供未知任务自动恢复，也不升级旧的 RT 已暴露 manifest。 -测试只覆盖合成凭据、任务 PostgreSQL、TLS 和 fake Engine；没有真实 provider 或现有节点操作。闭源 kernel 在投影更新后是否重新加载 AT，仍需独立业务验证。旧 HA credential monitor 不能因此直接启用。 +初始化 CLI 的测试覆盖合成凭据、任务 PostgreSQL、TLS 和 fake Engine，没有真实供应商或现有节点操作。另有 [预编译凭据投影测试](../../test/e2e/prebuilt-credential-projection.e2e.test.mjs)：实际预编译 ELF 内核和 CLI 在本机隔离环境中，向本地合成供应商分别发送原子替换前后的 AT；内核与 CLI 进程及 socket 保持不变，投影中不含 RT。该结果确认被测制品可读取更新后的 AT 投影。 + +真实模型推理、真实 RT 刷新和既有生产节点行为仍未验证。普通 Claude Docker HA 的合成多节点验证范围见 [部署验证记录](kubernetes-ha.md#验证记录)。旧 HA credential monitor 仍保持停用；上述测试不改变支持范围，部署依赖 [#649](https://github.com/dofastted/vm2api/pull/649) 仍为 open。 diff --git a/docs/deploy/kubernetes-ha.md b/docs/deploy/kubernetes-ha.md index d22cb72f0..0e1aca711 100644 --- a/docs/deploy/kubernetes-ha.md +++ b/docs/deploy/kubernetes-ha.md @@ -2,7 +2,9 @@ `deploy/k8s-ha` 使用 `VM2API_CONTROL_PLANE_MODE=kubernetes-ha`。默认部署两个控制面 Pod，通过 PostgreSQL 共享状态，通过固定证书指纹的 TLS 节点接口执行请求。控制面不选举推理 leader，各副本独立处理请求。 -该部署清单已通过 Kustomize 渲染和隔离 k3d 集群的服务端试运行。双进程正常 HTTP/SSE 测试使用真实 PostgreSQL、Go 节点协议和合成执行器。多节点滚动更新、异常退出恢复及既有预编译内核的业务请求验收尚未完成。本页不作为生产启用依据。 +候选 `f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0` 已在隔离 k3d 集群验证普通 Claude Docker HA 的跨节点并发、扩缩容、滚动更新、控制面异常退出恢复和 HPA 扩容。测试使用真实 PostgreSQL、Go TLS 节点协议，以及合成的本地内核和供应商；具体范围见下方验证记录。运行镜像通过 rootfs 层摘要与七个关键运行文件的 SHA-256 对照确认，未仅依赖进程报告的版本字段。 + +预编译内核的 access token（AT）投影重载已有单独的本机合成供应商测试。真实模型请求、真实 refresh token（RT）刷新、成功的 WebSocket 模型推理和 Ingress 运行行为仍未验证。部署依赖 [#649](https://github.com/dofastted/vm2api/pull/649) 仍为 open；本页不作为生产启用依据。 ## 支持范围 @@ -22,7 +24,7 @@ ## 安装前提 1. PostgreSQL 17、相同业务 schema、相同业务加密密钥。所有副本使用相同管理员配置和初始 API key。 -2. 至少两个可调度的 Kubernetes 节点。必需的 Pod 反亲和保证副本分布在不同节点。默认滚动更新为 `maxUnavailable: 0, maxSurge: 1`，因此两个副本更新时需要第三个可调度节点。 +2. 至少两个可调度的 Kubernetes 节点。必需的 Pod 反亲和保证副本分布在不同节点。默认滚动更新为 `maxUnavailable: 0, maxSurge: 1`，因此两个副本更新时需要第三个可调度节点。HPA 最大副本数也应预留这一节点；默认策略需要至少 `maxReplicas + 1` 个可调度节点才能在扩容上限继续滚动更新。 3. 已安装并校验固定镜像 ID、网络 ID 和 UID/GID 的远端 Go 节点服务。节点的原始 Docker/SSH 权限不能交给控制面 Pod。 4. 通过可信初始化安装独立 allocation、command 私有 authority、节点专用只读 PostgreSQL role，以及授权的 allocation → 首次凭据发布 → launch 回执链。普通应用启动不会自行安装这些授权。相关接口见 [节点授权](../plans/2026-10-22-node-command-authority.md) 和 [首次发布](../plans/2026-10-22-node-initial-publication.md)。可信离线初始化与槽位创建命令见 [节点初始化 CLI](ha-node-bootstrap.md)。该命令不安装节点服务或配置宿主 Docker。 @@ -76,3 +78,20 @@ Pod 使用 UID/GID 1000、只读根文件系统，不挂载 Kubernetes API token 可选 `deploy/k8s-ha/hpa.example.yaml` 使用 CPU 指标，需要 metrics-server。调整最大副本数以匹配节点数和 PostgreSQL 容量。启用 HPA 后，不应继续通过应用清单覆盖 `spec.replicas`。 Pod 中断预算要求自愿驱逐时至少保留一个副本。它不替代节点故障处理，也不延长长连接的终止宽限期。入口层的 WebSocket、SSE 缓冲和超时仍需按实际网关配置验证。 + +## 验证记录 + +以下记录对应候选 `f53e005f` 的隔离 k3d 环境，执行端内核与供应商均为本地合成服务，没有使用真实供应商凭据。 + +| 场景 | 已验证行为 | +| --- | --- | +| 两个活动 Pod | 分布于不同节点，并发派发请求；共享面板会话、账号/VM/API key 准入、路由更新和计费状态 | +| HTTP 与 SSE | 按原节点完成证明记录计费并释放占位；取消等待者释放其预留；未支持的管理动作在宿主操作前拒绝 | +| 手动扩缩容与滚动更新 | 两副本扩至三副本、缩回两副本并滚动重启；期间 45 次 HTTP/SSE 请求均为 200，采样可用副本数至少为二 | +| 控制面强制退出 | 通过 CRI 立即终止一个控制面容器；原节点未完成时继续保留容量，收到原 command 的完成回执后由其他副本恢复容量并只计费一次 | +| HPA | 在测试覆盖配置 `minReplicas: 2`、`maxReplicas: 3`、CPU 阈值 1% 下自动扩至三个就绪副本；31 次请求均为 200。未验证默认 70% 阈值下的生产容量 | +| WebSocket 排空 | 升级成功、Claude Responses 按既有策略拒绝；滚动更新时以 1001 关闭空闲连接，重试后可连接新 Pod。未验证成功的 WebSocket 模型推理 | + +另有 [预编译凭据投影测试](../../test/e2e/prebuilt-credential-projection.e2e.test.mjs)：使用实际预编译 ELF 内核和 CLI，将仅含 AT 的凭据文件原子替换。两次请求分别使用替换前后的合成 AT，内核与 CLI 进程及 socket 保持不变。该结果证明被测制品能读取更新后的投影，不证明真实供应商接受凭据、真实 RT 轮换或既有生产节点行为。Ingress 的 WebSocket/SSE 缓冲与超时配置仍需在实际入口中验证。 + +固定源码检查、独立审查和全量既有失败详见 [HA 验证记录](../plans/2026-10-22-kubernetes-ha-verification.md)。 diff --git a/docs/plans/2026-10-21-kubernetes-review-progress.md b/docs/plans/2026-10-21-kubernetes-review-progress.md index b8dfa58f7..6b0b6894a 100644 --- a/docs/plans/2026-10-21-kubernetes-review-progress.md +++ b/docs/plans/2026-10-21-kubernetes-review-progress.md @@ -1,6 +1,6 @@ # Kubernetes 独立审查与定向验证（#648，进行中） -本报告记录固定源码的检查结果，不作为多节点高可用验收。控制面仍保留 `ControlPlaneLock` 和 `replicas: 1`。已发布的 Draft PR #651 仍为早期提交；后续本地实现尚未推送。 +本报告前五轮记录单实例基础阶段的固定源码结果，不作为后续 HA 验收。显式 `kubernetes-ha` 和独立双节点部署验证已在后续增量加入，默认 `kubernetes` 清单仍为单实例。已发布的 Draft PR #651 仍为早期提交；后续本地实现尚未推送。 ## 全量检查 @@ -60,3 +60,19 @@ NUL 问题动态复现得到 PostgreSQL `22021`。修复恢复依赖分支已有 定向协议检查使用专用测试 PostgreSQL、合成请求、模拟节点或测试自己的子进程。KSG2 集成使用合成 executor 身份，没有调用真实 Docker Engine；它不能证明可信启动或权限隔离。旧 E2E harness 的 mock gateway 仍会创建真实本机 Docker 网络，已在 `f9f396dd` 隔离；此前资源按精确 fixture、network ID、无端点和无 helper 证据核对后，仅删除 10 个测试网络。6 个带业务端点的网络和 3 个固定名网络保留。生产 capability 来源、节点文件发布和管理授权、多节点运行及故障接管仍在实现。旧预编译 payload 继续复用。 结构化统计、日志摘要与边界见 [验证证据](evidence/kubernetes/review-progress/verification.json)。已精确删除 9 个完成检查的 root 工作树或源码副本；另有固定检查脚本自行清理的目录。专用 PostgreSQL 与共享 Go 缓存在后续验证结束前保留。没有修改生产环境。 + +## HA 增量的独立审查 + +第六轮审查源码 `a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f`，读取 95 文件增量。发现 1 项 P1、2 项 P2，分别涉及配置刷新重启旧 monitor、Responses 500 回归和旧 attempt 被清理。已用实际 PG/HTTP 复现并修复，见 [第六轮报告](evidence/kubernetes/review-progress/independent-review-a6b67f3b.md)。 + +第七轮审查源码 `ceb2775221fb73549e542855356073bacf8e54a8`，独立会话 `01a15f94-f885-7b82-b9b0-879ec03faf2d`，读取 79 文件增量。发现 2 项 P2：明确未发送的刷新 grant 占用 generation；派发前取消的非空临时 command ID 导致计费预约保留。已分别通过真实 PG/TLS 和正常 handler 失败规格修复，见 [第七轮报告](evidence/kubernetes/review-progress/independent-review-ceb27752.md)。该轮没有覆盖后来的 bootstrap CLI、恢复扫描监控器和滚动终止修复。 + +后续同行审查另复现 allocation pin 拒绝的未开始处理，以及扫描进程 SIGKILL 留下 activity 的问题。两项均已完成针对性修复。滚动更新测试还复现了已接受 SSE 的 keep-alive 连接在排空后持续复用。`f53e005f` 修正后，独立进程的完整 SSE 和正常退出检查通过；专用 k3d 的滚动期间 25 次业务请求均为 200，采样可用副本数至少为二。 + +第八轮审查源码 `f53e005f`，独立会话 `01a15fb1-f02a-7761-9bf2-7cdaa751f3aa`，读取 27 文件增量。新增 2 项 P2：初始化 CLI 接受保留 VM ID，以及节点初始化隐式迁移业务 schema。`48aa2a28` 修正，相关固定源码检查 50/50。见 [第八轮报告](evidence/kubernetes/review-progress/independent-review-f53e005f.md)。 + +第九轮审查源码 `48aa2a28`，独立会话 `01a15fc4-4eb3-72f3-a643-07fd87e6f797`，复核上述修复。新增混合大小写 schema 被 `to_regnamespace` 折叠小写而错误拒绝的 P2；实际 PG 失败规格 `9ae50abd` 先于一行修复 `ff4e4348`。见 [第九轮报告](evidence/kubernetes/review-progress/independent-review-48aa2a28.md)。 + +多副本实现、固定源码检查、既有全量失败及运行镜像的具体证据，见 [HA 验证记录](2026-10-22-kubernetes-ha-verification.md)。此处保留早期记录的历史范围，不把早期单实例结果替换为 HA 验收。 + +第十轮审查源码 `ff4e4348`，独立会话 `01a15fc9-6ab0-7da1-98e3-a3a61d0fe2e9`，确认混合大小写修复，未发现新增可证 P0/P1/P2。主会话的固定归档 CLI/数据库检查为 25/25，779 文件格式检查通过。见 [第十轮报告](evidence/kubernetes/review-progress/independent-review-ff4e4348.md)。 diff --git a/docs/plans/2026-10-22-kubernetes-ha-verification.md b/docs/plans/2026-10-22-kubernetes-ha-verification.md new file mode 100644 index 000000000..a26c02e7f --- /dev/null +++ b/docs/plans/2026-10-22-kubernetes-ha-verification.md @@ -0,0 +1,64 @@ +# Kubernetes HA 验证记录（#648） + +本次实现增加显式 `kubernetes-ha` 模式。普通 Claude Docker 槽位的控制面副本通过 PostgreSQL 共享状态，各副本独立处理请求。默认 `kubernetes` 部署仍为单实例。本记录不表示 #648 的全部后端和协议已完成，也不作为生产部署批准。 + +依赖 PostgreSQL 改造 PR [#649](https://github.com/dofastted/vm2api/pull/649)。测试复用既有预编译内核和 CLI，没有其源码改动。 + +## 固定源码与运行镜像 + +| 源码 | 检查 | 结果 | +| --- | --- | --- | +| `f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0` | PG、TLS 节点、恢复扫描、计费和 HTTP/SSE 排空定向检查 | 73/73 | +| `48aa2a284ab057c2b525d5426c6e7dd1c74a744d` | 初始化前提、默认数据库迁移、计费、HA 启动、双副本请求和排空 | 50/50 | +| `ff4e43486fa98af21aee366f8c3779202e81e1d0` | 混合大小写 schema、两个 CLI 和默认数据库回归；固定源码归档 | 25/25 | + +`f53e005f` 镜像的实际 containerd image ID 为 `sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e`。`48aa2a28` 为 `sha256:a4927a617e980e4ca20636810281b201f9d469f76107587fb8f9feaead95586e`。两次均比对 Docker 与 containerd 的 rootfs 层，并将运行容器的关键源码文件与固定归档逐字节比对；没有仅依赖版本或 SHA 字段。见 [运行身份](evidence/kubernetes/ha-verification/48aa2a28-runtime-identity.json)。 + +`ff4e4348` 相对运行中的 `48aa2a28` 只改变 CLI 使用的只读迁移前提查询及其测试。最后这项修复通过固定归档的实际 CLI/PG 测试验证；没有将此前集群镜像结果标成 `ff4e4348` 的运行验收。 + +## 专用 Kubernetes 环境 + +仅使用本任务独立 k3d 集群、任务命名空间和专用 PostgreSQL。未修改生产服务、生产容器、生产网络或默认 kubeconfig。集群有三个可调度节点，支持两个副本及额外一个滚动更新副本。 + +| 场景 | 固定源码 | 实际结果 | +| --- | --- | --- | +| 两节点上的活动副本 | `f53e005f`、`48aa2a28` | 两 Pod 并发派发；跨 Pod 面板 cookie、路由刷新、key 并发限制共享；账号/VM 容量满时第三个不同 key 不进入执行端，取消后释放其预留；9/9 场景通过 | +| HTTP/SSE 与计费 | 同上 | 正常响应和完整 SSE 结束；原节点完成证明关联原命令；唯一结算并释放准入 | +| 手动扩缩容和滚动更新 | `f53e005f` | 2→3、3→2、滚动重启分别发起 19、1、25 次业务请求，均为 200；采样可用副本数不低于二 | +| 控制面异常退出 | `f53e005f` | 通过 CRI 立即停止任务控制面容器，退出码 137；未完成的原节点继续占用容量；另一副本读取原命令持久完成回执后恢复容量，5 个输入 token、7 个输出 token 只计费一次；随后请求成功 | +| HPA | `f53e005f` | 使用实际 metrics-server，以测试覆盖值 CPU 1%、min=2/max=3 自动扩至三个就绪副本；31 次业务请求为 200。没有验证默认 70% 阈值下的容量 | +| WebSocket | `f53e005f` | 升级、Claude Responses 的既有 400 拒绝、滚动时 1001 关闭空闲连接、有界重试后重连通过。立即重连发生过连接错误，原因未查明；未声称零错误重连或成功模型推理 | + +控制面、PostgreSQL、固定证书的 Go TLS 节点和持久日志为实际运行进程。内核和模型供应商为本地合成服务，节点 Engine 为 fixture。上述结果不能替代真实供应商推理、真实 Docker 执行隔离或生产节点验收。各场景的结构化结果见 [证据目录](evidence/kubernetes/ha-verification)。 + +实际预编译 ELF 的单独 [AT 投影测试](../../test/e2e/prebuilt-credential-projection.e2e.test.mjs) 使用本地合成 OAuth 供应商。替换仅含 AT 的文件后，两次请求分别携带新旧合成 AT；内核和 CLI PID、socket 与重启计数不变。它不验证真实 RT 刷新或真实供应商接受凭据。 + +## 全量检查与已知失败 + +固定 `4de403f9ee5411fee1c2266ef80cfd4315267a4b` 的全量结果如下，不能称为全绿： + +| 检查 | 文件 | 测试数 | 通过 | 失败 | 跳过 | +| --- | ---: | ---: | ---: | ---: | ---: | +| unit | 395 | 4373 | 4339 | 24 | 10 | +| E2E | 35 | 135 | 117 | 17 | 1 | + +7 项 unit、17 项 E2E 的规范化错误摘要与此前固定基线一致。新增 17 项 unit 均为 runner 遗漏数据库 fixture 自动初始化，实际错误为 `database not opened — await openDatabase() first`。`0d9c4348` 修正 fixture 识别后，通过实际 isolated runner 复验相关三个文件 18/18。未把这一复验合并成不存在的全量通过数。失败名称及摘要见 [checks.json](evidence/kubernetes/ha-verification/checks.json)。 + +`4de403f9` 的 Go race 为 299 通过、1 辅助跳过。之后到 `48aa2a28` 没有 worker 源码变化。`f53e005f` 的 Go vet、gofmt、779 文件 Biome、336 个源/脚本 MJS 语法、4 项部署静态检查及 1 项 HA entrypoint 检查通过。`ff4e4348` 的 779 文件格式检查及修改的两个 MJS 语法检查也通过。 + +## 独立会话审查 + +审查使用独立 Codex CLI 会话和固定、干净的 detached worktree。审查会话不运行测试、服务或部署命令，不访问凭据；动态复现和修复检查由主会话另行执行。 + +- `f53e005f`，会话 `01a15fb1-f02a-7761-9bf2-7cdaa751f3aa`：2 项 P2（保留 VM ID、CLI 隐式迁移业务 schema），已在 `48aa2a28` 修复。[原报告](evidence/kubernetes/review-progress/independent-review-f53e005f.md)。 +- `48aa2a28`，会话 `01a15fc4-4eb3-72f3-a643-07fd87e6f797`：确认上轮两处修复，新增混合大小写 schema 名称误判 P2，已用实际 PG 失败规格复现，并在 `ff4e4348` 修复。[原报告](evidence/kubernetes/review-progress/independent-review-48aa2a28.md)。 + +- `ff4e4348`，会话 `01a15fc9-6ab0-7da1-98e3-a3a61d0fe2e9`：核对混合大小写修复及两文件增量，未发现新增可证 P0/P1/P2。[原报告](evidence/kubernetes/review-progress/independent-review-ff4e4348.md)。 + +更早轮次及各自范围见 [历史审查记录](2026-10-21-kubernetes-review-progress.md)。 + +## 范围限制 + +HA 仅支持普通 Claude Docker 的 Messages 及 Claude Chat/Completions 转换。GPT/Codex、API 后端、Search、原生执行器、KVM、终端和旧管理入口均明确拒绝。Claude Responses 保持原有拒绝行为，成功 WebSocket 模型推理不在本次支持范围。 + +任务集群没有安装 Ingress controller，Ingress 的实际 SSE 缓冲、WebSocket 和超时行为未验证。真实模型、真实 RT 刷新、已有生产节点及生产容量未验证。没有对生产环境部署本分支。 diff --git a/docs/plans/evidence/kubernetes/ha-verification/48aa2a28-ha-smoke.json b/docs/plans/evidence/kubernetes/ha-verification/48aa2a28-ha-smoke.json new file mode 100644 index 000000000..a4f161c2c --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/48aa2a28-ha-smoke.json @@ -0,0 +1,89 @@ +{ + "source_sha": "48aa2a284ab057c2b525d5426c6e7dd1c74a744d", + "namespace": "vm2api-648-ha-01a15dc9", + "boundary": "real two-node Kubernetes gateways, real PG/TLS Go node, synthetic kernel/provider", + "checks": [ + { + "name": "two_ready_pods_on_distinct_nodes", + "passed": true, + "nodes": [ + "k3d-vm2api-648-01a15dc9-agent-1", + "k3d-vm2api-648-01a15dc9-server-0" + ] + }, + { + "name": "cross_pod_panel_session", + "passed": true + }, + { + "name": "both_pods_dispatch_concurrently", + "passed": true, + "held": { + "keys": 2, + "executions": 2, + "turns": 2 + } + }, + { + "name": "global_account_vm_cap_and_cancelled_waiter_release", + "passed": true, + "kernelActive": 2, + "held": { + "keys": 2, + "executions": 2, + "turns": 2 + } + }, + { + "name": "http_sse_original_node_completion_and_billing", + "passed": true, + "ledger": [ + { + "outcome": "charged", + "n": 2 + } + ], + "held": { + "keys": 0, + "executions": 0, + "turns": 0 + }, + "kernelPeak": 2 + }, + { + "name": "global_key_concurrency_across_pods", + "passed": true, + "deniedStatus": 429 + }, + { + "name": "cross_pod_routing_refresh", + "passed": true + }, + { + "name": "unsupported_management_refuses_before_host_io", + "passed": true + }, + { + "name": "websocket_claude_responses_rejection", + "passed": true, + "inference_success": false + } + ], + "pods": [ + { + "name": "vm2api-b6b455568-668kl", + "uid": "c4a88f21-13ed-422d-b701-c2f391af7c3a", + "node": "k3d-vm2api-648-01a15dc9-agent-1", + "image": "docker.io/library/vm2api:kubernetes-ha-48aa2a28", + "imageID": "sha256:a4927a617e980e4ca20636810281b201f9d469f76107587fb8f9feaead95586e" + }, + { + "name": "vm2api-b6b455568-fj55g", + "uid": "c23b600e-e9be-4ac6-994b-297f3a6b7a0b", + "node": "k3d-vm2api-648-01a15dc9-server-0", + "image": "docker.io/library/vm2api:kubernetes-ha-48aa2a28", + "imageID": "sha256:a4927a617e980e4ca20636810281b201f9d469f76107587fb8f9feaead95586e" + } + ], + "passed": true +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/48aa2a28-runtime-identity.json b/docs/plans/evidence/kubernetes/ha-verification/48aa2a28-runtime-identity.json new file mode 100644 index 000000000..e5cece999 --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/48aa2a28-runtime-identity.json @@ -0,0 +1,19 @@ +{ + "source_sha": "48aa2a284ab057c2b525d5426c6e7dd1c74a744d", + "docker_image_config_id": "sha256:a362a7f9cf98a6de4399adad4b70c5f8e25fb6943b1e9a7b00b20e7ed452e002", + "containerd_image_id": "sha256:a4927a617e980e4ca20636810281b201f9d469f76107587fb8f9feaead95586e", + "rootfs_diff_ids_match": true, + "rootfs_layer_count": 24, + "rootfs_diff_ids_sha256": "4fef9951b1368c31922a79344fc7dc6b04b3ddce27b8c9b74951ec1de426ec64", + "runtime_pod": "vm2api-b6b455568-668kl", + "runtime_files_sha256": { + "src/server.mjs": "3c87b8534a0486030ed2cca85885c0f1dd058551bbfb7d14bf3914bcc5a60897", + "src/lib/core/ha-recovery-monitor.mjs": "a787e1b98be6af95d85d8bd265e578fecd9ff68c43ae4eb315dcd4e3cba0d440", + "src/lib/core/ha-request-billing.mjs": "4411f6e37bfad824c5de49d9b0cf0c2f507848ef104c7f45e8ceb492ab0ce37e", + "src/lib/db/global-maintenance.mjs": "d2e928ef142faf0cb1666dac1275c957af0c0576739314911bce38e650798051", + "src/lib/db/database.mjs": "a4cb51f852cdbb349fe25d52d23a33cd9010316ab0ce1d4c4ed93ca5b023b070", + "src/lib/cluster/node-credential-refresh-grants.mjs": "3f25249f9d293c38c1c9b7e9f2bd8dcdd8aae501636ec1b9bc21082b2c6db0bc", + "src/lib/db/node-management-authority.mjs": "eb810d961e028ba4e18836024c960319aa7101367e02e6c630fab6c3e77d869a", + "scripts/ha-node-bootstrap.mjs": "0abf715c0680a2355e7930c9ec4adaeda54993a143a8687a9b784d42e89e0ddf" + } +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/checks.json b/docs/plans/evidence/kubernetes/ha-verification/checks.json new file mode 100644 index 000000000..483fb1d0a --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/checks.json @@ -0,0 +1,345 @@ +{ + "checks": [ + { + "source_sha": "ff4e43486fa98af21aee366f8c3779202e81e1d0", + "kind": "bootstrap_and_database_final", + "files_under_test": 2, + "files": 1, + "totals": { + "tests": 25, + "pass": 25, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "819387c9bfc1e5b3b9d97949eabc2c54eecc435b43dfd51a47fe4b52ad7dd582" + }, + { + "source_sha": "48aa2a284ab057c2b525d5426c6e7dd1c74a744d", + "kind": "targeted", + "files_under_test": 6, + "files": 1, + "totals": { + "tests": 50, + "pass": 50, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "e901d9329ff2f8737a8a2991f119c2c2697d5f9c9dad5c428248be3a8ce211e7" + }, + { + "source_sha": "f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0", + "kind": "targeted", + "files": 1, + "totals": { + "tests": 73, + "pass": 73, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "4913c39084d5bc1b02896629e1b8f8d0d1a5bb58c179702f217df03591fc9a6e" + }, + { + "source_sha": "4de403f9ee5411fee1c2266ef80cfd4315267a4b", + "kind": "full_unit", + "files": 395, + "totals": { + "tests": 4373, + "pass": 4339, + "fail": 24, + "cancelled": 0, + "skipped": 10 + }, + "failures": [ + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "prepareCliHopBody drops metadata and CLI-owned system but keeps official agent leftover", + "error_sha256": "1afab608e2476a88783807d547c9b239ad4ac831679731a3ed4d44812b0c31d2", + "matches_verified_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "every cli-hop system becomes the identity plus the caller body", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_verified_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop normalizes mixed caller markers to the official template", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_verified_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop drops caller system/tools markers and restamps last plus penultimate user", + "error_sha256": "7862d1144e6748d63aa6e98dabf5a9fa7d8d1582f7fbfe851d978eb15f6d5c92", + "matches_verified_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop short history only marks the last message", + "error_sha256": "81b2ee3a4bb8b1ab4f351bede4398032dc9637c683b82b146630fd6857e94c2e", + "matches_verified_baseline": true + }, + { + "file": "test/unit/cli-hop-body.test.mjs", + "test": "cli-hop uses the inbound session TTL on both message anchors", + "error_sha256": "01732f4307f58f479e9b3d8329d92a93d9c748c58028276022257d3645e77e87", + "matches_verified_baseline": true + }, + { + "file": "test/unit/convert-cache-shape.test.mjs", + "test": "openai.chat multi-turn cli-hop marks last and penultimate user", + "error_sha256": "312320875ebe3fc483f8bb570b16a93bb368278d2b6a78f71a763436e2a81b3f", + "matches_verified_baseline": true + }, + { + "file": "test/unit/ha-recovery-monitor.test.mjs", + "test": "HA monitor immediately scans 20 live-owner parents, advances its local cursor every five seconds and resumes after stop", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-monitor.test.mjs", + "test": "HA monitor leaves a connected owner alone then recovers its exact original proof and bills once after disconnection", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-monitor.test.mjs", + "test": "a real PG scan error completes only the scanner activity and logs its public code without retaining a scanner claim", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-monitor.test.mjs", + "test": "HA monitor retains original unknown execution and quota when the node is unavailable", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-monitor.test.mjs", + "test": "HA monitor retains original unknown execution and quota when the node is not_found", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-monitor.test.mjs", + "test": "HA monitor serializes runOnce and stop waits for its original in-flight read and durable settlement", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-scan.test.mjs", + "test": "scan paginates live-owner candidates and never treats old request timestamps as owner loss", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-scan.test.mjs", + "test": "scan continues after one exact recovery throws and excludes fully released settled requests", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/ha-recovery-scan.test.mjs", + "test": "scan includes already paid but unreleased reservations and excludes broken activity/epoch links", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "real PostgreSQL grants drive pinned TLS initial publication and launch without holding a transaction over node I/O", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "a lost publication acknowledgement retains the original VM operation and does not issue another command", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "failure to commit the received node receipt retains ownership and never repeats the mutation", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "a payload digest mismatch is rejected before TLS or any publication bytes", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "DRAINING refusal before issuing a publication sends no bytes and releases only that unstarted operation", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "a certificate pin mismatch sends no application bytes and retains no active VM operation", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "an extra secret field in a completed node acknowledgement is rejected and never persisted or exposed", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + }, + { + "file": "test/unit/node-management-client.test.mjs", + "test": "a lost launch acknowledgement preserves the exact launch operation after Docker start", + "error_sha256": "b80aa5a1ce275b6e57778f3d885c22a44b929c3c0d8ffc2447f8a48a74e9762c", + "matches_verified_baseline": false + } + ], + "log_sha256": "f8e4aae9e978bdc72a40df02adc395827c4f69c7932e73d0a6a2fd08af44b5bc" + }, + { + "source_sha": "4de403f9ee5411fee1c2266ef80cfd4315267a4b", + "kind": "full_e2e", + "files": 35, + "totals": { + "tests": 135, + "pass": 117, + "fail": 17, + "cancelled": 0, + "skipped": 1 + }, + "failures": [ + { + "file": "test/e2e/account-pool-failover.e2e.test.mjs", + "test": "account1 quota exhaustion rotates to account2 and commits sticky final account", + "error_sha256": "a25cb59468e4decc45b1d8b3a66b0a4fbd2195af9f498da39aa83e92169825b5", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/auto-mode.e2e.test.mjs", + "test": "classifier terminal/error/cancel/concurrency boundaries through actual ELF", + "error_sha256": "84e06146cf59485c3b46e05070f6a9a184ecb010a57ef8ab1b62a3ebae90c36a", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/completions.e2e.test.mjs", + "test": "POST /v1/completions old OpenAI prompt format", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/credentials.e2e.test.mjs", + "test": "reset destroys home and recreates the same slot without oauth", + "error_sha256": "dfd84d60931072a864eb94f48ce13d4223e963ae919898236f1859bedc8d917e", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/crs-default.e2e.test.mjs", + "test": "default POST /v1/messages uses Go worker pool, not CLI", + "error_sha256": "fdb99383e1e333346a42b82cbf8092357575350f6a03f1ec297c9cee064b1e6d", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/errors.e2e.test.mjs", + "test": "unknown model is rejected without hop", + "error_sha256": "6ea819807eaee98e6a4c38817c9f011fb426e6a5ff499fa441d8f7bb07371461", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/health.e2e.test.mjs", + "test": "harness /health 200 and honest capabilities", + "error_sha256": "c23279127cad70a092d0cc7bb6ac400e05aff03e4614d1eaaa40252878c221c2", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes unofficial: Go relay keeps persona/tools and never launches CLI", + "error_sha256": "97ae4156650fa0de2f613aa7753666781545cb2fe7818350a6df904ab696cdbc", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/hermes.e2e.test.mjs", + "test": "Hermes system blob without UA still classifies as hermes", + "error_sha256": "a2c6319267f1b752b63d96a41e15b1a5133952e88f0c145bc96dda53fd31cbe5", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS default keeps thinking + max_tokens + temperature in official body", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "CRS keeps a long caller system in the structured official prompt", + "error_sha256": "ba0ac5e90ea008839723c328e8b172228820837ab895e425d2190c1aa5caa708", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/hop-meta.e2e.test.mjs", + "test": "legacy CLI selector cannot drop HTTP request parameters", + "error_sha256": "ebe0dbf58f38b7e1df9e2c55463c11e712927dcbc047102419ad10a8d38f1420", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP worker preserves thinking, sampling and max_tokens", + "error_sha256": "8353154a58e84007d91c2d217c516dc771e359ddb43d527c7db6734ab5cff6ca", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/messages.e2e.test.mjs", + "test": "HTTP path preserves long system without CLI argv truncation", + "error_sha256": "085c13ab1e54fb324bce5520b12006fbf7f84b4a69a1c5cf9e09727b5b33da99", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses non-stream shape: object + output_text + usage", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses stream includes [DONE]", + "error_sha256": "0be25feaedaa697c09835d15c8cfdae694de2710330cadfadcbdfa146625a099", + "matches_verified_baseline": true + }, + { + "file": "test/e2e/openai-responses.e2e.test.mjs", + "test": "POST /v1/responses tools → function_call in output", + "error_sha256": "a7c7c455e65d737b0a872d5d7b4cbe5a5c2372e3117e3f361d6dbe8f223c1392", + "matches_verified_baseline": true + } + ], + "log_sha256": "c6a7bb44205cbc7a5d87fad73d37f508e2bbccc77f3fb5c130e19dd49a3d4098" + }, + { + "source_sha": "0d9c434866532cfccd952ac990881400ed364dd1", + "kind": "isolated_runner_pg_fixture_recheck", + "files": 3, + "totals": { + "tests": 18, + "pass": 18, + "fail": 0, + "cancelled": 0, + "skipped": 0 + }, + "failures": [], + "log_sha256": "b65a5919348845238e561354be2083d032fe522005725e04b189748df2073fb3" + } + ], + "note": "Full suites were not all green. 17 new unit failures were missing automatic test PG initialization; the runner fix was verified separately, not represented as a new full-suite pass.", + "final_static_checks": { + "source_sha": "ff4e43486fa98af21aee366f8c3779202e81e1d0", + "biome_files": 779, + "biome_pass": true, + "changed_mjs_syntax_pass": true + } +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-crash.json b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-crash.json new file mode 100644 index 000000000..49e531be7 --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-crash.json @@ -0,0 +1,71 @@ +{ + "source_sha": "f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0", + "boundary": "one task control-plane container killed via CRI timeout0; original synthetic node kept alive", + "checks": [ + { + "name": "owner_killed_but_unfinished_original_node_keeps_capacity", + "passed": true, + "held": { + "keys": 1, + "executions": 1 + } + }, + { + "name": "original_completed_receipt_recovers_capacity_and_bills_once", + "passed": true, + "ledger": [ + { + "outcome": "charged", + "command_id": "0f991d2e-2013-4f05-bda1-54087f4ed1e9", + "tokens_in": "5", + "tokens_out": "7" + } + ], + "held": { + "keys": 0, + "executions": 0 + } + }, + { + "name": "other_replicas_continue_and_new_request_succeeds", + "passed": true + } + ], + "target": { + "name": "vm2api-5ccbfd86f-68772", + "uid": "2ffcdd02-5520-4332-8934-0d4529e0e0f2", + "node": "k3d-vm2api-648-01a15dc9-server-0", + "containerId": "61b909fc65a6a8a73f331685d2b728279a4dd8cbdedcec016cc9b79b40acb2d2", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e", + "restartsBefore": 1 + }, + "commandId": "0f991d2e-2013-4f05-bda1-54087f4ed1e9", + "originalNodeBinding": { + "kind": "kernel", + "vmId": "command-vm", + "fence": { + "executor": { + "kind": "container", + "started_at": "2026-10-22T00:00:00.000000001Z", + "container_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "node_boot_id": "6d7ddbe3-d683-427d-bc46-cde2fc1e2e5a", + "docker_daemon_id": "synthetic-daemon" + }, + "generation": "1", + "incarnation": "78b28d80f6a2ec29ad81ee8869a5799c" + }, + "nodeId": "command-node", + "protocol": "KSG2" + }, + "termination": { + "containerID": "containerd://61b909fc65a6a8a73f331685d2b728279a4dd8cbdedcec016cc9b79b40acb2d2", + "exitCode": 137, + "finishedAt": "2026-10-21T18:26:30Z", + "reason": "Error", + "startedAt": "2026-10-21T18:25:46Z" + }, + "clientResult": { + "error": "TypeError" + }, + "passed": true +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-hpa.json b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-hpa.json new file mode 100644 index 000000000..df8d17173 --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-hpa.json @@ -0,0 +1,612 @@ +{ + "source_sha": "f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0", + "test_override": { + "minReplicas": 2, + "maxReplicas": 3, + "cpu_percent": 1 + }, + "samples": [ + { + "status": { + "currentMetrics": null, + "desiredReplicas": 0 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "currentMetrics": null, + "desiredReplicas": 0 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "currentMetrics": null, + "desiredReplicas": 0 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "currentMetrics": null, + "desiredReplicas": 0 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "currentMetrics": null, + "desiredReplicas": 0 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "currentMetrics": null, + "desiredReplicas": 0 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "currentMetrics": null, + "desiredReplicas": 0 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "recommended size matches current size", + "reason": "ReadyForNewScale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the desired count is within the acceptable range", + "reason": "DesiredWithinRange", + "status": "False", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 1, + "averageValue": "2m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "recommended size matches current size", + "reason": "ReadyForNewScale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the desired count is within the acceptable range", + "reason": "DesiredWithinRange", + "status": "False", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 1, + "averageValue": "2m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "recommended size matches current size", + "reason": "ReadyForNewScale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the desired count is within the acceptable range", + "reason": "DesiredWithinRange", + "status": "False", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 1, + "averageValue": "2m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "recommended size matches current size", + "reason": "ReadyForNewScale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the desired count is within the acceptable range", + "reason": "DesiredWithinRange", + "status": "False", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 1, + "averageValue": "2m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "recommended size matches current size", + "reason": "ReadyForNewScale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the desired count is within the acceptable range", + "reason": "DesiredWithinRange", + "status": "False", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 1, + "averageValue": "2m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "recommended size matches current size", + "reason": "ReadyForNewScale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the desired count is within the acceptable range", + "reason": "DesiredWithinRange", + "status": "False", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 1, + "averageValue": "2m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "recommended size matches current size", + "reason": "ReadyForNewScale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the desired count is within the acceptable range", + "reason": "DesiredWithinRange", + "status": "False", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 1, + "averageValue": "2m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "replicas": 2, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA controller was able to update the target scale to 3", + "reason": "SucceededRescale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:59Z", + "message": "the desired replica count is more than the maximum replica count", + "reason": "TooManyReplicas", + "status": "True", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 11, + "averageValue": "28m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 3, + "lastScaleTime": "2026-10-21T18:28:59Z" + }, + "replicas": 3, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA controller was able to update the target scale to 3", + "reason": "SucceededRescale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:59Z", + "message": "the desired replica count is more than the maximum replica count", + "reason": "TooManyReplicas", + "status": "True", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 11, + "averageValue": "28m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 3, + "lastScaleTime": "2026-10-21T18:28:59Z" + }, + "replicas": 3, + "ready": 2, + "available": 2 + }, + { + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA controller was able to update the target scale to 3", + "reason": "SucceededRescale", + "status": "True", + "type": "AbleToScale" + }, + { + "lastTransitionTime": "2026-10-21T18:28:44Z", + "message": "the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request)", + "reason": "ValidMetricFound", + "status": "True", + "type": "ScalingActive" + }, + { + "lastTransitionTime": "2026-10-21T18:28:59Z", + "message": "the desired replica count is more than the maximum replica count", + "reason": "TooManyReplicas", + "status": "True", + "type": "ScalingLimited" + } + ], + "currentMetrics": [ + { + "resource": { + "current": { + "averageUtilization": 11, + "averageValue": "28m" + }, + "name": "cpu" + }, + "type": "Resource" + } + ], + "currentReplicas": 2, + "desiredReplicas": 3, + "lastScaleTime": "2026-10-21T18:28:59Z" + }, + "replicas": 3, + "ready": 3, + "available": 3 + } + ], + "requests": [ + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + }, + { + "status": 200 + } + ], + "passed": true +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-lifecycle.json b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-lifecycle.json new file mode 100644 index 000000000..8d7a1ceda --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-lifecycle.json @@ -0,0 +1,348 @@ +{ + "source_sha": "f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0", + "namespace": "vm2api-648-ha-01a15dc9", + "checks": [ + { + "name": "scale_two_to_three", + "passed": true, + "target": 3, + "requests": 19, + "minAvailable": 2, + "pods": [ + { + "name": "vm2api-5cffcbb4c7-f42qm", + "uid": "ba415d10-1a01-475d-8bff-cbc185a0456e", + "node": "k3d-vm2api-648-01a15dc9-agent-0", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + }, + { + "name": "vm2api-5cffcbb4c7-fwrgw", + "uid": "36a30ca6-278d-4451-bea8-951dc7be916f", + "node": "k3d-vm2api-648-01a15dc9-agent-1", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + }, + { + "name": "vm2api-5cffcbb4c7-sx6tk", + "uid": "2c8e22da-76e7-44f9-b8eb-b3016311a67e", + "node": "k3d-vm2api-648-01a15dc9-server-0", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + } + ] + }, + { + "name": "scale_three_to_two", + "passed": true, + "target": 2, + "requests": 1, + "minAvailable": 2, + "pods": [ + { + "name": "vm2api-5cffcbb4c7-f42qm", + "uid": "ba415d10-1a01-475d-8bff-cbc185a0456e", + "node": "k3d-vm2api-648-01a15dc9-agent-0", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + }, + { + "name": "vm2api-5cffcbb4c7-sx6tk", + "uid": "2c8e22da-76e7-44f9-b8eb-b3016311a67e", + "node": "k3d-vm2api-648-01a15dc9-server-0", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + } + ] + }, + { + "name": "rolling_restart", + "passed": true, + "target": 2, + "requests": 25, + "minAvailable": 2, + "pods": [ + { + "name": "vm2api-5ccbfd86f-68772", + "uid": "2ffcdd02-5520-4332-8934-0d4529e0e0f2", + "node": "k3d-vm2api-648-01a15dc9-server-0", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + }, + { + "name": "vm2api-5ccbfd86f-9wjjb", + "uid": "b1be4ded-b032-467d-b803-4c374a140b2f", + "node": "k3d-vm2api-648-01a15dc9-agent-1", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + } + ] + } + ], + "requests": [ + { + "status": 200, + "stream": false, + "ok": true, + "ms": 170 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 142 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 141 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 143 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 352 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 124 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 146 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 153 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 145 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 128 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 125 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 123 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 108 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 98 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 112 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 105 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 117 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 108 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 102 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 119 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 123 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 110 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 126 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 108 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 107 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 114 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 105 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 121 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 111 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 99 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 103 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 113 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 106 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 105 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 107 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 120 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 112 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 113 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 121 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 164 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 124 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 656 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 128 + }, + { + "status": 200, + "stream": true, + "ok": true, + "ms": 101 + }, + { + "status": 200, + "stream": false, + "ok": true, + "ms": 110 + } + ], + "passed": true +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-smoke.json b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-smoke.json new file mode 100644 index 000000000..f05a1d714 --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-smoke.json @@ -0,0 +1,89 @@ +{ + "source_sha": "f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0", + "namespace": "vm2api-648-ha-01a15dc9", + "boundary": "real two-node Kubernetes gateways, real PG/TLS Go node, synthetic kernel/provider", + "checks": [ + { + "name": "two_ready_pods_on_distinct_nodes", + "passed": true, + "nodes": [ + "k3d-vm2api-648-01a15dc9-server-0", + "k3d-vm2api-648-01a15dc9-agent-1" + ] + }, + { + "name": "cross_pod_panel_session", + "passed": true + }, + { + "name": "both_pods_dispatch_concurrently", + "passed": true, + "held": { + "keys": 2, + "executions": 2, + "turns": 2 + } + }, + { + "name": "global_account_vm_cap_and_cancelled_waiter_release", + "passed": true, + "kernelActive": 2, + "held": { + "keys": 2, + "executions": 2, + "turns": 2 + } + }, + { + "name": "http_sse_original_node_completion_and_billing", + "passed": true, + "ledger": [ + { + "outcome": "charged", + "n": 2 + } + ], + "held": { + "keys": 0, + "executions": 0, + "turns": 0 + }, + "kernelPeak": 2 + }, + { + "name": "global_key_concurrency_across_pods", + "passed": true, + "deniedStatus": 429 + }, + { + "name": "cross_pod_routing_refresh", + "passed": true + }, + { + "name": "unsupported_management_refuses_before_host_io", + "passed": true + }, + { + "name": "websocket_claude_responses_rejection", + "passed": true, + "inference_success": false + } + ], + "pods": [ + { + "name": "vm2api-5ccbfd86f-68772", + "uid": "2ffcdd02-5520-4332-8934-0d4529e0e0f2", + "node": "k3d-vm2api-648-01a15dc9-server-0", + "image": "docker.io/library/vm2api:kubernetes-ha-f53e005f", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + }, + { + "name": "vm2api-5ccbfd86f-9wjjb", + "uid": "b1be4ded-b032-467d-b803-4c374a140b2f", + "node": "k3d-vm2api-648-01a15dc9-agent-1", + "image": "docker.io/library/vm2api:kubernetes-ha-f53e005f", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + } + ], + "passed": true +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-ws-drain.json b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-ws-drain.json new file mode 100644 index 000000000..c5fc0c514 --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-ha-ws-drain.json @@ -0,0 +1,45 @@ +{ + "source_sha": "f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0", + "scope": "WebSocket upgrade, unsupported Claude Responses error, graceful idle close and reconnect; no successful model WS inference", + "oldPods": [ + { + "uid": "3cc88f75-d577-44e2-ab29-cf000efc682e", + "name": "vm2api-6b86c684f-h4lbn", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + }, + { + "uid": "0efe76ee-49c8-4f02-a7f3-85954a0cd6a9", + "name": "vm2api-6b86c684f-khv2s", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + } + ], + "closed": { + "code": 1001, + "reason": "Control plane is draining" + }, + "reconnectAttempts": [ + { + "ok": false, + "code": "Error" + }, + { + "ok": true + } + ], + "newPods": [ + { + "uid": "2a9ae89d-8d7d-49b5-9699-44c9effbbb03", + "name": "vm2api-7cd4b59c76-9wvbx", + "node": "k3d-vm2api-648-01a15dc9-agent-0", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + }, + { + "uid": "0a967cf7-0f2e-4a03-bad8-e79607f98dec", + "name": "vm2api-7cd4b59c76-trfn2", + "node": "k3d-vm2api-648-01a15dc9-agent-1", + "imageID": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e" + } + ], + "reconnected": true, + "passed": true +} diff --git a/docs/plans/evidence/kubernetes/ha-verification/f53e005f-runtime-identity.json b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-runtime-identity.json new file mode 100644 index 000000000..2c774da59 --- /dev/null +++ b/docs/plans/evidence/kubernetes/ha-verification/f53e005f-runtime-identity.json @@ -0,0 +1,18 @@ +{ + "source_sha": "f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0", + "docker_image_config_id": "sha256:c4dc3bbbae262b271a1c754c461fab9111855e45d18988586c032aa6ca596c5d", + "containerd_image_id": "sha256:f2ce099f07273f1a187c01cfbbaffaee24f2cad57239b7627da917d17d5dcb3e", + "rootfs_diff_ids_match": true, + "rootfs_layer_count": 24, + "rootfs_diff_ids_sha256": "18e07ea1a8b123baf2fab6907e8e453558e5192931ccbd1980f1240214bbed32", + "runtime_pod": "vm2api-5ccbfd86f-68772", + "runtime_files_sha256": { + "src/server.mjs": "3c87b8534a0486030ed2cca85885c0f1dd058551bbfb7d14bf3914bcc5a60897", + "src/lib/core/ha-recovery-monitor.mjs": "a787e1b98be6af95d85d8bd265e578fecd9ff68c43ae4eb315dcd4e3cba0d440", + "src/lib/core/ha-request-billing.mjs": "4411f6e37bfad824c5de49d9b0cf0c2f507848ef104c7f45e8ceb492ab0ce37e", + "src/lib/db/global-maintenance.mjs": "d2e928ef142faf0cb1666dac1275c957af0c0576739314911bce38e650798051", + "src/lib/cluster/node-credential-refresh-grants.mjs": "3f25249f9d293c38c1c9b7e9f2bd8dcdd8aae501636ec1b9bc21082b2c6db0bc", + "src/lib/db/node-management-authority.mjs": "eb810d961e028ba4e18836024c960319aa7101367e02e6c630fab6c3e77d869a", + "scripts/ha-node-bootstrap.mjs": "6827e12e9064fd8e5e702cf55a66e1b1eb6725df68ff8d21e2897dea79be8f4f" + } +} diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-48aa2a28.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-48aa2a28.md new file mode 100644 index 000000000..dfd3656d0 --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-48aa2a28.md @@ -0,0 +1,20 @@ +发现 **1 个新增 P2**，未发现新增 P0/P1。 + +1. **P2：大小写混合的业务 schema 被误判为不存在。** + 位置：[src/lib/db/database.mjs:273](https://github.com/dofastted/vm2api/blob/48aa2a284ab057c2b525d5426c6e7dd1c74a744d/src/lib/db/database.mjs#L273)。配置校验和数据库构造函数均允许大小写混合名称，迁移也使用双引号保留大小写。但这里将原始名称传给 `to_regnamespace`，该函数会将未加双引号的名称折叠为小写。 + 触发条件：已迁移 schema 为 `review_Mixed`，且不存在 `review_mixed`。两个 CLI 命令均经 `main → createDatabase → verifyCurrentMigrations`，在私有初始化或节点动作前错误返回 `database_migrations_not_current`，导致这类已有部署无法初始化或创建槽位。 + 最小修复：第一个参数改为 `quoteIdentifier(db.schema)`，或通过 `pg_namespace.nspname=$1` 精确查询。 + 主会话复现建议：在隔离 PostgreSQL 17 中，经授权迁移流程创建并升级 `review_Mixed`，配置有效 reader 后执行 `trusted-init`。预期成功；按 HEAD 静态路径会被前置校验拒绝。现有新增测试使用小写 schema，未覆盖此条件。 + +上一轮两项 P2 的原触发路径已通过静态核对： + +- reserved VM ID 在 [scripts/ha-node-bootstrap.mjs:113](https://github.com/dofastted/vm2api/blob/48aa2a284ab057c2b525d5426c6e7dd1c74a744d/scripts/ha-node-bootstrap.mjs#L113) 被拒绝，早于数据库、grant 和节点动作。 +- CLI 使用 `verifyMigrations: true`。首次核验早于私有协调 schema 初始化，bootstrap 活动内再次核验。CLI 不再应用业务迁移，默认 server 自动迁移分支保留。 + +已读取全部 4 个文件增量、相关调用链与测试。已核对事务只读设置、迁移锁与维护锁顺序、已有 schema 拒绝路径、并发路径、回滚及连接池清理；除上述问题，未发现其他可证新增 P0/P1/P2。 + +审查 HEAD：`48aa2a284ab057c2b525d5426c6e7dd1c74a744d` +增量基线：`f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0` +首尾 HEAD 一致，工作区干净。 + +本轮未运行测试或动态复现。提供的 61/61、k3d 和 ELF 验证结果未独立复验；明示 unsupported 范围未计为缺陷。本结论不构成发布批准。 diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-a6b67f3b.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-a6b67f3b.md new file mode 100644 index 000000000..e32d5a22f --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-a6b67f3b.md @@ -0,0 +1,56 @@ +新增 **1 项 P1、2 项 P2**。未发现其他可证的新增 P0/P1/P2。以下均为静态源码结论，合成复现交主会话执行。 + +**1. P1：允许的 routing 更新会重新启动 HA 已禁用的后台探测，产生持久未知任务。** + +位置：[src/server.mjs:1794](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/server.mjs#L1794)、[src/lib/admin/routing-runtime.mjs:169](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/lib/admin/routing-runtime.mjs#L169)、[src/lib/oauth/usage-probe-monitor.mjs:415](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/lib/oauth/usage-probe-monitor.mjs#L415)。 + +触发条件：PG 原配置保留 `usage_probe.enabled=true`；存在具备凭据标记、代理元数据且需要用量探测的普通 Claude VM；HA 启动后保存一次允许的 `pool` 等 routing 配置。 + +调用链： + +- HA 仅在启动回调中跳过 monitor `start()`。 +- `SharedRoutingConfig.patch/refresh → applySharedRouting` 仍调用 health、usage monitor 的 `setConfig()`。 +- `setConfig()` 默认 `restart=true`，重新安装定时器。 +- usage 定时器取得共享后台任务声明，再进入 `server.mjs:634 → buildProbeOne`。 +- `buildProbeOne` 构造的旧执行上下文没有 capability。`callWorkerGet → requireNodeCapability` 抛出 `slotgate_capability_required`。 +- 该异常没有 `taskNotStarted/taskTerminal`，因此 [shared-background-tasks.mjs:131](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/lib/core/shared-background-tasks.mjs#L131) 将任务和 activity 标为未知。 + +影响：一次正常配置保存可以留下 `finished_at IS NULL` 的后台任务，持续阻止全局维护。该任务不属于请求恢复服务可处理的执行图。 + +最小修复：HA 下配置应用不得重启未支持的 monitor；同时在按需 `onUsageProbe` 入口、取得后台声明之前拒绝探测。 + +合成复现：使用隔离 PG 和上述合成 VM，保留实际 routing、usage monitor、`buildProbeOne` 与 capability guard；保存 `pool` 配置后推进测试时钟。断言旧 worker 调用为零，并检查 `usage-probe:` 是否留下未知任务及未完成 activity。 + +**2. P2：HTTP Claude Responses 从原有 400 回归为 500。** + +位置：[src/server.mjs:1295](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/server.mjs#L1295)、[src/lib/protocol/handle-protocol.mjs:562](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/lib/protocol/handle-protocol.mjs#L562)。 + +`POST /v1/responses` 和 `/responses` 被 HA 路由允许，但不在建立请求 context 的 HTTP 路径集合内。外层已登记 `requestActivities`，所以 `dispatchProtocol` 的再次包装直接复用外层，不会补建 context。 + +协议入口随后无条件调用 `currentHaRequestContext()`，抛出 `ha_request_context_required`。请求在鉴权、模型检查和原有 `handle-protocol.mjs:932` 的 `400 protocol_not_allowed` 分支之前返回 500。 + +最小修复：区分“需要协议 context”和“允许执行恢复”的路径；为 Responses 建立不可恢复的 context，保留原有拒绝行为。 + +合成复现：通过实际 server HTTP 包装器发送合成 Claude Responses 请求，不替换 `currentHaRequestContext`。预期为原有 400；当前调用链为 500。现有 [协议 fixture:16](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/test/fixtures/ha-protocol-boundaries.mjs#L16) 固定替换了该函数，因此未覆盖此问题。 + +**3. P2：启动清理会删除恢复服务必需的原 attempt，导致已完成作业无法恢复。** + +位置：[src/lib/core/ha-request-recovery.mjs:103](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/lib/core/ha-request-recovery.mjs#L103)、[src/lib/db/repos/request-attempts-repo.mjs:60](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/lib/db/repos/request-attempts-repo.mjs#L60)、[src/lib/admin/routing-runtime.mjs:843](https://github.com/dofastted/vm2api/blob/a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f/src/lib/admin/routing-runtime.mjs#L843)。 + +触发条件：HA 孤儿请求超过日志保留期，仍持有执行准入或 API key 预约；随后另一 Pod 启动。 + +启动时 `initPoolRuntime → attemptsRepo.cleanup` 只按 `started_at` 删除 `request_attempts`，没有保护待恢复请求。新增的执行历史保护只作用于 `execution_admissions`，无法保留 attempt。 + +恢复服务要求每个 job 恰好匹配一个原 attempt。删除后，`recover()` 在读取节点完成证明之前返回 `request_job_binding_unproven`。即使原节点已有可信 completed，执行准入、席位和 session turn 仍无法通过当前恢复服务释放。 + +最小修复：attempt 清理增加待恢复执行图和未释放 API key 预约的保留条件，并与恢复事务协调清理并发。 + +合成复现：沿用 `ha-request-recovery.test.mjs` 的真实关联图，将 attempt 时间设为保留期之前；断开原 owner，执行实际 `cleanup()`，再提供原 command/Fence 的 completed。当前结果应为 `request_job_binding_unproven`，资源继续占用。 + +**覆盖与未验证** + +审查 SHA：`a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f`；基线：`e9076922a4af1f43840e6f9db8381790c809ee21`。审查前后 HEAD 一致，工作区干净；95 文件增量及必要调用链已读取，`git diff --check` 无输出。 + +覆盖了授权与回执链、私有 authority/角色/epoch/锁序、Go staging 与固定路由、capability 和 PG job 绑定、HA 启动与后台入口、019/020 请求关联及恢复、断连终态与历史 receipt、backup 同 schema 外键清理及外部依赖保护。 + +未运行测试、服务或任何外部环境操作。并发时序、TLS/文件持久化和运行制品行为均未动态验证。未将其他 worktree 的 refresh、billing recovery、notstarted seal 计划记为本增量缺陷。部署仍为 `replicas: 1`、`Recreate`。**本审查不批准发布。** diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-ceb27752.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-ceb27752.md new file mode 100644 index 000000000..807f73f16 --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-ceb27752.md @@ -0,0 +1,31 @@ +发现 **2 项新增 P2**，未发现可证的新增 P0/P1。 + +1. **P2：明确未发送的刷新 grant 会永久占用 generation** + + 位置：[src/lib/db/node-credential-refresh-authority.mjs:51](https://github.com/dofastted/vm2api/blob/ceb2775221fb73549e542855356073bacf8e54a8/src/lib/db/node-credential-refresh-authority.mjs#L51)、[src/lib/cluster/node-credential-refresh-grants.mjs:158](https://github.com/dofastted/vm2api/blob/ceb2775221fb73549e542855356073bacf8e54a8/src/lib/cluster/node-credential-refresh-grants.mjs#L158)。 + + 触发条件：`issue()` 已提交，但刷新连接在 TLS 请求发送前失败。客户端将其标为 `vmOperationNotStarted`，共享刷新随后完成后台任务并释放 VM operation。释放仅设置 grant 的 `revoked_at`，其 `receipt` 仍为 NULL。 + + `refresh_pending_generation` 的唯一条件只检查 `receipt IS NULL`，因此该已撤销记录仍占用 generation。下次合法刷新会得到 `node_refresh_conflict`。该错误又不属于明确未开始的分类，随后会将新的后台任务和 VM operation 标为未知，导致该账号持续无法刷新。 + + 最小修复：在节点锁下，为明确未发送的 grant 记录终态并解除其 generation 占用；发送后结果未知的 grant 继续保留占用。 + + 合成复现：使用现有 PG 刷新 fixture，让第一次 `grants.refresh()` 连接关闭的本地端口。确认其返回未开始、后台任务结束、VM operation 释放。保持 generation 不变，第二次使用有效节点连接刷新；当前代码会报 `node_refresh_conflict`，而非取得新授权。 + +2. **P2：派发前取消请求会因未绑定的 command ID 留下 API key 预约** + + 位置：[src/lib/protocol/handle-protocol.mjs:231](https://github.com/dofastted/vm2api/blob/ceb2775221fb73549e542855356073bacf8e54a8/src/lib/protocol/handle-protocol.mjs#L231)、[src/lib/core/ha-request-billing.mjs:315](https://github.com/dofastted/vm2api/blob/ceb2775221fb73549e542855356073bacf8e54a8/src/lib/core/ha-request-billing.mjs#L315)。 + + 触发条件：客户端在 `prepareRust()` 的节点健康检查期间断开。该健康检查未传递取消信号，仍可返回成功。随后 `streamGoWorker()` 在调用 `onExecutionStart` 前发现取消，返回带随机 `requestId` 的 `started:false / ended:true / not_started` 结果，此时没有持久 job。 + + 新计费逻辑仅依据非空 `requestId` 调用 `selectHaBillableCommand()`，因此抛出 `ha_billable_command_mismatch`，无法进入已有的 `not_dispatched` 分支。异常设置 `billingFailed`，外层 finally 保留 API key 预约。原控制面仍连接时，其他副本也不能恢复该预约。 + + 最小修复：先在 parent 锁下确认执行图没有 job，再记录 `not_dispatched`；存在 job 时继续校验原 command，不能仅依据 `started:false` 释放。 + + 合成复现：暂停正常 Messages 请求在 `callAttempt` 内的节点健康响应，断开客户端，再返回健康成功。确认节点未收到推理、`execution_jobs` 为零。当前代码会出现上述异常，且 `api_key_admissions.released_at` 保持 NULL。现有 `not-started` fixture 将 `requestId` 固定为 NULL，未覆盖此路径。 + +审查对象为 `ceb2775221fb73549e542855356073bacf8e54a8`，基线为 `a6b67f3b664cb2616cd43aa053f20bcc8fac2f2f`。开始和结束时 HEAD 均一致，工作区干净。已通读全部 79 个文件的增量及必要原调用链，`git diff --check` 无输出。 + +上轮三项均见对应源码修正：HA 路由更新禁止重启旧 monitor；Responses 建立不可恢复 context；attempt 清理保护待处理执行图并取得 parent 锁。以上仅作静态复核。 + +按要求未运行测试、服务或动态复现，未修改文件，未使用子 agent、网络或外部环境。真实 provider、现有内核读取新 AT、多节点部署行为均未验证。未将声明 unsupported 的路径、新 CLI bootstrap 或未交付的自动 recovery monitor 计为已完成功能。本次审查不构成发布批准。 diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-f53e005f.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-f53e005f.md new file mode 100644 index 000000000..b6de287aa --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-f53e005f.md @@ -0,0 +1,17 @@ +发现 **2 项新增 P2**，未发现可证的新增 P0/P1。 + +1. **P2：CLI 接受保留 VM ID，创建后导致整个 HA catalog 不可用。** + [scripts/ha-node-bootstrap.mjs:111](https://github.com/dofastted/vm2api/blob/f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0/scripts/ha-node-bootstrap.mjs#L111) 仅检查字符格式，允许 `active`、`create`、`*-chat` 等现有保留名称。三个节点操作成功后，这些 ID 会写入 `vms`。随后 [ha-vm-catalog.mjs:26](https://github.com/dofastted/vm2api/blob/f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0/src/lib/vm/ha-vm-catalog.mjs#L26) 因 `isValidVmId()` 拒绝记录而清空 catalog；[server.mjs:1373](https://github.com/dofastted/vm2api/blob/f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0/src/server.mjs#L1373) 每次业务请求都会刷新，因此影响已有正常槽位的请求。 + 最小修复：配置校验同时调用现有 `isValidVmId()`，在任何节点操作前拒绝。 + 主会话合成复现：将 bootstrap fixture 的 `vm.id` 改为 `active`，完成 `slot-create` 后调用 `HaVmCatalog.refresh()`，检查 `ha_vm_catalog_unavailable`。修复后应在签发 grant 前拒绝，且节点调用数为零。 + +2. **P2：`trusted-init` 隐式迁移业务 schema，超出 private DDL 范围。** + [scripts/ha-node-bootstrap.mjs:360](https://github.com/dofastted/vm2api/blob/f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0/scripts/ha-node-bootstrap.mjs#L360) 调用 `createDatabase()`，经 [database.mjs:277](https://github.com/dofastted/vm2api/blob/f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0/src/lib/db/database.mjs#L277) 执行并提交全部待应用业务迁移。reader 角色检查发生在此后。目标 schema 落后时，即使命令随后返回 `bootstrap_reader_invalid`，业务表和迁移记录也已经改变；不存在的 schema 还会被自动创建。 + 最小修复：CLI 使用不自动迁移的连接初始化，只读核验业务迁移版本，不满足前提则拒绝，再执行授权的 private DDL。 + 主会话合成复现：在任务专用 DB 建立仅应用到 `020` 的 schema，配置不存在的 reader，执行 `trusted-init`；检查失败后 `021` 及其业务字段是否已提交。 + +已完整读取 27 个文件的增量，并核对相关 grant、私有 SQL、节点 reader 锁、owner lifetime、epoch、计费父锁、扫描恢复及 server/Kustomize 调用链。上轮两项 P2 的修复约束在源码中可见，其余审查范围未发现可证新增 P0/P1/P2。 + +审查 SHA：`f53e005f4794782b0a8e0b4bb152f68d4a3a7aa0`；基线：`ceb2775221fb73549e542855356073bacf8e54a8`。开始与结束时工作区均干净。 + +以上为静态结论，合成复现未执行。未修改文件、运行测试或服务，也未访问网络、数据库、容器或其他 worktree。新镜像 K8s keepalive 复验及实际 provider RT 刷新仍未验证。本审查不构成发布批准。 diff --git a/docs/plans/evidence/kubernetes/review-progress/independent-review-ff4e4348.md b/docs/plans/evidence/kubernetes/review-progress/independent-review-ff4e4348.md new file mode 100644 index 000000000..994f77c3b --- /dev/null +++ b/docs/plans/evidence/kubernetes/review-progress/independent-review-ff4e4348.md @@ -0,0 +1,13 @@ +结论：原 P2 在源码层面已修复；本次增量未发现新增可证的 P0/P1/P2。 + +- 审查 SHA：`ff4e43486fa98af21aee366f8c3779202e81e1d0` +- 基线：`48aa2a284ab057c2b525d5426c6e7dd1c74a744d` +- 状态：审查前后 HEAD 一致，工作区干净。 + +原缺陷触发条件是：混合大小写 schema 已正确迁移，且不存在对应小写 namespace。[database.mjs:273](https://github.com/dofastted/vm2api/blob/ff4e43486fa98af21aee366f8c3779202e81e1d0/src/lib/db/database.mjs#L273) 现在将 `quoteIdentifier(db.schema)` 传给 `to_regnamespace`，保留标识符大小写，消除了原先错误返回 `database_migrations_not_current` 的原因。 + +新增[回归规格:222](https://github.com/dofastted/vm2api/blob/ff4e43486fa98af21aee366f8c3779202e81e1d0/test/unit/ha-node-bootstrap.test.mjs#L222) 使用 `_Mixed` schema，前后断言小写 namespace 不存在，依次断言两个 CLI 成功及 VM 记录写入。Git 历史确认 `9ae50abd…` 先单独增加规格，随后 `ff4e4348…` 仅修改上述一行源码。 + +覆盖：已读取两文件全部增量，并核对标识符处理、迁移核验事务、两个 CLI 入口，以及测试初始化、隔离和清理调用链。 + +未验证：未运行测试；修复前失败结果和所述固定提交归档的 25/25 结果未独立复验。全程仅在本 worktree 内只读检查，未修改文件、创建子 agent 或访问外部环境。本审查不构成发布批准。 From b8806d9c8853567082d073334c0ecbe9cd15085a Mon Sep 17 00:00:00 2001 From: cedar-cli Date: Thu, 22 Oct 2026 09:29:38 +0000 Subject: [PATCH 469/473] feat: official Claude Code and Codex passthrough (#674) * feat(protocol): official Claude Code passthrough for cli-hop Co-authored-by: ted * feat(codex): add official Codex Responses passthrough --------- Co-authored-by: cedar Co-authored-by: ted --- CHANGELOG.md | 5 + docs/PROTOCOL.md | 4 +- src/lib/protocol/codex-convert.mjs | 24 +++- src/lib/protocol/codex-route.mjs | 10 ++ src/lib/protocol/handle-codex.mjs | 9 +- src/lib/protocol/handle-protocol.mjs | 64 +++++---- src/lib/protocol/outbound-attempt.mjs | 46 +++++++ src/lib/vm/slot-engine.mjs | 6 +- test/unit/codex-convert.test.mjs | 35 +++++ test/unit/codex-route.test.mjs | 9 ++ test/unit/handle-codex.test.mjs | 57 ++++++++ test/unit/official-passthrough.test.mjs | 151 ++++++++++++++++++++++ web/src/features/settings/openai-pane.tsx | 15 +++ 13 files changed, 405 insertions(+), 30 deletions(-) create mode 100644 test/unit/official-passthrough.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 51047800a..20f0e4c16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 未发布 + +- 新增官方 Codex 透传：开启 `codex.official_passthrough` 后，官方 Codex 家族 `/v1/responses` 保留 `instructions`、`input`、tools、reasoning、`prompt_cache_key`、`previous_response_id`、`conversation_id`/`session_id` 和官方 metadata，不再按 attempt 重建 `prompt_cache_key`；只清掉 API key/authorization/client_metadata。会话使用 `mode=passthrough`。OpenAI 兼容客户端的 body 归一化不变。 +- 新增官方 Claude Code 透传：开启 `compatibility.official_cc_passthrough` 后，官方家族 cli-hop 保留调用方 system/tools/messages、cache_control 位置与 TTL、temperature、context_management、tool_choice 和 thinking，只重绑槽位 `metadata.user_id`，不再注入 `official_full` 完整 prompt、不重打 1h 断点、不按 attempt 重 mint session。内核 persona 投影为 `zero`，避免二次注入。非官方客户端仍走原 persona/rewrite。（#674） + ## 1.3.211 — 2026-10-22 - 单测对齐 cli-hop 缓存标记已写入的会话 TTL（默认 `1h`，请求带 `5m` 则为 `5m`），以及 OAuth persona 三块 system（billing / identity / persona）；修正 `officialMessageMarker` 过时注释。纯测试修复，无运行时行为变化。（#675） diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index 345913332..0ae510dd8 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -123,7 +123,9 @@ Rikka / 客户端 `search_web`、`scrape_web` **不是** Anthropic 自带搜索 3. 更早的型号仍逐块过滤：签名短于 24、空签名、dummy 签名删掉，合法签名块留下。`redacted_thinking` 不看 `data`，只在签名达到 24 字符时留。 4. 保留路径（cli-hop 首跳，以及默认 HTTP hop）不走上面的拆链，只在 `stripInvalidThinkingBlocks` 的这一支里处理：`redacted_thinking` 不看 `data` 和签名，原样留。`thinking` 只有签名可用（非空、不少于 24、非 dummy）才留，空正文也留；缺失、空、短于 24、dummy 的删掉，不再补 `signature: ""`。不走保留路径时，`redacted_thinking` 仍按上面第 2、3 条删。当前没有按账号归属过滤签名的步骤。 -HTTP hop 与 cli-hop（`prepareCliHopBody`）共用内容预过滤，但缓存职责不同。cli-hop 首跳保留签名可用的 thinking 指纹，没有可用签名的删掉；签名修复重试才清除其余块。官方家族开启 `official_cc_passthrough` 时，Node 保留 system/tools/messages 的 caller marker、显式 TTL、缺失 TTL 和位置，不添点、不裁剪；超过四点、thinking marker、非法 TTL 或 5m（含缺失）后出现 1h，在调度前返回 `400 invalid_cache_control`。 +HTTP hop 与 cli-hop（`prepareCliHopBody`）共用内容预过滤，但缓存职责不同。cli-hop 首跳保留签名可用的 thinking 指纹，没有可用签名的删掉；签名修复重试才清除其余块。官方家族开启 `official_cc_passthrough` 时，cli-hop 走 `prepareOfficialPassthroughBody`：保留调用方真实 Claude Code 的 system/tools/messages、cache_control 位置与 TTL、`temperature`、`context_management`、`tool_choice` 和 thinking，不注入 billing/system、不重打断点、不改工具名；只把 `metadata.user_id` 重绑到槽位设备/账号，并使用调用方 session（`mode=passthrough`）。该模式面向官方客户端；内核 `persona_preset` 投影为 `zero`，避免再叠加完整官方 prompt。非官方请求仍走原 rewrite；`official_cc_passthrough` 对它们不生效。 + +Codex hop 的 `codex.official_passthrough=true` 只对 `openai.responses` 的官方 Codex 客户端生效：保留 `instructions`、`input`、tools、reasoning、`prompt_cache_key`、`previous_response_id`、`conversation_id`/`session_id`、官方 metadata 和 sampling 字段，只清 API key/authorization/client_metadata；outbound session 使用 `passthrough`。OpenAI 兼容客户端的 Responses 归一化与 Chat/Completions 转换不变。 关闭透传或非官方请求仍走 Node rewrite：清除 caller marker，生成最后消息及 `messages.length>=4` 时倒数第二 user 的点，TTL 用入站会话 pin 值。应用 persona 后，模板显式 TTL 保留，只补自产缺失 TTL；超额时优先删 tools、system 尾部的自产点。非法模板返回 request-scoped 400，零上游执行，不换号、不冷却。 diff --git a/src/lib/protocol/codex-convert.mjs b/src/lib/protocol/codex-convert.mjs index f98c45349..4b32b420f 100644 --- a/src/lib/protocol/codex-convert.mjs +++ b/src/lib/protocol/codex-convert.mjs @@ -15,11 +15,11 @@ const IDENTITY_KEYS = [ 'client_metadata', ] -export function stripCodexIdentity(body = {}) { +export function stripCodexIdentity(body = {}, { preserveMetadata = false } = {}) { if (!body || typeof body !== 'object') return {} const next = { ...body } for (const key of IDENTITY_KEYS) delete next[key] - if (next.metadata && typeof next.metadata === 'object') { + if (!preserveMetadata && next.metadata && typeof next.metadata === 'object') { const metadata = { ...next.metadata } delete metadata.user_id delete metadata.device_id @@ -29,6 +29,23 @@ export function stripCodexIdentity(body = {}) { return next } +/** + * Official Codex Responses passthrough. + * + * The official CLI already sends the complete Responses body: instructions, + * input items, tools, reasoning, service_tier, prompt_cache_key and session + * fields. Normalizing it (system→developer, max_output/temperature strip, + * prompt_cache_key rebuild) changes the cached prefix and the session identity. + * Only strip credentials/client routing metadata; keep the wire body intact. + */ +export function toOfficialCodexPassthrough(body = {}) { + return { + ok: true, + body: stripCodexIdentity(body, { preserveMetadata: true }), + converted: false, + } +} + function textParts(content) { if (typeof content === 'string') return content if (!Array.isArray(content)) return content?.text || '' @@ -307,7 +324,8 @@ export function anthropicToCodexResponses(body = {}) { }) } -export function toCodexResponses(protocol, body, convert = {}) { +export function toCodexResponses(protocol, body, convert = {}, { officialPassthrough = false } = {}) { + if (officialPassthrough && protocol === 'openai.responses') return toOfficialCodexPassthrough(body) const stripped = stripCodexIdentity(body) if (protocol === 'openai.responses') { return { diff --git a/src/lib/protocol/codex-route.mjs b/src/lib/protocol/codex-route.mjs index d0d0c69b0..119c1dea9 100644 --- a/src/lib/protocol/codex-route.mjs +++ b/src/lib/protocol/codex-route.mjs @@ -39,6 +39,9 @@ export const DEFAULT_CODEX_ROUTING = Object.freeze({ claude_code: 'reject', unknown: 'allow', }, + // Official Codex CLI already sends the complete Responses body. Keep it + // byte/field-equivalent instead of normalizing/rebuilding the cache key. + official_passthrough: false, quota: normalizeOpenAIQuotaPolicy(), }) @@ -104,6 +107,12 @@ export function validateInferenceWebsocketPatch(body = {}) { if (Object.prototype.hasOwnProperty.call(codex, 'analytics')) { validateEnabledToggle(codex.analytics, 'codex.analytics', 'invalid_codex_analytics') } + if ( + Object.prototype.hasOwnProperty.call(codex, 'official_passthrough') && + typeof codex.official_passthrough !== 'boolean' + ) { + throw typedError('codex.official_passthrough 必须是布尔', 'invalid_codex_official_passthrough') + } if (Object.prototype.hasOwnProperty.call(codex, 'model_mappings')) { validateCodexModelMappingsPatch(codex.model_mappings) } @@ -147,6 +156,7 @@ export function normalizeCodexRouting(raw = {}) { anthropic_to_codex: src.convert?.anthropic_to_codex === true, }, clients, + official_passthrough: src.official_passthrough === true, quota: normalizeOpenAIQuotaPolicy(src.quota), model_mappings: normalizeCodexModelMappings(src.model_mappings), } diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index 772964936..dbe92cb0b 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -183,7 +183,7 @@ export function isRetryableSameCodexAccount( return false } -function applyCodexRebuildBody(body, sessionId, mode) { +export function applyCodexRebuildBody(body, sessionId, mode) { const out = { ...body } if (mode !== 'rebuild') return out out.prompt_cache_key = sessionId @@ -540,7 +540,10 @@ export async function handleCodexProtocol({ error: { type: 'permission_error', code: restriction.code, message: restriction.message }, }) } - const converted = toCodexResponses(protocol, ctx.body, codex.convert) + const officialCodexPassthrough = codex.official_passthrough === true && restriction.kind === 'official_codex' + const converted = toCodexResponses(protocol, ctx.body, codex.convert, { + officialPassthrough: officialCodexPassthrough, + }) if (!converted.ok) { stats.errors++ logBag.via = 'codex-kernel' @@ -566,7 +569,7 @@ export async function handleCodexProtocol({ }) const stream = inbound?.stream !== false && ctx.body?.stream !== false const inboundSession = sessionFrom(req, converted.body) - const sessionMode = outboundSessionMode(routing) + const sessionMode = officialCodexPassthrough ? 'passthrough' : outboundSessionMode(routing) const callerSession = extractCallerSession({ inbound: converted.body, body: converted.body, diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index f467703d1..355b39ce0 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -82,7 +82,7 @@ import { } from '../core/errors.mjs' import { resolveWorkspaceMode, isOfficialClaudeClient } from './workspace-mode.mjs' import { officialMessagesBody } from './anthropic-messages.mjs' -import { prepareOutboundEnvelope, prepareCliHopBody } from './outbound-attempt.mjs' +import { prepareOutboundEnvelope, prepareCliHopBody, prepareOfficialPassthroughBody } from './outbound-attempt.mjs' import { loadVmIdentity, OFFICIAL_CLI_VERSION } from '../identity/vm-identity.mjs' import { touchTelemetrySession } from '../vm/worker-telemetry.mjs' import { @@ -1281,8 +1281,15 @@ export function createHandleProtocol(deps) { const keptSession = extra.freshSlot ? '' : attemptBound?.sessionId || '' const keptAccount = extra.freshSlot ? '' : attemptBound?.accountId || '' const keptVm = extra.freshSlot ? '' : attemptBound?.vmId || '' + const routingNow = getRouting() + const cliHop = resolveOfficialCcInference(selected.vm, routingNow) === 'cli-hop' + // Official Claude Code already carries the real system, cache markers + // and sampling fields. In this mode the gateway only rebinds the slot + // identity; it must not rebuild the body or re-mint the session. + const officialPassthrough = officialTraffic && cliHop && officialCcPassthroughFromRouting(routingNow) const attemptSessionId = resolveOutboundSessionId(callerSession, { ...sessionContext, + mode: officialPassthrough ? 'passthrough' : sessionContext.mode, accountId: selected.accountId, boundSessionId: keptSession, boundAccountId: keptAccount, @@ -1295,7 +1302,6 @@ export function createHandleProtocol(deps) { if (identity && attemptSessionId) identity.callerSessionId = attemptSessionId const credMode = credentialModeFromOauth(selected.vm?.claude || {}) const modeOverride = slotPersonaModeOverride(selected.vm) - const routingNow = getRouting() const repaired = extra.repaired === true const preserveThinkingBlocks = !repaired && @@ -1304,8 +1310,6 @@ export function createHandleProtocol(deps) { // cli-node receives the cleaned body after this boundary. Keep every // thinking block on the first hop and only let a repaired retry strip it. const preserveCliHopThinkingBlocks = !repaired - const officialPassthrough = officialTraffic && officialCcPassthroughFromRouting(routingNow) - const cliHop = resolveOfficialCcInference(selected.vm, routingNow) === 'cli-hop' const resolvedPersona = resolveSlotPersonaPreset(selected.vm, routingNow) const preserveCallerEnvironment = !presetFlagEnabled( routingNow?.compatibility, @@ -1316,23 +1320,34 @@ export function createHandleProtocol(deps) { if (cliHop) { // The front rewrite already produces the official two-block shape. const panelRewrite = false - hopBody = prepareCliHopBody(repaired ? body : hopBody, { - stream: upstreamStream, - repaired, - cacheTtl: requestedCacheTtl, - requestContext, - preserveSystem: false, - preserveThinkingBlocks: preserveCliHopThinkingBlocks, - preserveCallerEnvironment, - autoCompactWindow: ctx.autoCompactWindow, - routing: routingNow, - }) - if (!requestContext) { - hopBody = ensureBillingAttribution(hopBody, { - firstUserText, + if (officialPassthrough && !requestContext) { + hopBody = prepareOfficialPassthroughBody(hopBody, { + stream: upstreamStream, + identity, + inbound, + reqHeaders: req.headers, sessionId: attemptSessionId, - cliVersion: OFFICIAL_CLI_VERSION, }) + logBag.official_passthrough = true + } else { + hopBody = prepareCliHopBody(repaired ? body : hopBody, { + stream: upstreamStream, + repaired, + cacheTtl: requestedCacheTtl, + requestContext, + preserveSystem: false, + preserveThinkingBlocks: preserveCliHopThinkingBlocks, + preserveCallerEnvironment, + autoCompactWindow: ctx.autoCompactWindow, + routing: routingNow, + }) + if (!requestContext) { + hopBody = ensureBillingAttribution(hopBody, { + firstUserText, + sessionId: attemptSessionId, + cliVersion: OFFICIAL_CLI_VERSION, + }) + } } hopBody = await materializeRemoteImageSources(hopBody) if (panelRewrite) { @@ -1433,7 +1448,7 @@ export function createHandleProtocol(deps) { boundVmId: keptVm, vmId: selected.vmId, epoch: attemptStartedAt, - mode: sessionMode, + mode: officialPassthrough ? 'passthrough' : sessionMode, routing: routingNow, clientDiscriminator, firstUserText, @@ -1441,11 +1456,16 @@ export function createHandleProtocol(deps) { apiKeyId: req.apiKeyRecord?.id ?? '', stream: upstreamStream, cacheControlLimit: Number(getRouting()?.compatibility?.cache_control_limit) || 4, - toolNameRewrite: openaiCompat ? false : getRouting()?.compatibility?.tool_name_rewrite !== false, + toolNameRewrite: officialPassthrough + ? false + : openaiCompat + ? false + : getRouting()?.compatibility?.tool_name_rewrite !== false, cacheTtl, cacheBreakpoints, preserveThinkingBlocks, - preserveCallerEnvironment, + preserveCallerEnvironment: officialPassthrough ? true : preserveCallerEnvironment, + officialPassthrough, reqHeaders: req.headers, homeDir: selected.exec?.homeDir || '', credentialMode: credMode, diff --git a/src/lib/protocol/outbound-attempt.mjs b/src/lib/protocol/outbound-attempt.mjs index 2146c2451..dc5d4c4b2 100644 --- a/src/lib/protocol/outbound-attempt.mjs +++ b/src/lib/protocol/outbound-attempt.mjs @@ -335,6 +335,41 @@ export function prepareCliHopBody( } return body } + +/** + * Official Claude Code passthrough. + * + * The caller already sends the genuine Claude Code wire body: system layout, + * tool schemas, cache_control positions/TTLs, sampling fields and thinking + * blocks. Rebuilding it (official_full persona, 1h marker restamping, field + * cleanup) only inflates the prefix and invalidates the caller's cache. + * + * Keep the canonical Messages shape and rebind only the slot identity in + * metadata. Real system/tools/messages/cache_control stay byte-equivalent at + * the object level; no billing block, no marker restamp, no tool rename. + */ +export function prepareOfficialPassthroughBody( + canonicalBody, + { + stream = true, + identity = null, + inbound = {}, + reqHeaders = {}, + sessionId = '', + mode = 'passthrough', + sessionContext = {}, + } = {}, +) { + let body = officialMessagesBody(canonicalBody, { stream, preserveContentShape: true }) + body = applyCrsIdentityReplace(body, identity, inbound, reqHeaders, { + ...sessionContext, + officialClient: true, + mode, + sessionId, + }) + return body +} + /** Only legacy Haiku rejects thinking; Haiku 5.5 supports adaptive thinking. */ export function pinHaikuCliThinking(body = {}) { if (!body || typeof body !== 'object') return body @@ -481,7 +516,18 @@ export function prepareOutboundEnvelope({ authScheme, credentialMode, want1m, + officialPassthrough = false, } = {}) { + if (officialPassthrough) { + const headers = { + ...prepareOutboundHeaders(reqHeaders, homeDir, identity, canonicalBody?.model || inbound?.model, { + credentialMode, + want1m: want1m === true, + }), + } + if (stream) headers.accept = 'text/event-stream' + return { body: canonicalBody, headers, toolNames: {} } + } const prepared = prepareOutboundAttempt({ requestContext, canonicalBody, diff --git a/src/lib/vm/slot-engine.mjs b/src/lib/vm/slot-engine.mjs index 53fd6ad22..4edf6a7b1 100644 --- a/src/lib/vm/slot-engine.mjs +++ b/src/lib/vm/slot-engine.mjs @@ -208,9 +208,13 @@ export function parseOfficialCcInferencePatch(value) { } export function resolveSlotPersonaPreset(vm, routing = {}) { + const compat = routing?.compatibility || {} + // Official-only deployments keep the caller's genuine Claude Code wire body. + // The kernel must not append the full official prompt on top of it; `zero` + // is the only kernel layout that still keeps the caller system block. + if (compat.official_cc_passthrough === true) return 'zero' const fromVm = normalizeSlotPersonaPreset(vm?.persona_preset, { inherit: true }) if (fromVm) return fromVm - const compat = routing?.compatibility || {} if (compat.persona_preset != null && String(compat.persona_preset).trim() !== '') { return normalizePersonaPreset(compat.persona_preset) } diff --git a/test/unit/codex-convert.test.mjs b/test/unit/codex-convert.test.mjs index 4c0b1ed73..afeada923 100644 --- a/test/unit/codex-convert.test.mjs +++ b/test/unit/codex-convert.test.mjs @@ -408,3 +408,38 @@ test('completed snapshots recover unregistered function and custom calls in ever } assert.equal(frames.find((frame) => frame.type === 'message_delta').delta.stop_reason, 'tool_use') }) + +test('official Codex passthrough keeps instructions/input/cache/session fields intact', () => { + const input = { + model: 'gpt-5.5', + instructions: 'You are Codex, a coding agent.', + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }], + tools: [{ type: 'function', name: 'shell', parameters: { type: 'object' } }], + reasoning: { effort: 'high' }, + service_tier: 'priority', + max_output_tokens: 8192, + temperature: 0.2, + prompt_cache_key: 'caller-session-key', + previous_response_id: 'resp_123', + conversation_id: 'conv_123', + session_id: 'sess_123', + metadata: { user_id: 'caller-user', topic: 'keep' }, + store: false, + } + const out = toCodexResponses('openai.responses', input, {}, { officialPassthrough: true }) + assert.equal(out.ok, true) + assert.equal(out.converted, false) + assert.deepEqual(out.body, input, 'official body must be preserved field-for-field') +}) + +test('official Codex passthrough still strips credentials', () => { + const out = toCodexResponses( + 'openai.responses', + { model: 'gpt-5.5', input: [], api_key: 'sk-live', authorization: 'Bearer x', client_metadata: { x: 1 } }, + {}, + { officialPassthrough: true }, + ) + assert.equal(out.body.api_key, undefined) + assert.equal(out.body.authorization, undefined) + assert.equal(out.body.client_metadata, undefined) +}) diff --git a/test/unit/codex-route.test.mjs b/test/unit/codex-route.test.mjs index 1707fedbc..2796bc6c3 100644 --- a/test/unit/codex-route.test.mjs +++ b/test/unit/codex-route.test.mjs @@ -18,6 +18,15 @@ test('default routing allows responses and chat, rejects anthropic', () => { assert.equal(isCodexProtocolAllowed('anthropic.messages', routing).code, 'protocol_not_allowed') }) +test('official Codex passthrough defaults off and round-trips', () => { + assert.equal(normalizeCodexRouting().official_passthrough, false) + assert.equal(normalizeCodexRouting({ official_passthrough: true }).official_passthrough, true) + assert.throws( + () => validateInferenceWebsocketPatch({ codex: { official_passthrough: 'yes' } }), + (err) => err.code === 'invalid_codex_official_passthrough', + ) +}) + test('disabled hop rejects every protocol', () => { const routing = { codex: normalizeCodexRouting({ enabled: false }) } assert.equal(isCodexProtocolAllowed('openai.responses', routing).code, 'codex_disabled') diff --git a/test/unit/handle-codex.test.mjs b/test/unit/handle-codex.test.mjs index dc6fe9040..aa1393691 100644 --- a/test/unit/handle-codex.test.mjs +++ b/test/unit/handle-codex.test.mjs @@ -9,6 +9,7 @@ import { handleCodexProtocol, codexFailureHold, codexFailoverBudget, + applyCodexRebuildBody, } from '../../src/lib/protocol/handle-codex.mjs' import { persistCodexUsage, getVm } from '../../src/lib/vm/vm-registry.mjs' import { StickyRouter } from '../../src/lib/pool/sticky-router.mjs' @@ -1286,6 +1287,47 @@ test('codex passthrough without an inbound session derives a stable fallback ses fs.rmSync(root, { recursive: true, force: true }) }) +test('official Codex passthrough keeps native body, cache key and session', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-official-pass-')) + const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) + writeGptVm(root, 'vm-gpt-a') + const inbound = 'sess-official-pass' + const body = { + model: 'gpt-5.5', + instructions: 'You are Codex, a coding agent.', + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }], + stream: false, + prompt_cache_key: 'cache-official-pass', + conversation_id: inbound, + session_id: inbound, + max_output_tokens: 8192, + temperature: 0.2, + reasoning: { effort: 'high' }, + } + const { envelopes } = await hopCodex({ + root, + stickyRouter: sticky, + routing: { codex: { official_passthrough: true } }, + headers: { + 'user-agent': 'codex_cli_rs/0.160.0 (linux x86_64)', + originator: 'codex_cli_rs', + 'session-id': inbound, + }, + body, + }) + const sent = envelopes[0] + assert.deepEqual(sent.body.instructions, body.instructions) + assert.deepEqual(sent.body.input, body.input) + assert.equal(sent.body.prompt_cache_key, 'cache-official-pass') + assert.equal(sent.body.conversation_id, inbound) + assert.equal(sent.body.session_id, inbound) + assert.equal(sent.body.max_output_tokens, 8192) + assert.equal(sent.body.temperature, 0.2) + assert.equal(sent.session.session_id, inbound) + sticky.db?.close?.() + fs.rmSync(root, { recursive: true, force: true }) +}) + test('openai.responses stream names each kernel SSE frame after its payload type', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-sse-names-')) writeGptVm(root, 'vm-gpt-a') @@ -1404,3 +1446,18 @@ test('tool argument disagreement terminates the client stream instead of recordi fs.rmSync(root, { recursive: true, force: true }) } }) + +test('Codex passthrough leaves prompt_cache_key/conversation/session untouched', () => { + const body = { + model: 'gpt-5.5', + input: [], + prompt_cache_key: 'caller-key', + conversation_id: 'conv', + session_id: 'sess', + } + assert.deepEqual(applyCodexRebuildBody(body, 'proxy-session', 'passthrough'), body) + const rebuilt = applyCodexRebuildBody(body, 'proxy-session', 'rebuild') + assert.equal(rebuilt.prompt_cache_key, 'proxy-session') + assert.equal(rebuilt.conversation_id, undefined) + assert.equal(rebuilt.session_id, undefined) +}) diff --git a/test/unit/official-passthrough.test.mjs b/test/unit/official-passthrough.test.mjs new file mode 100644 index 000000000..da39f56fd --- /dev/null +++ b/test/unit/official-passthrough.test.mjs @@ -0,0 +1,151 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { + prepareCliHopBody, + prepareOfficialPassthroughBody, + prepareOutboundEnvelope, +} from '../../src/lib/protocol/outbound-attempt.mjs' +import { resolveSlotPersonaPreset } from '../../src/lib/vm/slot-engine.mjs' + +const identity = { + deviceId: 'dev-slot-01', + machineId: 'mid-slot-01', + accountUuid: 'acct-slot-01', + vmId: 'vm-01', +} + +function officialBody() { + return { + model: 'claude-opus-5-5', + max_tokens: 128000, + stream: true, + temperature: 1, + metadata: { + user_id: JSON.stringify({ + device_id: 'caller-device', + account_uuid: 'caller-account', + session_id: 'caller-session', + }), + }, + system: [ + { + type: 'text', + text: 'x-anthropic-billing-header: cc_version=2.1.306.de1; cc_entrypoint=claude-vscode; cch=abcde;', + }, + { type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }, + { + type: 'text', + text: 'You are an interactive agent that helps users with software engineering tasks.', + cache_control: { type: 'ephemeral', ttl: '1h' }, + }, + ], + tools: [ + { + name: 'Bash', + input_schema: { type: 'object' }, + cache_control: { type: 'ephemeral', ttl: '1h' }, + }, + { + name: 'Read', + input_schema: { type: 'object' }, + cache_control: { type: 'ephemeral', ttl: '5m' }, + }, + ], + tool_choice: { type: 'auto' }, + context_management: { edits: [{ type: 'clear_thinking_20251015', keep: 'all' }] }, + messages: [ + { role: 'user', content: [{ type: 'text', text: 'first', cache_control: { type: 'ephemeral' } }] }, + { + role: 'assistant', + content: [{ type: 'text', text: 'answer', cache_control: { type: 'ephemeral', ttl: '1h' } }], + }, + { role: 'user', content: [{ type: 'text', text: 'last' }] }, + ], + } +} + +test('official passthrough keeps system/tools/messages/cache markers and sampling fields', () => { + const input = officialBody() + const body = prepareOfficialPassthroughBody(input, { + stream: true, + identity, + inbound: input, + reqHeaders: { 'user-agent': 'claude-cli/2.1.306 (external, claude-vscode)' }, + sessionId: 'caller-session', + }) + + assert.deepEqual(body.system, input.system, 'system and cache_control must not be rewritten') + assert.deepEqual(body.tools, input.tools, 'tool schemas and markers must not be rewritten') + assert.deepEqual(body.messages, input.messages, 'messages and markers must not be rewritten') + assert.equal(body.temperature, 1) + assert.deepEqual(body.context_management, input.context_management) + assert.deepEqual(body.tool_choice, input.tool_choice) + assert.equal( + JSON.stringify(body.system).includes('cc_entrypoint=sdk-cli'), + false, + 'must not add a proxy billing block', + ) + assert.equal(JSON.stringify(body.system).includes('ttl":"1h"'), true, 'caller 1h markers stay as sent') + + const metadata = JSON.parse(body.metadata.user_id) + assert.equal(metadata.device_id, 'dev-slot-01') + assert.equal(metadata.account_uuid, 'acct-slot-01') + assert.equal(metadata.session_id, 'caller-session') +}) + +test('official passthrough envelope returns the prepared body untouched', () => { + const input = officialBody() + const body = prepareOfficialPassthroughBody(input, { + stream: true, + identity, + inbound: input, + reqHeaders: {}, + sessionId: 'caller-session', + }) + const envelope = prepareOutboundEnvelope({ + canonicalBody: body, + inbound: input, + identity, + officialClient: true, + unofficial: false, + stream: true, + officialPassthrough: true, + reqHeaders: {}, + credentialMode: 'oauth', + routing: {}, + }) + assert.equal(envelope.body, body, 'no sanitize / CCH / tool rename on official passthrough') + assert.equal(envelope.headers.accept, 'text/event-stream') + assert.equal(envelope.toolNames && Object.keys(envelope.toolNames).length, 0) +}) + +test('official passthrough does not run the cli-hop rewrite path', () => { + const rewritten = prepareCliHopBody(officialBody(), { stream: true }) + assert.equal(rewritten.system[0].text, "You are Claude Code, Anthropic's official CLI for Claude.") + assert.equal( + rewritten.system.length < officialBody().system.length, + true, + 'the legacy path collapses/rebuilds the caller system', + ) + assert.notDeepEqual(rewritten.messages, officialBody().messages) +}) + +test('official_cc_passthrough projects the kernel to zero persona', () => { + assert.equal(resolveSlotPersonaPreset({}, { compatibility: { official_cc_passthrough: true } }), 'zero') + assert.equal(resolveSlotPersonaPreset({ persona_preset: 'official_full' }, { compatibility: {} }), 'official_full') +}) + +test('official passthrough does not inflate a 30k-char caller system', () => { + const input = officialBody() + input.system[2].text = `You are an interactive agent.\n${'x'.repeat(30_000)}` + const body = prepareOfficialPassthroughBody(input, { + stream: true, + identity, + inbound: input, + reqHeaders: {}, + sessionId: 'caller-session', + }) + assert.equal(body.system[2].text.length, input.system[2].text.length) + assert.equal(JSON.stringify(body.system).includes('__PERSONA_AGENT__'), false) + assert.equal(JSON.stringify(body.system).includes('cc_entrypoint=sdk-cli'), false) +}) diff --git a/web/src/features/settings/openai-pane.tsx b/web/src/features/settings/openai-pane.tsx index af51a305b..40db80a72 100644 --- a/web/src/features/settings/openai-pane.tsx +++ b/web/src/features/settings/openai-pane.tsx @@ -168,6 +168,21 @@ export function OpenaiPane({ aria-label='允许官方 Codex 客户端' /> + + update({ official_passthrough: on })} + aria-label='官方 Codex 客户端透传' + /> + Date: Thu, 22 Oct 2026 09:35:07 +0000 Subject: [PATCH 470/473] release: cut v1.3.212 Co-authored-by: ted --- CHANGELOG.md | 4 +++- VERSION | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20f0e4c16..f76349227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,12 @@ # Changelog -## 未发布 +## 1.3.212 — 2026-10-22 - 新增官方 Codex 透传：开启 `codex.official_passthrough` 后，官方 Codex 家族 `/v1/responses` 保留 `instructions`、`input`、tools、reasoning、`prompt_cache_key`、`previous_response_id`、`conversation_id`/`session_id` 和官方 metadata，不再按 attempt 重建 `prompt_cache_key`；只清掉 API key/authorization/client_metadata。会话使用 `mode=passthrough`。OpenAI 兼容客户端的 body 归一化不变。 - 新增官方 Claude Code 透传：开启 `compatibility.official_cc_passthrough` 后，官方家族 cli-hop 保留调用方 system/tools/messages、cache_control 位置与 TTL、temperature、context_management、tool_choice 和 thinking，只重绑槽位 `metadata.user_id`，不再注入 `official_full` 完整 prompt、不重打 1h 断点、不按 attempt 重 mint session。内核 persona 投影为 `zero`，避免二次注入。非官方客户端仍走原 persona/rewrite。（#674） +已部署机升级：更新 Node 控制面与 `web/dist` 后重启一次。启用时在面板打开对应开关并保存投影；不必 `wrap-cli/sync`、不必 `fetch:payload`、不必重建槽。不要覆盖 `routing.json`、`vms/`、`data/`、`.env`。 + ## 1.3.211 — 2026-10-22 - 单测对齐 cli-hop 缓存标记已写入的会话 TTL（默认 `1h`，请求带 `5m` 则为 `5m`），以及 OAuth persona 三块 system（billing / identity / persona）；修正 `officialMessageMarker` 过时注释。纯测试修复，无运行时行为变化。（#675） diff --git a/VERSION b/VERSION index 0f2d0c300..3090871f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.211 +1.3.212 From a8a945173622970e77b1e731f2259c6f72410ec0 Mon Sep 17 00:00:00 2001 From: ted Date: Thu, 22 Oct 2026 18:01:05 +0800 Subject: [PATCH 471/473] =?UTF-8?q?fix(213):=20=E5=AE=98=E6=96=B9=E9=80=8F?= =?UTF-8?q?=E4=BC=A0=E4=BB=8D=20rebuild=20outbound=20session=EF=BC=8C?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E7=BB=A7=E7=BB=AD=E5=AE=8C=E5=85=A8=E9=80=8F?= =?UTF-8?q?=E4=BC=A0=20(#676)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: rebuild outbound session on official passthrough paths Official Claude Code and Codex content passthrough (#674) incorrectly forced session mode passthrough, reusing inbound session IDs. Restore attempt/VM rebuild semantics for outbound session while keeping body content passthrough and slot identity rebind. Co-authored-by: ted * fix: rebuild Codex prompt_cache_key on official passthrough Force rebuild session mode for official CC/Codex passthrough paths so prompt_cache_key follows outbound session, matching non-passthrough hop semantics while keeping instructions/input/tools content intact. Co-authored-by: ted * chore: biome format assert.match in handle-codex test Co-authored-by: ted --------- Co-authored-by: Cursor Agent --- CHANGELOG.md | 4 ++++ docs/PROTOCOL.md | 4 ++-- src/lib/protocol/codex-convert.mjs | 8 ++++---- src/lib/protocol/handle-codex.mjs | 2 +- src/lib/protocol/handle-protocol.mjs | 9 +++++---- src/lib/protocol/outbound-attempt.mjs | 2 +- test/unit/handle-codex.test.mjs | 12 +++++++----- test/unit/official-passthrough.test.mjs | 7 +++++-- web/src/features/settings/openai-pane.tsx | 2 +- web/src/features/settings/protocol-pane.tsx | 2 +- 10 files changed, 31 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f76349227..cc1e4ae88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 未发布 + +- 官方透传（`compatibility.official_cc_passthrough`、`codex.official_passthrough`）：system/instructions、messages/input、tools、cache_control、采样字段等 body 仍完全透传；outbound session 与 Codex `prompt_cache_key` 恢复 attempt/VM rebuild/mint（不再沿用入站 session 或调用方 cache key）；`metadata.user_id` 继续重绑槽位 device/account。默认开关仍关。 + ## 1.3.212 — 2026-10-22 - 新增官方 Codex 透传：开启 `codex.official_passthrough` 后，官方 Codex 家族 `/v1/responses` 保留 `instructions`、`input`、tools、reasoning、`prompt_cache_key`、`previous_response_id`、`conversation_id`/`session_id` 和官方 metadata，不再按 attempt 重建 `prompt_cache_key`；只清掉 API key/authorization/client_metadata。会话使用 `mode=passthrough`。OpenAI 兼容客户端的 body 归一化不变。 diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index 0ae510dd8..d60461bb0 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -123,9 +123,9 @@ Rikka / 客户端 `search_web`、`scrape_web` **不是** Anthropic 自带搜索 3. 更早的型号仍逐块过滤：签名短于 24、空签名、dummy 签名删掉，合法签名块留下。`redacted_thinking` 不看 `data`，只在签名达到 24 字符时留。 4. 保留路径（cli-hop 首跳，以及默认 HTTP hop）不走上面的拆链，只在 `stripInvalidThinkingBlocks` 的这一支里处理：`redacted_thinking` 不看 `data` 和签名，原样留。`thinking` 只有签名可用（非空、不少于 24、非 dummy）才留，空正文也留；缺失、空、短于 24、dummy 的删掉，不再补 `signature: ""`。不走保留路径时，`redacted_thinking` 仍按上面第 2、3 条删。当前没有按账号归属过滤签名的步骤。 -HTTP hop 与 cli-hop（`prepareCliHopBody`）共用内容预过滤，但缓存职责不同。cli-hop 首跳保留签名可用的 thinking 指纹，没有可用签名的删掉；签名修复重试才清除其余块。官方家族开启 `official_cc_passthrough` 时，cli-hop 走 `prepareOfficialPassthroughBody`：保留调用方真实 Claude Code 的 system/tools/messages、cache_control 位置与 TTL、`temperature`、`context_management`、`tool_choice` 和 thinking，不注入 billing/system、不重打断点、不改工具名；只把 `metadata.user_id` 重绑到槽位设备/账号，并使用调用方 session（`mode=passthrough`）。该模式面向官方客户端；内核 `persona_preset` 投影为 `zero`，避免再叠加完整官方 prompt。非官方请求仍走原 rewrite；`official_cc_passthrough` 对它们不生效。 +HTTP hop 与 cli-hop（`prepareCliHopBody`）共用内容预过滤，但缓存职责不同。cli-hop 首跳保留签名可用的 thinking 指纹，没有可用签名的删掉；签名修复重试才清除其余块。官方家族开启 `official_cc_passthrough` 时，cli-hop 走 `prepareOfficialPassthroughBody`：保留调用方真实 Claude Code 的 system/tools/messages、cache_control 位置与 TTL、`temperature`、`context_management`、`tool_choice` 和 thinking，不注入 billing/system、不重打断点、不改工具名；只把 `metadata.user_id` 重绑到槽位设备/账号。出站 `metadata.user_id.session_id` 按 attempt/VM rebuild 语义 mint（不沿用入站 session）。该模式面向官方客户端；内核 `persona_preset` 投影为 `zero`，避免再叠加完整 official prompt。非官方请求仍走原 rewrite；`official_cc_passthrough` 对它们不生效。 -Codex hop 的 `codex.official_passthrough=true` 只对 `openai.responses` 的官方 Codex 客户端生效：保留 `instructions`、`input`、tools、reasoning、`prompt_cache_key`、`previous_response_id`、`conversation_id`/`session_id`、官方 metadata 和 sampling 字段，只清 API key/authorization/client_metadata；outbound session 使用 `passthrough`。OpenAI 兼容客户端的 Responses 归一化与 Chat/Completions 转换不变。 +Codex hop 的 `codex.official_passthrough=true` 只对 `openai.responses` 的官方 Codex 客户端生效：保留 `instructions`、`input`、tools、reasoning、`previous_response_id`、官方 metadata 和 sampling 字段，只清 API key/authorization/client_metadata；`prompt_cache_key` 与内核 `session-id`/sticky outbound session 按 attempt/VM rebuild 写入（不沿用入站 session 或调用方 cache key）。OpenAI 兼容客户端的 Responses 归一化与 Chat/Completions 转换不变。 关闭透传或非官方请求仍走 Node rewrite：清除 caller marker，生成最后消息及 `messages.length>=4` 时倒数第二 user 的点，TTL 用入站会话 pin 值。应用 persona 后，模板显式 TTL 保留，只补自产缺失 TTL；超额时优先删 tools、system 尾部的自产点。非法模板返回 request-scoped 400，零上游执行，不换号、不冷却。 diff --git a/src/lib/protocol/codex-convert.mjs b/src/lib/protocol/codex-convert.mjs index 4b32b420f..d1118ac8a 100644 --- a/src/lib/protocol/codex-convert.mjs +++ b/src/lib/protocol/codex-convert.mjs @@ -33,10 +33,10 @@ export function stripCodexIdentity(body = {}, { preserveMetadata = false } = {}) * Official Codex Responses passthrough. * * The official CLI already sends the complete Responses body: instructions, - * input items, tools, reasoning, service_tier, prompt_cache_key and session - * fields. Normalizing it (system→developer, max_output/temperature strip, - * prompt_cache_key rebuild) changes the cached prefix and the session identity. - * Only strip credentials/client routing metadata; keep the wire body intact. + * input items, tools, reasoning, service_tier and sampling fields. Normalizing + * it (system→developer, max_output/temperature strip) changes the cached prefix. + * Only strip credentials/client routing metadata; keep content fields intact. + * Outbound session and prompt_cache_key rebuild happen later on the Codex hop. */ export function toOfficialCodexPassthrough(body = {}) { return { diff --git a/src/lib/protocol/handle-codex.mjs b/src/lib/protocol/handle-codex.mjs index dbe92cb0b..23f5640e9 100644 --- a/src/lib/protocol/handle-codex.mjs +++ b/src/lib/protocol/handle-codex.mjs @@ -569,7 +569,7 @@ export async function handleCodexProtocol({ }) const stream = inbound?.stream !== false && ctx.body?.stream !== false const inboundSession = sessionFrom(req, converted.body) - const sessionMode = officialCodexPassthrough ? 'passthrough' : outboundSessionMode(routing) + const sessionMode = officialCodexPassthrough ? 'rebuild' : outboundSessionMode(routing) const callerSession = extractCallerSession({ inbound: converted.body, body: converted.body, diff --git a/src/lib/protocol/handle-protocol.mjs b/src/lib/protocol/handle-protocol.mjs index 355b39ce0..6e430b0ed 100644 --- a/src/lib/protocol/handle-protocol.mjs +++ b/src/lib/protocol/handle-protocol.mjs @@ -1284,12 +1284,13 @@ export function createHandleProtocol(deps) { const routingNow = getRouting() const cliHop = resolveOfficialCcInference(selected.vm, routingNow) === 'cli-hop' // Official Claude Code already carries the real system, cache markers - // and sampling fields. In this mode the gateway only rebinds the slot - // identity; it must not rebuild the body or re-mint the session. + // and sampling fields. Content stays on the wire; slot identity and + // outbound session still follow attempt/VM rebuild semantics. const officialPassthrough = officialTraffic && cliHop && officialCcPassthroughFromRouting(routingNow) + const attemptSessionMode = officialPassthrough ? 'rebuild' : sessionContext.mode const attemptSessionId = resolveOutboundSessionId(callerSession, { ...sessionContext, - mode: officialPassthrough ? 'passthrough' : sessionContext.mode, + mode: attemptSessionMode, accountId: selected.accountId, boundSessionId: keptSession, boundAccountId: keptAccount, @@ -1448,7 +1449,7 @@ export function createHandleProtocol(deps) { boundVmId: keptVm, vmId: selected.vmId, epoch: attemptStartedAt, - mode: officialPassthrough ? 'passthrough' : sessionMode, + mode: officialPassthrough ? 'rebuild' : sessionMode, routing: routingNow, clientDiscriminator, firstUserText, diff --git a/src/lib/protocol/outbound-attempt.mjs b/src/lib/protocol/outbound-attempt.mjs index dc5d4c4b2..c0d4e0b58 100644 --- a/src/lib/protocol/outbound-attempt.mjs +++ b/src/lib/protocol/outbound-attempt.mjs @@ -356,7 +356,7 @@ export function prepareOfficialPassthroughBody( inbound = {}, reqHeaders = {}, sessionId = '', - mode = 'passthrough', + mode = 'rebuild', sessionContext = {}, } = {}, ) { diff --git a/test/unit/handle-codex.test.mjs b/test/unit/handle-codex.test.mjs index aa1393691..8daf5b58c 100644 --- a/test/unit/handle-codex.test.mjs +++ b/test/unit/handle-codex.test.mjs @@ -1287,7 +1287,7 @@ test('codex passthrough without an inbound session derives a stable fallback ses fs.rmSync(root, { recursive: true, force: true }) }) -test('official Codex passthrough keeps native body, cache key and session', async () => { +test('official Codex passthrough keeps native content and rebuilds session plus cache key', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-codex-official-pass-')) const sticky = new StickyRouter({ dataDir: path.join(root, 'data'), config: { sticky: { enabled: true } } }) writeGptVm(root, 'vm-gpt-a') @@ -1318,12 +1318,14 @@ test('official Codex passthrough keeps native body, cache key and session', asyn const sent = envelopes[0] assert.deepEqual(sent.body.instructions, body.instructions) assert.deepEqual(sent.body.input, body.input) - assert.equal(sent.body.prompt_cache_key, 'cache-official-pass') - assert.equal(sent.body.conversation_id, inbound) - assert.equal(sent.body.session_id, inbound) + assert.equal(sent.body.prompt_cache_key, sent.session.session_id) + assert.notEqual(sent.body.prompt_cache_key, 'cache-official-pass') + assert.equal(sent.body.conversation_id, undefined) + assert.equal(sent.body.session_id, undefined) assert.equal(sent.body.max_output_tokens, 8192) assert.equal(sent.body.temperature, 0.2) - assert.equal(sent.session.session_id, inbound) + assert.notEqual(sent.session.session_id, inbound) + assert.match(sent.session.session_id, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) sticky.db?.close?.() fs.rmSync(root, { recursive: true, force: true }) }) diff --git a/test/unit/official-passthrough.test.mjs b/test/unit/official-passthrough.test.mjs index da39f56fd..1077e5e04 100644 --- a/test/unit/official-passthrough.test.mjs +++ b/test/unit/official-passthrough.test.mjs @@ -5,6 +5,7 @@ import { prepareOfficialPassthroughBody, prepareOutboundEnvelope, } from '../../src/lib/protocol/outbound-attempt.mjs' +import { rebuildOutboundSession } from '../../src/lib/identity/identity-rewrite.mjs' import { resolveSlotPersonaPreset } from '../../src/lib/vm/slot-engine.mjs' const identity = { @@ -66,12 +67,13 @@ function officialBody() { test('official passthrough keeps system/tools/messages/cache markers and sampling fields', () => { const input = officialBody() + const outboundSessionId = rebuildOutboundSession({ identity: 'caller-session', epoch: 'attempt:1' }) const body = prepareOfficialPassthroughBody(input, { stream: true, identity, inbound: input, reqHeaders: { 'user-agent': 'claude-cli/2.1.306 (external, claude-vscode)' }, - sessionId: 'caller-session', + sessionId: outboundSessionId, }) assert.deepEqual(body.system, input.system, 'system and cache_control must not be rewritten') @@ -90,7 +92,8 @@ test('official passthrough keeps system/tools/messages/cache markers and samplin const metadata = JSON.parse(body.metadata.user_id) assert.equal(metadata.device_id, 'dev-slot-01') assert.equal(metadata.account_uuid, 'acct-slot-01') - assert.equal(metadata.session_id, 'caller-session') + assert.equal(metadata.session_id, outboundSessionId) + assert.notEqual(metadata.session_id, 'caller-session') }) test('official passthrough envelope returns the prepared body untouched', () => { diff --git a/web/src/features/settings/openai-pane.tsx b/web/src/features/settings/openai-pane.tsx index 40db80a72..5d9db5f5d 100644 --- a/web/src/features/settings/openai-pane.tsx +++ b/web/src/features/settings/openai-pane.tsx @@ -172,7 +172,7 @@ export function OpenaiPane({ label='官方 Codex 客户端透传' desc={ codex.official_passthrough === true - ? '开：保留官方 Responses body、instructions、prompt_cache_key 与 session，不再按 attempt 重建。' + ? '开：保留 instructions/input/tools 等官方 body；session 与 prompt_cache_key 仍按槽位重建。' : '关：官方 Codex 也走 body 归一化和 prompt_cache_key 重建。' } > diff --git a/web/src/features/settings/protocol-pane.tsx b/web/src/features/settings/protocol-pane.tsx index 253373c29..55fc2317a 100644 --- a/web/src/features/settings/protocol-pane.tsx +++ b/web/src/features/settings/protocol-pane.tsx @@ -105,7 +105,7 @@ export function ProtocolPane({ label='官方 Claude Code 家族透传' desc={ compat.official_cc_passthrough === true - ? '开：直连和被识别为中转的官方家族保留入站 system，不套用当前提示词档。' + ? '开：保留入站 system/tools/messages 与 cache 标记；出站 session 仍按槽位重建，device 绑槽位。' : '关：官方家族也套用协议里的 system 档。打开后才透传。' } > From 2afeb3dd3507711765c8ef7d0fa5c50c96be3177 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 22 Oct 2026 10:05:25 +0000 Subject: [PATCH 472/473] release: cut v1.3.213 Co-authored-by: ted --- CHANGELOG.md | 6 +++++- VERSION | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc1e4ae88..152b80e40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,11 @@ ## 未发布 -- 官方透传（`compatibility.official_cc_passthrough`、`codex.official_passthrough`）：system/instructions、messages/input、tools、cache_control、采样字段等 body 仍完全透传；outbound session 与 Codex `prompt_cache_key` 恢复 attempt/VM rebuild/mint（不再沿用入站 session 或调用方 cache key）；`metadata.user_id` 继续重绑槽位 device/account。默认开关仍关。 +## 1.3.213 — 2026-10-22 + +- 官方透传（`compatibility.official_cc_passthrough`、`codex.official_passthrough`）：system/instructions、messages/input、tools、cache_control、采样字段等 body 仍完全透传；outbound session 与 Codex `prompt_cache_key` 恢复 attempt/VM rebuild/mint（不再沿用入站 session 或调用方 cache key）；`metadata.user_id` 继续重绑槽位 device/account。默认开关仍关。（#676） + +已部署机升级：更新 Node 控制面与 `web/dist` 后重启一次。启用时在面板打开对应开关并保存投影；不必 `wrap-cli/sync`、不必 `fetch:payload`、不必重建槽。不要覆盖 `routing.json`、`vms/`、`data/`、`.env`。 ## 1.3.212 — 2026-10-22 diff --git a/VERSION b/VERSION index 3090871f5..c24fd63d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.212 +1.3.213 From 2adb0816f9571f9140aab1386bcaa60d06661816 Mon Sep 17 00:00:00 2001 From: liubing Date: Tue, 27 Oct 2026 12:24:34 +0800 Subject: [PATCH 473/473] feat: add manual Claude subscription lookup credentials --- docs/claude-subscription.md | 13 ++ src/lib/admin/panel-acl.mjs | 8 +- src/lib/admin/panel-api.mjs | 4 + src/lib/admin/panel-routes.mjs | 94 +++++++++ .../022_claude_subscriptions.sql | 10 + .../db/repos/claude-subscriptions-repo.mjs | 111 +++++++++++ src/lib/oauth/claude-subscription.mjs | 129 +++++++++++++ test/unit/claude-subscription.test.mjs | 167 ++++++++++++++++ test/unit/claude-subscriptions-repo.test.mjs | 98 ++++++++++ test/unit/database.test.mjs | 7 +- test/unit/panel-claude-subscription.test.mjs | 177 +++++++++++++++++ .../vm/claude-subscription-button.tsx | 180 ++++++++++++++++++ .../features/vm/claude-subscription.test.ts | 26 +++ web/src/features/vm/claude-subscription.ts | 34 ++++ web/src/features/vm/detail-account-tab.tsx | 2 + web/src/features/vm/vm-detail-card.tsx | 2 + web/src/features/vm/vm-list-table.tsx | 23 ++- web/src/features/vm/vm-rack-blade.tsx | 13 +- web/src/features/vm/vm-rack-unit.tsx | 29 ++- web/src/types/panel-vm.ts | 15 ++ 20 files changed, 1121 insertions(+), 21 deletions(-) create mode 100644 docs/claude-subscription.md create mode 100644 src/lib/db/postgres-migrations/022_claude_subscriptions.sql create mode 100644 src/lib/db/repos/claude-subscriptions-repo.mjs create mode 100644 src/lib/oauth/claude-subscription.mjs create mode 100644 test/unit/claude-subscription.test.mjs create mode 100644 test/unit/claude-subscriptions-repo.test.mjs create mode 100644 test/unit/panel-claude-subscription.test.mjs create mode 100644 web/src/features/vm/claude-subscription-button.tsx create mode 100644 web/src/features/vm/claude-subscription.test.ts create mode 100644 web/src/features/vm/claude-subscription.ts diff --git a/docs/claude-subscription.md b/docs/claude-subscription.md new file mode 100644 index 000000000..8f71823f8 --- /dev/null +++ b/docs/claude-subscription.md @@ -0,0 +1,13 @@ +# Claude 订阅信息 + +虚拟机列表的账号列显示查询到的套餐和订阅日期。自动续费显示“续费”，明确结束的订阅显示“到期”。接口未返回日期时显示“未知”，不会用 OAuth 令牌有效期或额度重置时间代替。 + +在虚拟机详情卡或完整详情页的账号页签点击“订阅信息”，输入 Claude 网页的 `sessionKey`（`sk-ant-sid…`），点击“保存并查询”。查询成功且账号、组织与槽位匹配后才保存。可在同一窗口替换、手动刷新或删除查询凭据。 + +保存需要配置现有的 `VM2API_DB_SECRET` 或 `KIN_DB_SECRET`。凭据使用 AES-256-GCM 加密存储在独立的 `claude_subscriptions` 表中，前端不回显明文。删除查询凭据同时删除该槽位的订阅查询记录，不修改 OAuth 凭据或调度设置。删除虚拟机时数据库自动删除关联记录。 + +查询通过槽位已有的代理出口，只向 `https://claude.ai` 发送账号核对与订阅详情 GET 请求，不换取 OAuth 凭据，不调用模型。读取面板中的缓存不会触发上游查询。Cookie 失效或访问受限后需手动处理；查询失败保留上次成功的结果和时间，并标记待更新。账号或组织变更后不展示旧账号记录。 + +当前接口：`GET /api/panel/vms/:id/claude-subscription` 读取脱敏缓存；`PUT` 保存并查询；`POST` 使用已存凭据刷新；`DELETE` 删除。普通用户仅能操作自己的虚拟机；`super` 角色仅可读取缓存。数据表由 PostgreSQL 迁移 `022_claude_subscriptions.sql` 创建。 + +Claude 的 `subscription_details` 为网页内部接口，其可用性以实际账号权限和响应为准。`sessionKey` 本身仍是网页会话凭据；限定用途由本功能的调用路径保证。 diff --git a/src/lib/admin/panel-acl.mjs b/src/lib/admin/panel-acl.mjs index 02015ab4e..18c0640d2 100644 --- a/src/lib/admin/panel-acl.mjs +++ b/src/lib/admin/panel-acl.mjs @@ -147,7 +147,7 @@ const LOG_STATS_GET = new Set([ function isSuperVmRead(path) { if (path === '/api/panel/vms/fleet-status' || path === '/api/panel/vms/create-options') return false - return /^\/api\/panel\/vms\/[^/]+$/.test(path) + return /^\/api\/panel\/vms\/[^/]+(?:\/claude-subscription)?$/.test(path) } function isSuperSchedule(method, path) { @@ -171,6 +171,7 @@ function userVmPathAllowed(method, path) { if (rest === 'oauth/credential' || rest === 'package') return false return ( rest === 'test-schedule' || + rest === 'claude-subscription' || rest.startsWith('oauth/') || rest === 'probe' || rest === 'count-tokens' || @@ -198,11 +199,12 @@ function userVmPathAllowed(method, path) { rest === 'openai-quota/reset' || rest === 'claude-reset/query' || rest === 'claude-reset/redeem' || + rest === 'claude-subscription' || rest.startsWith('oauth/') ) } - if (method === 'PUT') return rest === 'test-schedule' || rest === 'oauth/credential' - if (method === 'DELETE') return !rest || rest === 'test-schedule' + if (method === 'PUT') return rest === 'test-schedule' || rest === 'oauth/credential' || rest === 'claude-subscription' + if (method === 'DELETE') return !rest || rest === 'test-schedule' || rest === 'claude-subscription' return false } diff --git a/src/lib/admin/panel-api.mjs b/src/lib/admin/panel-api.mjs index 225be9b24..d8ecc9dc2 100644 --- a/src/lib/admin/panel-api.mjs +++ b/src/lib/admin/panel-api.mjs @@ -48,6 +48,7 @@ import { hardBlockOf } from '../pool/rate-limit-service.mjs' import { unitCircuit } from '../pool/unit-circuit.mjs' import { isKubernetesMode } from '../core/control-plane-mode.mjs' import { getDb } from '../db/database.mjs' +import { stampClaudeSubscriptions } from '../db/repos/claude-subscriptions-repo.mjs' import { ExecutionAdmissionsRepo } from '../db/repos/execution-admissions-repo.mjs' import { accountIdOf, normalizePoolRouting } from '../pool/pool-scheduler.mjs' import { codexMaxSessions } from '../pool/codex-slot-pool.mjs' @@ -648,6 +649,7 @@ export async function buildDashboard({ ) const snap = accountQuota.snapshot() await stampSharedCircuitViews(vms, cfg.paths.project, routingConfig) + await stampClaudeSubscriptions(vms) const accounts = liveAccounts(snap.accounts, listed) const peak5 = Math.max(0, ...accounts.map((a) => Number(a.unified?.['5h']?.utilization || 0)), 0) const peak7 = Math.max(0, ...accounts.map((a) => Number(a.unified?.['7d']?.utilization || 0)), 0) @@ -778,6 +780,7 @@ export async function buildVmList({ }), ) await stampSharedCircuitViews(vms, cfg.paths.project, routingConfig) + await stampClaudeSubscriptions(vms) const accounts = (() => { try { return accountQuota?.snapshot?.().accounts || [] @@ -846,6 +849,7 @@ export async function buildVmDetail({ role, }) await stampSharedCircuitViews([summary], cfg.paths.project, routingConfig) + await stampClaudeSubscriptions([summary]) const acc = findAccount(accountQuota, summary) const billing = await (async () => { try { diff --git a/src/lib/admin/panel-routes.mjs b/src/lib/admin/panel-routes.mjs index 118b9748a..298c18b0b 100644 --- a/src/lib/admin/panel-routes.mjs +++ b/src/lib/admin/panel-routes.mjs @@ -35,6 +35,13 @@ import { RuntimeSettings as SettingsRepo } from '../db/settings-state.mjs' import { parseCodexImportPayload, upsertCodexAccount, readCodexAccounts } from '../vm/codex-slot.mjs' import { generateAuthUrl, exchangeAuthCode, normalizeOauthFlavor } from '../oauth/oauth-auth-url.mjs' import { sessionKeyToOAuth, panelImportErrorPayload } from '../oauth/cookie-auth.mjs' +import { + normalizeSubscriptionSessionKey, + queryClaudeSubscription, + subscriptionError, + subscriptionIdentity, +} from '../oauth/claude-subscription.mjs' +import { ClaudeSubscriptionsRepo } from '../db/repos/claude-subscriptions-repo.mjs' import { enrichOauthIdentity } from '../oauth/oauth-identity.mjs' import { generateCodexAuthUrl, exchangeCodexAuthCode } from '../oauth/codex-oauth.mjs' import { @@ -2876,6 +2883,93 @@ export function createPanelHandler(ctx) { }) } + // The billing cookie is separate from worker/OAuth credentials. GET never calls Claude. + if ( + ['GET', 'PUT', 'POST', 'DELETE'].includes(req.method) && + /^\/api\/panel\/vms\/[^/]+\/claude-subscription$/.test(p) + ) { + const id = p.split('/')[4] + const vm = getVm(cfg.paths.project, id) + if (!vm) return json(res, 404, { ok: false, error: { message: 'vm not found' } }) + if (isCodexVm(vm)) return json(res, 400, { ok: false, error: { message: '该功能仅适用于 Claude 订阅账号' } }) + const repo = new ClaudeSubscriptionsRepo() + const identity = subscriptionIdentity(vm) + if (!identity) + return json(res, 400, { + ok: false, + error: { + code: 'subscription_identity_required', + message: subscriptionError('subscription_identity_required').message, + }, + }) + const assertCurrent = () => { + const current = getVm(cfg.paths.project, id) + const live = subscriptionIdentity(current) + if ( + !live || + live.account_uuid !== identity.account_uuid || + live.org_uuid !== identity.org_uuid || + current.owner_user_id !== vm.owner_user_id + ) { + throw subscriptionError('subscription_changed', 409) + } + } + let version + try { + if (req.method === 'GET') return json(res, 200, panel.ok(await repo.view(identity))) + if (req.method === 'DELETE') return json(res, 200, panel.ok(await repo.remove(identity))) + repo.requireEncryption() + const slotProxy = requireSlotProxy(vm) + if (!slotProxy.ok) return json(res, slotProxy.status, { ok: false, error: { message: slotProxy.message } }) + let sessionKey + if (req.method === 'PUT') { + const body = await readBody(req, 8192).catch(() => { + throw subscriptionError('subscription_session_invalid', 400) + }) + sessionKey = normalizeSubscriptionSessionKey(body?.session_key) + } else { + const saved = await repo.credential(identity) + sessionKey = saved.sessionKey + version = saved.version + } + const snapshot = await queryClaudeSubscription({ + vm, + sessionKey, + cfg, + proxyUrl: slotProxy.proxyUrl, + request: ctx.claudeSubscriptionRequest, + }) + assertCurrent() + const result = version + ? await repo.update(identity, version, { snapshot }) + : await repo.save(identity, sessionKey, snapshot) + return json(res, 200, panel.ok(result)) + } catch (error) { + if (!String(error.code || '').startsWith('subscription_')) throw error + try { + assertCurrent() + } catch (changed) { + error = changed + } + let data + try { + data = + version && error.code !== 'subscription_changed' + ? await repo.update(identity, version, { error: error.code }) + : await repo.view(identity) + } catch (changed) { + if (changed.code !== 'subscription_changed') throw changed + error = changed + data = await repo.view(identity) + } + return json(res, error.status || 502, { + ok: false, + error: { code: error.code, message: error.message }, + data, + }) + } + } + // GET /api/panel/vms/:id if (req.method === 'GET' && /^\/api\/panel\/vms\/[^/]+$/.test(p)) { const id = p.split('/').pop() diff --git a/src/lib/db/postgres-migrations/022_claude_subscriptions.sql b/src/lib/db/postgres-migrations/022_claude_subscriptions.sql new file mode 100644 index 000000000..dc8702816 --- /dev/null +++ b/src/lib/db/postgres-migrations/022_claude_subscriptions.sql @@ -0,0 +1,10 @@ +CREATE TABLE claude_subscriptions ( + vm_id text PRIMARY KEY REFERENCES vms(id) ON DELETE CASCADE, + account_uuid text NOT NULL, + org_uuid text NOT NULL, + version uuid NOT NULL, + encrypted_session_key text NOT NULL, + snapshot jsonb NOT NULL, + checked_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_error text +); diff --git a/src/lib/db/repos/claude-subscriptions-repo.mjs b/src/lib/db/repos/claude-subscriptions-repo.mjs new file mode 100644 index 000000000..b0d1d8871 --- /dev/null +++ b/src/lib/db/repos/claude-subscriptions-repo.mjs @@ -0,0 +1,111 @@ +import crypto from 'node:crypto' +import { getDb, isDbOpen } from '../database.mjs' +import { decryptString, encryptString, encryptionEnabled } from '../secure.mjs' +import { subscriptionError, subscriptionIdentity } from '../../oauth/claude-subscription.mjs' + +function matches(row, identity) { + return ( + identity && + row?.vm_id === identity.vm_id && + row.account_uuid === identity.account_uuid && + row.org_uuid === identity.org_uuid + ) +} + +function publicView(row) { + return { + configured: !!row, + ...row?.snapshot, + checked_at: row?.checked_at?.toISOString() || null, + last_error: row?.last_error || null, + } +} + +export class ClaudeSubscriptionsRepo { + constructor(db = getDb()) { + this.db = db + } + + requireEncryption() { + if (!encryptionEnabled()) throw subscriptionError('subscription_encryption_required', 503) + } + + async row(identity) { + const row = await this.db.prepare('SELECT * FROM claude_subscriptions WHERE vm_id=$1').get(identity.vm_id) + return matches(row, identity) ? row : null + } + + async view(identity) { + return { ...publicView(await this.row(identity)), encryption_available: encryptionEnabled() } + } + + async credential(identity) { + this.requireEncryption() + const row = await this.row(identity) + if (!row) throw subscriptionError('subscription_credential_missing', 400) + try { + return { sessionKey: decryptString(row.encrypted_session_key), version: row.version } + } catch { + throw subscriptionError('subscription_session_invalid', 422) + } + } + + async save(identity, sessionKey, snapshot) { + this.requireEncryption() + await this.db + .prepare(`INSERT INTO claude_subscriptions + (vm_id,account_uuid,org_uuid,version,encrypted_session_key,snapshot) + VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (vm_id) DO UPDATE SET + account_uuid=excluded.account_uuid,org_uuid=excluded.org_uuid,version=excluded.version, + encrypted_session_key=excluded.encrypted_session_key,snapshot=excluded.snapshot, + checked_at=clock_timestamp(),last_error=NULL`) + .run( + identity.vm_id, + identity.account_uuid, + identity.org_uuid, + crypto.randomUUID(), + encryptString(sessionKey), + JSON.stringify(snapshot), + ) + return this.view(identity) + } + + async update(identity, version, { snapshot = null, error = null }) { + const result = await this.db + .prepare(`UPDATE claude_subscriptions SET + snapshot=COALESCE($5::jsonb,snapshot),last_error=$6,checked_at=clock_timestamp() + WHERE vm_id=$1 AND account_uuid=$2 AND org_uuid=$3 AND version=$4`) + .run( + identity.vm_id, + identity.account_uuid, + identity.org_uuid, + version, + snapshot ? JSON.stringify(snapshot) : null, + error, + ) + if (result.changes !== 1) throw subscriptionError('subscription_changed', 409) + return this.view(identity) + } + + async remove(identity) { + await this.db + .prepare('DELETE FROM claude_subscriptions WHERE vm_id=$1 AND account_uuid=$2 AND org_uuid=$3') + .run(identity.vm_id, identity.account_uuid, identity.org_uuid) + return this.view(identity) + } +} + +/** Bulk metadata projection; neither decrypts nor publishes cookies. Account changes hide old observations. */ +export async function stampClaudeSubscriptions(vms) { + if (!isDbOpen() || !vms.length) return + const rows = await getDb() + .prepare( + 'SELECT vm_id,account_uuid,org_uuid,snapshot,checked_at,last_error FROM claude_subscriptions WHERE vm_id=ANY($1::text[])', + ) + .all(vms.map((vm) => vm.id)) + const byId = new Map(rows.map((row) => [row.vm_id, row])) + for (const vm of vms) { + const row = byId.get(vm.id) + if (matches(row, subscriptionIdentity(vm))) vm.claude_subscription = publicView(row) + } +} diff --git a/src/lib/oauth/claude-subscription.mjs b/src/lib/oauth/claude-subscription.mjs new file mode 100644 index 000000000..6b80d0687 --- /dev/null +++ b/src/lib/oauth/claude-subscription.mjs @@ -0,0 +1,129 @@ +import { forwardApi, readApiJson } from '../transport/api-kernel-client.mjs' + +const MESSAGES = { + subscription_identity_required: '请先导入包含账号和组织身份的 Claude 凭据', + subscription_session_invalid: '需要 sk-ant-sid 开头的 Claude 网页 sessionKey', + subscription_session_expired: '网页 sessionKey 已失效，请重新保存', + subscription_account_mismatch: 'sessionKey 的账号或组织与当前槽位不一致', + subscription_access_denied: 'Claude 网页访问受限或当前账号无账单读取权限', + subscription_rate_limited: 'Claude 查询频率受限，请稍后重试', + subscription_response_invalid: 'Claude 订阅数据格式无法识别', + subscription_request_failed: 'Claude 订阅查询失败，请稍后重试', + subscription_encryption_required: '保存订阅查询凭据需要配置 VM2API_DB_SECRET 或 KIN_DB_SECRET', + subscription_credential_missing: '请先保存订阅查询 sessionKey', + subscription_changed: '账号或查询凭据已变更，请重新查询', +} + +export function subscriptionError(code, status = 502) { + return Object.assign(new Error(MESSAGES[code] || MESSAGES.subscription_request_failed), { code, status }) +} + +export function subscriptionIdentity(vm) { + const account_uuid = vm?.claude?.account_uuid || vm?.account_uuid + const org_uuid = vm?.claude?.org_uuid || vm?.org_uuid + return account_uuid && org_uuid ? { vm_id: vm.id, account_uuid, org_uuid } : null +} + +export function normalizeSubscriptionSessionKey(value) { + if (typeof value !== 'string' || !/^sk-ant-sid[A-Za-z0-9_-]{1,4096}$/.test(value.trim())) { + throw subscriptionError('subscription_session_invalid', 400) + } + return value.trim() +} + +function billingDate(value) { + if (value == null) return null + if (typeof value !== 'string') throw subscriptionError('subscription_response_invalid') + const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value) + if (!dateOnly && !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(value)) { + throw subscriptionError('subscription_response_invalid') + } + const time = Date.parse(value) + const day = new Date(value.slice(0, 10)) + if (!Number.isFinite(time) || Number.isNaN(day.getTime()) || day.toISOString().slice(0, 10) !== value.slice(0, 10)) { + throw subscriptionError('subscription_response_invalid') + } + return { value: dateOnly ? value : new Date(time).toISOString(), date_only: dateOnly } +} + +export function parseClaudeSubscription(body) { + const fields = ['next_charge_at', 'next_charge_date', 'plan_ending_at', 'plan_ending_before'] + if ( + !body || + !['active', 'trialing', 'canceled'].includes(body.status) || + !fields.every((key) => Object.hasOwn(body, key)) + ) { + throw subscriptionError('subscription_response_invalid') + } + const end = billingDate(body.plan_ending_at) || billingDate(body.plan_ending_before) + const renew = + !end && body.status !== 'canceled' ? billingDate(body.next_charge_at) || billingDate(body.next_charge_date) : null + return { + status: body.status, + expires_at: end?.value || null, + expires_date_only: end?.date_only || false, + renews_at: renew?.value || null, + renews_date_only: renew?.date_only || false, + } +} + +function verifiedOrganization(account, identity) { + if (account?.uuid !== identity.account_uuid || !Array.isArray(account.memberships)) { + throw subscriptionError('subscription_account_mismatch', 409) + } + const membership = account.memberships.find((item) => item?.organization?.uuid === identity.org_uuid) + if (!membership) throw subscriptionError('subscription_account_mismatch', 409) + return membership.organization +} + +function planName(org, account) { + const tier = String(org.rate_limit_tier || org.organization_type || '').toLowerCase() + if (tier.includes('max')) { + const multiplier = tier.match(/(?:^|_)(5|20)x(?:_|$)/)?.[1] + return multiplier ? `Max ${multiplier}x` : 'Max' + } + if (tier.includes('enterprise')) return 'Enterprise' + if (tier.includes('team')) return 'Team' + if (tier.includes('pro')) return 'Pro' + if (account.has_claude_max === true) return 'Max' + if (account.has_claude_pro === true) return 'Pro' + return null +} + +/** Fixed-origin GETs only. Never exchanges the cookie for OAuth or touches inference credentials. */ +export async function queryClaudeSubscription({ vm, sessionKey, cfg, proxyUrl, request }) { + const identity = subscriptionIdentity(vm) + if (!identity) throw subscriptionError('subscription_identity_required', 400) + const cookie = normalizeSubscriptionSessionKey(sessionKey) + const signal = AbortSignal.timeout(15_000) + const transport = + request || + (async (url, options) => { + const response = await forwardApi({ cfg, proxyUrl, url, ...options, body: '', timeoutMs: 15_000 }) + return { status: response.statusCode, body: await readApiJson(response, 256 * 1024) } + }) + async function get(path) { + let response + try { + response = await transport(`https://claude.ai/api${path}`, { + method: 'GET', + headers: { Cookie: `sessionKey=${cookie}`, Accept: 'application/json' }, + signal, + }) + } catch { + throw subscriptionError('subscription_request_failed') + } + if (response.status === 401) throw subscriptionError('subscription_session_expired', 422) + if (response.status === 403) throw subscriptionError('subscription_access_denied') + if (response.status === 429) throw subscriptionError('subscription_rate_limited', 429) + if (response.status !== 200) throw subscriptionError('subscription_request_failed') + return response.body + } + const account = await get('/account') + const org = verifiedOrganization(account, identity) + const dates = parseClaudeSubscription( + await get(`/organizations/${encodeURIComponent(identity.org_uuid)}/subscription_details`), + ) + verifiedOrganization(await get('/account'), identity) + return { ...dates, plan: planName(org, account), fetched_at: new Date().toISOString() } +} diff --git a/test/unit/claude-subscription.test.mjs b/test/unit/claude-subscription.test.mjs new file mode 100644 index 000000000..8396de692 --- /dev/null +++ b/test/unit/claude-subscription.test.mjs @@ -0,0 +1,167 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { + normalizeSubscriptionSessionKey, + parseClaudeSubscription, + queryClaudeSubscription, +} from '../../src/lib/oauth/claude-subscription.mjs' + +const vm = { id: 'vm-fixture', claude: { account_uuid: 'fixture-account', org_uuid: 'fixture-org' } } +const cookie = 'sk-ant-sid01-synthetic-subscription' +const account = { + uuid: 'fixture-account', + memberships: [{ organization: { uuid: 'fixture-org', rate_limit_tier: 'default_claude_max_20x' } }], +} +const details = { + status: 'active', + next_charge_at: null, + next_charge_date: '2026-11-05', + plan_ending_at: null, + plan_ending_before: null, +} + +test('renewal and scheduled expiration keep their meaning and calendar precision', () => { + assert.deepEqual(parseClaudeSubscription(details), { + status: 'active', + expires_at: null, + expires_date_only: false, + renews_at: '2026-11-05', + renews_date_only: true, + }) + const expires = parseClaudeSubscription({ + ...details, + plan_ending_at: '2026-11-06T01:02:03Z', + plan_ending_before: '2026-11-07', + }) + assert.equal(expires.expires_at, '2026-11-06T01:02:03.000Z') + assert.equal(expires.renews_at, null) + assert.equal(expires.expires_date_only, false) + assert.equal(parseClaudeSubscription({ ...details, status: 'canceled' }).renews_at, null) + assert.equal( + parseClaudeSubscription({ ...details, status: 'canceled', plan_ending_before: '2026-11-05' }).expires_date_only, + true, + ) +}) + +test('missing dates remain unknown; malformed and unrelated dates are rejected', () => { + const empty = parseClaudeSubscription({ ...details, next_charge_date: null }) + assert.equal(empty.renews_at, null) + for (const body of [ + {}, + { status: 'active', resets_at: '2026-11-05' }, + { ...details, status: 'unknown' }, + { ...details, next_charge_date: '2026-02-30' }, + { ...details, next_charge_date: '2026-02-30T01:02:03Z' }, + { ...details, next_charge_date: 123 }, + { ...details, next_charge_date: '2026-11-05T01:02:03' }, + ]) { + assert.throws(() => parseClaudeSubscription(body), { code: 'subscription_response_invalid' }) + } +}) + +test('only a single web session cookie value is accepted, without header injection', () => { + assert.equal(normalizeSubscriptionSessionKey(` ${cookie} `), cookie) + for (const key of [ + 'sk-ant-api03-synthetic', + 'sk-ant-oat01-synthetic', + `sessionKey=${cookie}`, + `${cookie}; other=x`, + `${cookie}\r\nX-Header: x`, + null, + ]) { + assert.throws(() => normalizeSubscriptionSessionKey(key), { code: 'subscription_session_invalid' }) + } +}) + +test('subscription query sends exactly three fixed-origin GETs and verifies account and organization', async () => { + const requests = [] + const result = await queryClaudeSubscription({ + vm, + sessionKey: cookie, + request: async (url, options) => { + requests.push(url) + assert.equal(options.method, 'GET') + assert.deepEqual(options.headers, { Cookie: `sessionKey=${cookie}`, Accept: 'application/json' }) + return { status: 200, body: url.endsWith('/account') ? account : details } + }, + }) + assert.deepEqual(requests, [ + 'https://claude.ai/api/account', + 'https://claude.ai/api/organizations/fixture-org/subscription_details', + 'https://claude.ai/api/account', + ]) + assert.equal(result.plan, 'Max 20x') + assert.equal(result.renews_at, '2026-11-05') + assert.equal(JSON.stringify(result).includes(cookie), false) +}) + +test('foreign account or organization is rejected before any billing request', async () => { + for (const body of [ + { ...account, uuid: 'other-account' }, + { ...account, memberships: [] }, + ]) { + let calls = 0 + await assert.rejects( + queryClaudeSubscription({ + vm, + sessionKey: cookie, + request: async () => { + calls += 1 + return { status: 200, body } + }, + }), + { code: 'subscription_account_mismatch' }, + ) + assert.equal(calls, 1) + } +}) + +test('a web account switch during billing is rejected', async () => { + let calls = 0 + await assert.rejects( + queryClaudeSubscription({ + vm, + sessionKey: cookie, + request: async () => { + calls += 1 + return { + status: 200, + body: calls === 2 ? details : calls === 3 ? { ...account, uuid: 'other-account' } : account, + } + }, + }), + { code: 'subscription_account_mismatch' }, + ) +}) + +test('HTTP and network errors never include the cookie or upstream response text', async () => { + for (const [status, code] of [ + [401, 'subscription_session_expired'], + [403, 'subscription_access_denied'], + [429, 'subscription_rate_limited'], + [500, 'subscription_request_failed'], + [302, 'subscription_request_failed'], + ]) { + await assert.rejects( + queryClaudeSubscription({ vm, sessionKey: cookie, request: async () => ({ status, body: cookie }) }), + (error) => { + assert.equal(error.code, code) + assert.equal(error.message.includes(cookie), false) + return true + }, + ) + } + await assert.rejects( + queryClaudeSubscription({ + vm, + sessionKey: cookie, + request: async () => { + throw new Error(cookie) + }, + }), + (error) => { + assert.equal(error.message.includes(cookie), false) + return error.code === 'subscription_request_failed' + }, + ) +}) diff --git a/test/unit/claude-subscriptions-repo.test.mjs b/test/unit/claude-subscriptions-repo.test.mjs new file mode 100644 index 000000000..c8ada4e27 --- /dev/null +++ b/test/unit/claude-subscriptions-repo.test.mjs @@ -0,0 +1,98 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { getDb } from '../../src/lib/db/database.mjs' +import { ClaudeSubscriptionsRepo, stampClaudeSubscriptions } from '../../src/lib/db/repos/claude-subscriptions-repo.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' + +const identity = { vm_id: 'vm-subscription', account_uuid: 'fixture-account', org_uuid: 'fixture-org' } +const secret = 'sk-ant-sid01-synthetic-private-cookie' +const snapshot = { + plan: 'Max 5x', + expires_at: '2026-11-05', + expires_date_only: true, + fetched_at: '2026-10-27T00:00:00Z', +} + +async function fixture(t) { + const saved = { VM2API_DB_SECRET: process.env.VM2API_DB_SECRET, KIN_DB_SECRET: process.env.KIN_DB_SECRET } + t.after(() => { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }) + delete process.env.VM2API_DB_SECRET + process.env.KIN_DB_SECRET = 'synthetic-test-encryption-key' + const db = getDb() + await new VmsRepo(db).upsertFromVmJson({ id: identity.vm_id, claude: identity }) + return new ClaudeSubscriptionsRepo(db) +} + +test('subscription cookie is encrypted, survives a new repository instance, and never enters public metadata', async (t) => { + const repo = await fixture(t) + await repo.save(identity, secret, snapshot) + const stored = await getDb().prepare('SELECT * FROM claude_subscriptions').get() + assert.ok(stored.encrypted_session_key.startsWith('enc:v1:')) + assert.equal(JSON.stringify(stored).includes(secret), false) + assert.equal((await new ClaudeSubscriptionsRepo().credential(identity)).sessionKey, secret) + const view = await repo.view(identity) + assert.equal(view.plan, 'Max 5x') + assert.equal(view.configured, true) + assert.equal('encrypted_session_key' in view, false) + assert.equal(JSON.stringify(view).includes(secret), false) + const vms = [{ id: identity.vm_id, account_uuid: identity.account_uuid, org_uuid: identity.org_uuid }] + await stampClaudeSubscriptions(vms) + assert.equal(vms[0].claude_subscription.expires_at, '2026-11-05') + assert.equal(JSON.stringify(vms).includes(secret), false) +}) + +test('saving a cookie requires configured encryption and never falls back to plaintext', async (t) => { + const repo = await fixture(t) + delete process.env.KIN_DB_SECRET + delete process.env.VM2API_DB_SECRET + await assert.rejects(repo.save(identity, secret, snapshot), { code: 'subscription_encryption_required' }) + assert.equal((await repo.view(identity)).configured, false) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM claude_subscriptions').get()).n, 0) +}) + +test('reusing the slot for another account or organization cannot read or publish the previous subscription', async (t) => { + const repo = await fixture(t) + await repo.save(identity, secret, snapshot) + for (const changed of [{ account_uuid: 'other-account' }, { org_uuid: 'other-org' }]) { + assert.equal((await repo.view({ ...identity, ...changed })).configured, false) + await assert.rejects(repo.credential({ ...identity, ...changed }), { code: 'subscription_credential_missing' }) + const vms = [{ id: identity.vm_id, account_uuid: identity.account_uuid, org_uuid: identity.org_uuid, ...changed }] + await stampClaudeSubscriptions(vms) + assert.equal(vms[0].claude_subscription, undefined) + } +}) + +test('refresh failure retains the last successful dates and records the failure separately', async (t) => { + const repo = await fixture(t) + await repo.save(identity, secret, snapshot) + const { version } = await repo.credential(identity) + const view = await repo.update(identity, version, { error: 'subscription_session_expired' }) + assert.equal(view.expires_at, snapshot.expires_at) + assert.equal(view.fetched_at, snapshot.fetched_at) + assert.equal(view.last_error, 'subscription_session_expired') +}) + +test('a delayed refresh cannot overwrite a replaced or deleted cookie', async (t) => { + const repo = await fixture(t) + await repo.save(identity, secret, snapshot) + const { version } = await repo.credential(identity) + await repo.save(identity, `${secret}-new`, { ...snapshot, plan: 'Pro' }) + await assert.rejects(repo.update(identity, version, { snapshot }), { code: 'subscription_changed' }) + assert.equal((await repo.view(identity)).plan, 'Pro') + const replacement = await repo.credential(identity) + await repo.remove(identity) + await assert.rejects(repo.update(identity, replacement.version, { snapshot }), { code: 'subscription_changed' }) + assert.equal((await repo.view(identity)).configured, false) +}) + +test('deleting a VM cascades only its saved subscription credential and snapshot', async (t) => { + const repo = await fixture(t) + await repo.save(identity, secret, snapshot) + await new VmsRepo().remove(identity.vm_id) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM claude_subscriptions').get()).n, 0) +}) diff --git a/test/unit/database.test.mjs b/test/unit/database.test.mjs index b46995b07..e2ba8e7d5 100644 --- a/test/unit/database.test.mjs +++ b/test/unit/database.test.mjs @@ -35,7 +35,7 @@ after(async () => { test('PostgreSQL baseline includes business and historical tables and default groups', async () => { const db = await database() const rows = await db.prepare('SELECT table_name FROM information_schema.tables WHERE table_schema=$1').all(db.schema) - assert.equal(rows.length, 44) + assert.equal(rows.length, 45) for (const name of [ 'settings', 'users', @@ -59,6 +59,7 @@ test('PostgreSQL baseline includes business and historical tables and default gr 'background_task_runs', 'oauth_pkce_sessions', 'shell_tickets', + 'claude_subscriptions', ]) { assert.ok( rows.some((r) => r.table_name === name), @@ -119,7 +120,7 @@ test('concurrent initialization serializes migrations and checksums are enforced const opts = options() const [a, b] = await Promise.all([createDatabase(opts), createDatabase(opts)]) dbs.push(a, b) - assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 21) + assert.equal((await a.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 22) await a.prepare("UPDATE schema_migrations SET checksum='invalid'").run() await assert.rejects(applyMigrations(b), /checksum mismatch/) }) @@ -135,7 +136,7 @@ test('failed migrations rollback DDL and migration records', async () => { } await assert.rejects(applyMigrations(db, { migrationsDir: dir }), /999_bad.sql failed/) assert.equal((await db.prepare('SELECT to_regclass($1) AS name').get(db.schema + '.partial_write')).name, null) - assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 21) + assert.equal((await db.prepare('SELECT count(*) AS n FROM schema_migrations').get()).n, 22) fs.rmSync(dir, { recursive: true, force: true }) }) diff --git a/test/unit/panel-claude-subscription.test.mjs b/test/unit/panel-claude-subscription.test.mjs new file mode 100644 index 000000000..37e2ad68e --- /dev/null +++ b/test/unit/panel-claude-subscription.test.mjs @@ -0,0 +1,177 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { createPanelHandler } from '../../src/lib/admin/panel-routes.mjs' +import { buildVmList } from '../../src/lib/admin/panel-api.mjs' +import { getDb } from '../../src/lib/db/database.mjs' +import { VmsRepo } from '../../src/lib/db/repos/vms-repo.mjs' +import { authorizePanelRoute } from '../../src/lib/admin/panel-acl.mjs' + +const cookie = 'sk-ant-sid01-synthetic-billing-cookie' +const accountId = '11111111-1111-4111-8111-111111111111' +const orgId = '22222222-2222-4222-8222-222222222222' +const details = { + status: 'active', + next_charge_at: null, + next_charge_date: '2026-11-05', + plan_ending_at: null, + plan_ending_before: null, +} + +async function fixture(t) { + const previous = process.env.KIN_DB_SECRET + process.env.KIN_DB_SECRET = 'synthetic-subscription-test-key' + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-subscription-')) + t.after(() => { + if (previous === undefined) delete process.env.KIN_DB_SECRET + else process.env.KIN_DB_SECRET = previous + fs.rmSync(root, { recursive: true, force: true }) + }) + const vm = { + id: 'vm-billing', + status: 'stopped', + owner_user_id: 'tenant-a', + claude: { + account_uuid: accountId, + org_uuid: orgId, + email: 'fixture@example.com', + has_access: true, + account_tier: 'max', + }, + } + fs.mkdirSync(path.join(root, 'vms', vm.id, 'cli-home', '.claude'), { recursive: true }) + const recordPath = path.join(root, 'vms', `${vm.id}.json`) + fs.writeFileSync(recordPath, JSON.stringify(vm)) + const credentialPath = path.join(root, 'vms', vm.id, 'cli-home', '.claude', 'credentials.json') + fs.writeFileSync( + credentialPath, + JSON.stringify({ claudeAiOauth: { accessToken: 'synthetic-inference-token', refreshToken: 'synthetic-refresh' } }), + ) + await new VmsRepo().upsertFromVmJson(vm) + const state = { status: 200, calls: [], body: {}, response: {}, beforeReturn: null } + const cfg = { paths: { project: root }, limits: { max_body_bytes: 8192 } } + const handler = createPanelHandler({ + cfg, + requireAuth: () => true, + json: (_res, status, body) => { + state.response = { status, body } + return true + }, + readBody: async () => state.body, + requireSlotProxy: () => ({ ok: true, proxyUrl: 'socks5://synthetic.invalid:1080' }), + claudeSubscriptionRequest: async (url) => { + state.calls.push(url) + await state.beforeReturn?.(url) + return { + status: state.status, + body: url.endsWith('/account') + ? { + uuid: accountId, + memberships: [{ organization: { uuid: orgId, rate_limit_tier: 'default_claude_max_5x' } }], + } + : details, + } + }, + }) + return { + state, + root, + vm, + cfg, + recordPath, + credentialPath, + request: async (method, body = {}, role = 'admin', userId = 'tenant-a') => { + state.body = body + const req = { method, panelRole: role, panelUserId: userId, panelUser: 'fixture-user' } + await handler(req, {}, new URL(`http://fixture/api/panel/vms/${vm.id}/claude-subscription`)) + return state.response + }, + } +} + +test('subscription management ACL allows owned-user mutations and super metadata only', () => { + const url = '/api/panel/vms/vm-billing/claude-subscription' + for (const method of ['GET', 'PUT', 'POST', 'DELETE']) { + assert.equal(authorizePanelRoute(method, url, 'user').ok, true) + assert.equal(authorizePanelRoute(method, url, 'super').ok, method === 'GET') + } +}) + +test('save, read, replace, refresh and delete affect only the subscription credential', async (t) => { + const f = await fixture(t) + const beforeRecord = fs.readFileSync(f.recordPath, 'utf8') + const beforeCredential = fs.readFileSync(f.credentialPath, 'utf8') + let result = await f.request('PUT', { session_key: cookie }) + assert.equal(result.status, 200, JSON.stringify(result)) + assert.equal(result.body.data.plan, 'Max 5x') + assert.equal(result.body.data.renews_at, '2026-11-05') + assert.equal(JSON.stringify(result).includes(cookie), false) + const calls = f.state.calls.length + result = await f.request('GET') + assert.equal(f.state.calls.length, calls, 'GET metadata must not call Claude') + assert.equal(result.body.data.configured, true) + assert.equal((await f.request('PUT', { session_key: `${cookie}-replacement` })).status, 200) + assert.equal((await f.request('POST')).status, 200) + const list = await buildVmList({ cfg: f.cfg }) + assert.equal(list.data.items[0].claude_subscription.plan, 'Max 5x') + assert.equal(JSON.stringify(list).includes(cookie), false) + result = await f.request('DELETE') + assert.equal(result.status, 200) + assert.equal(result.body.data.configured, false) + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM claude_subscriptions').get()).n, 0) + assert.equal(fs.readFileSync(f.recordPath, 'utf8'), beforeRecord) + assert.equal(fs.readFileSync(f.credentialPath, 'utf8'), beforeCredential) +}) + +test('expired stored cookie records a query error without changing OAuth or previously captured dates', async (t) => { + const f = await fixture(t) + await f.request('PUT', { session_key: cookie }) + f.state.status = 401 + const result = await f.request('POST') + assert.equal(result.status, 422) + assert.equal(result.body.error.code, 'subscription_session_expired') + assert.equal(result.body.data.renews_at, '2026-11-05') + assert.equal(result.body.data.last_error, 'subscription_session_expired') + assert.equal(JSON.stringify(result).includes(cookie), false) +}) + +test('foreign tenants cannot read, replace, refresh or delete the saved billing cookie', async (t) => { + const f = await fixture(t) + await f.request('PUT', { session_key: cookie }) + const calls = f.state.calls.length + for (const method of ['GET', 'PUT', 'POST', 'DELETE']) { + assert.equal((await f.request(method, { session_key: cookie }, 'user', 'tenant-b')).status, 404) + } + assert.equal(f.state.calls.length, calls) + assert.equal((await f.request('GET')).body.data.configured, true) +}) + +test('a slot account change while querying discards the result and never publishes it to the new account', async (t) => { + const f = await fixture(t) + f.state.beforeReturn = (url) => { + if (url.endsWith('/subscription_details')) { + fs.writeFileSync( + f.recordPath, + JSON.stringify({ ...f.vm, claude: { ...f.vm.claude, account_uuid: 'other-account' } }), + ) + } + } + const result = await f.request('PUT', { session_key: cookie }) + assert.equal(result.status, 409, JSON.stringify(result)) + assert.equal(result.body.error.code, 'subscription_changed') + assert.equal((await getDb().prepare('SELECT count(*) AS n FROM claude_subscriptions').get()).n, 0) +}) + +test('invalid replacement input keeps the previously saved credential', async (t) => { + const f = await fixture(t) + await f.request('PUT', { session_key: cookie }) + const before = await getDb().prepare('SELECT version,encrypted_session_key FROM claude_subscriptions').get() + const result = await f.request('PUT', { session_key: 'sk-ant-api03-not-a-cookie' }) + assert.equal(result.status, 400) + assert.deepEqual( + await getDb().prepare('SELECT version,encrypted_session_key FROM claude_subscriptions').get(), + before, + ) +}) diff --git a/web/src/features/vm/claude-subscription-button.tsx b/web/src/features/vm/claude-subscription-button.tsx new file mode 100644 index 000000000..7cacb703f --- /dev/null +++ b/web/src/features/vm/claude-subscription-button.tsx @@ -0,0 +1,180 @@ +import { useRef, useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import type { ClaudeSubscription, Vm } from '@/types/panel-vm' +import { CalendarDays } from 'lucide-react' +import { toast } from 'sonner' +import { api } from '@/lib/api' +import { isCodexVm } from '@/lib/vm-kind' +import { Button } from '@/components/ui/button' +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { dashboardQueryOptions } from '@/features/overview/queries' +import { + SUBSCRIPTION_ERRORS, + subscriptionDate, + subscriptionDateText, +} from '@/features/vm/claude-subscription' +import { vmQueryOptions, vmsListQueryOptions } from '@/features/vm/queries' +import { useVmPerms } from '@/features/vm/vm-perms' + +export function ClaudeSubscriptionButton({ vm }: { vm: Vm }) { + const [open, setOpen] = useState(false) + const perms = useVmPerms() + if (isCodexVm(vm) || !perms.credential) return null + return ( + + + + + + + Claude 订阅信息 + + 保存当前账号的网页 sessionKey，用于查询订阅信息。 + + + {open ? ( + + ) : null} + + + ) +} + +function SubscriptionForm({ vm }: { vm: Vm }) { + const qc = useQueryClient() + const input = useRef(null) + const [hasInput, setHasInput] = useState(false) + const path = `/api/panel/vms/${encodeURIComponent(vm.id)}/claude-subscription` + const queryKey = [ + 'panel', + 'vm', + vm.id, + 'subscription', + vm.account_uuid, + vm.org_uuid, + path, + ] as const + const query = useQuery({ + queryKey, + queryFn: () => api(path), + retry: false, + }) + const data = query.data || vm.claude_subscription + const refresh = async () => { + await Promise.all([ + qc.invalidateQueries({ queryKey }), + qc.invalidateQueries({ queryKey: vmQueryOptions(vm.id).queryKey }), + qc.invalidateQueries({ queryKey: vmsListQueryOptions().queryKey }), + qc.invalidateQueries({ queryKey: dashboardQueryOptions().queryKey }), + ]) + } + const mutation = useMutation({ + mutationFn: (method: 'PUT' | 'POST' | 'DELETE') => { + const sessionKey = + method === 'PUT' ? input.current?.value.trim() : undefined + if (input.current) input.current.value = '' + setHasInput(false) + return api(path, { + method, + body: + method === 'PUT' + ? JSON.stringify({ session_key: sessionKey }) + : undefined, + }) + }, + onSuccess: (_data, method) => { + toast.success( + method === 'DELETE' ? '已删除查询凭据和订阅记录' : '已更新订阅信息' + ) + }, + onError: (error: Error) => toast.error(error.message), + onSettled: refresh, + }) + const busy = mutation.isPending + const canSave = + !!vm.account_uuid && !!vm.org_uuid && data?.encryption_available !== false + return ( +
+
+

当前套餐：{data?.plan || '未知'}

+

{subscriptionDateText(data)}

+

+ 最近成功查询： + {data?.fetched_at ? subscriptionDate(data.fetched_at) : '未查询'} +

+ {data?.last_error ? ( +

+ {SUBSCRIPTION_ERRORS[data.last_error] || '订阅查询失败'} + ；以上为上次查询结果。 +

+ ) : null} +
+ {query.error ? ( +

+ {query.error.message} +

+ ) : null} +
+ + setHasInput(!!event.target.value.trim())} + /> +

+ 仅用于核对账号和读取订阅信息。保存后不回显。 +

+ {data?.encryption_available === false ? ( +

+ 服务端尚未配置数据库加密密钥，暂不能保存。 +

+ ) : null} +
+
+ + + +
+
+ ) +} diff --git a/web/src/features/vm/claude-subscription.test.ts b/web/src/features/vm/claude-subscription.test.ts new file mode 100644 index 000000000..d54315559 --- /dev/null +++ b/web/src/features/vm/claude-subscription.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { subscriptionDate, subscriptionDateText } from './claude-subscription' + +describe('Claude subscription dates', () => { + it('preserves date-only billing values without a timezone conversion', () => { + expect(subscriptionDate('2026-11-05', true)).toBe('2026-11-05') + expect( + subscriptionDateText({ + configured: true, + expires_at: '2026-11-05', + expires_date_only: true, + }) + ).toBe('到期：2026-11-05') + }) + it('distinguishes renewal, expiration and unknown dates', () => { + expect( + subscriptionDateText({ + configured: true, + renews_at: '2026-11-05', + renews_date_only: true, + }) + ).toBe('续费：2026-11-05') + expect(subscriptionDateText({ configured: true })).toBe('订阅日期：未知') + expect(subscriptionDateText(undefined)).toBe('订阅日期：未获取') + }) +}) diff --git a/web/src/features/vm/claude-subscription.ts b/web/src/features/vm/claude-subscription.ts new file mode 100644 index 000000000..3cf2d6306 --- /dev/null +++ b/web/src/features/vm/claude-subscription.ts @@ -0,0 +1,34 @@ +import type { ClaudeSubscription } from '@/types/panel-vm' + +export function subscriptionDate( + value: string | null | undefined, + dateOnly = false +) { + if (!value) return '未知' + if (dateOnly) return value.slice(0, 10) + const date = new Date(value) + return Number.isNaN(date.getTime()) + ? '未知' + : date.toLocaleString('zh-CN', { hour12: false }) +} + +export function subscriptionDateText( + subscription: ClaudeSubscription | undefined +) { + if (subscription?.expires_at) { + return `到期：${subscriptionDate(subscription.expires_at, subscription.expires_date_only)}` + } + if (subscription?.renews_at) { + return `续费：${subscriptionDate(subscription.renews_at, subscription.renews_date_only)}` + } + return subscription?.configured ? '订阅日期：未知' : '订阅日期：未获取' +} + +export const SUBSCRIPTION_ERRORS: Record = { + subscription_session_expired: '网页 sessionKey 已失效，请重新保存', + subscription_account_mismatch: '网页账号或组织与当前槽位不一致', + subscription_access_denied: '网页访问受限或无账单读取权限', + subscription_rate_limited: '查询频率受限，请稍后重试', + subscription_response_invalid: '订阅数据格式无法识别', + subscription_request_failed: '订阅查询失败，请稍后重试', +} diff --git a/web/src/features/vm/detail-account-tab.tsx b/web/src/features/vm/detail-account-tab.tsx index f8a03c0f2..e6ddc2b57 100644 --- a/web/src/features/vm/detail-account-tab.tsx +++ b/web/src/features/vm/detail-account-tab.tsx @@ -23,6 +23,7 @@ import { import { StatusMark } from '@/components/status-mark' import { AllowedModelsCard } from '@/features/vm/allowed-models-card' import { AuthSchemeEditor } from '@/features/vm/auth-scheme-editor' +import { ClaudeSubscriptionButton } from '@/features/vm/claude-subscription-button' import { CodexCredentialPanel } from '@/features/vm/codex-credential-panel' import { ConvertOauthToSetupButton } from '@/features/vm/convert-oauth-button' import { CredentialEditorButton } from '@/features/vm/credential-editor' @@ -216,6 +217,7 @@ export function VmAccountTab(props: VmAccountTabProps) { )}
+ {gpt ? null : ( 刷新令牌 + ) : null}
) } diff --git a/web/src/features/vm/vm-rack-blade.tsx b/web/src/features/vm/vm-rack-blade.tsx index 32e01a614..0fd4467be 100644 --- a/web/src/features/vm/vm-rack-blade.tsx +++ b/web/src/features/vm/vm-rack-blade.tsx @@ -11,7 +11,7 @@ import { credEndpointLabel, credEndpointOf } from '@/lib/cred-type' import { fmtAgo, fmtNum, fmtUsd } from '@/lib/format' import { tierVisual } from '@/lib/tier-visual' import { cn } from '@/lib/utils' -import { slotCredentialLabel } from '@/lib/vm-kind' +import { isCodexVm, slotCredentialLabel } from '@/lib/vm-kind' import { claudeTier, credentialStatus, @@ -23,6 +23,7 @@ import { import { vmTotalCost, vmTotalTokens } from '@/lib/vm-usage' import { StatusMark } from '@/components/status-mark' import { ProxyChip } from '@/features/proxies/proxy-chip' +import { subscriptionDateText } from '@/features/vm/claude-subscription' import { runtimeTypeLabel, runtimeTypeOf } from '@/features/vm/machine-spec' import { SchedulableSwitch, @@ -66,7 +67,7 @@ export function tierStyle(vm: Vm): CSSProperties | undefined { export function TierBadge({ vm, className }: { vm: Vm; className?: string }) { const skin = tierVisual(vm) - if (skin.key === 'none') return null + if (skin.key === 'none' && !vm.claude_subscription?.plan) return null return ( - {claudeTier(vm).label} + {vm.claude_subscription?.plan || claudeTier(vm).label} ) } @@ -452,6 +453,12 @@ export const VmBlade = memo(function VmBlade({ > {slotCredentialLabel(vm)} + {!isCodexVm(vm) ? ( + + {subscriptionDateText(vm.claude_subscription)} + {vm.claude_subscription?.last_error ? ' · 待更新' : ''} + + ) : null} {restricted && state !== 'dead' ? (
- - {slotCredentialLabel(vm, true)} - +
+
+ {slotCredentialLabel(vm, true)} +
+ {!isCodexVm(vm) ? ( +
+ {subscriptionDateText(vm.claude_subscription)} + {vm.claude_subscription?.last_error ? ' · 待更新' : ''} +
+ ) : null} +
{state === 'live' ? ( {vmWindows(vm).map((w) => ( diff --git a/web/src/types/panel-vm.ts b/web/src/types/panel-vm.ts index 0fca2739f..6fb0fcaf4 100644 --- a/web/src/types/panel-vm.ts +++ b/web/src/types/panel-vm.ts @@ -188,7 +188,22 @@ export type VmPhysicalFeatures = { runtimes?: string | null } +export type ClaudeSubscription = { + configured: boolean + encryption_available?: boolean + plan?: string | null + status?: string + expires_at?: string | null + expires_date_only?: boolean + renews_at?: string | null + renews_date_only?: boolean + fetched_at?: string | null + checked_at?: string | null + last_error?: string | null +} + export type Vm = { + claude_subscription?: ClaudeSubscription id: string name?: string owner_user_id?: string | null